diff --git a/.evergreen-functions.yml b/.evergreen-functions.yml index 00e95d8f1..36a498810 100644 --- a/.evergreen-functions.yml +++ b/.evergreen-functions.yml @@ -538,7 +538,43 @@ functions: shell: bash <<: *e2e_include_expansions_in_env working_dir: src/github.com/mongodb/mongodb-kubernetes - binary: scripts/evergreen/run_python.sh pipeline.py --include ${image_name} --parallel --sign + binary: scripts/evergreen/run_python.sh scripts/release/main.py --parallel ${image_name} + + legacy_pipeline: + - *switch_context + - command: shell.exec + type: setup + params: + shell: bash + script: | + # Docker Hub workaround + # docker buildx needs the moby/buildkit image when setting up a builder so we pull it from our mirror + docker buildx create --driver=docker-container --driver-opt=image=268558157000.dkr.ecr.eu-west-1.amazonaws.com/docker-hub-mirrors/moby/buildkit:buildx-stable-1 --use + docker buildx inspect --bootstrap + - command: ec2.assume_role + display_name: Assume IAM role with permissions to pull Kondukto API token + params: + role_arn: ${kondukto_role_arn} + - command: shell.exec + display_name: Pull Kondukto API token from AWS Secrets Manager and write it to file + params: + silent: true + shell: bash + include_expansions_in_env: [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN] + script: | + set -e + # use AWS CLI to get the Kondukto API token from AWS Secrets Manager + kondukto_token=$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text) + # write the KONDUKTO_TOKEN environment variable to Silkbomb environment file + echo "KONDUKTO_TOKEN=$kondukto_token" > ${workdir}/silkbomb.env + - command: subprocess.exec + retry_on_failure: true + type: setup + params: + shell: bash + <<: *e2e_include_expansions_in_env + working_dir: src/github.com/mongodb/mongodb-kubernetes + binary: scripts/evergreen/run_python.sh pipeline.py --parallel ${image_name} --sign teardown_cloud_qa_all: - *switch_context diff --git a/.evergreen-periodic-builds.yaml b/.evergreen-periodic-builds.yaml index d1a6cd928..a2d799ee7 100644 --- a/.evergreen-periodic-builds.yaml +++ b/.evergreen-periodic-builds.yaml @@ -21,7 +21,7 @@ variables: tasks: - name: periodic_build_operator commands: - - func: pipeline + - func: legacy_pipeline vars: image_name: operator-daily @@ -35,49 +35,49 @@ tasks: - name: periodic_build_init_appdb commands: - - func: pipeline + - func: legacy_pipeline vars: image_name: init-appdb-daily - name: periodic_build_init_database commands: - - func: pipeline + - func: legacy_pipeline vars: image_name: init-database-daily - name: periodic_build_init_opsmanager commands: - - func: pipeline + - func: legacy_pipeline vars: image_name: init-ops-manager-daily - name: periodic_build_database commands: - - func: pipeline + - func: legacy_pipeline vars: image_name: database-daily - name: periodic_build_sbom_cli commands: - - func: pipeline + - func: legacy_pipeline vars: image_name: cli - name: periodic_build_ops_manager_6 commands: - - func: pipeline + - func: legacy_pipeline vars: image_name: ops-manager-6-daily - name: periodic_build_ops_manager_7 commands: - - func: pipeline + - func: legacy_pipeline vars: image_name: ops-manager-7-daily - name: periodic_build_ops_manager_8 commands: - - func: pipeline + - func: legacy_pipeline vars: image_name: ops-manager-8-daily @@ -91,7 +91,7 @@ tasks: exec_timeout_secs: 43200 commands: - func: enable_QEMU - - func: pipeline + - func: legacy_pipeline vars: image_name: mongodb-agent-daily @@ -99,7 +99,7 @@ tasks: exec_timeout_secs: 43200 commands: - func: enable_QEMU - - func: pipeline + - func: legacy_pipeline vars: image_name: mongodb-agent-1-daily @@ -107,19 +107,19 @@ tasks: - name: periodic_build_community_operator commands: - func: enable_QEMU - - func: pipeline + - func: legacy_pipeline vars: image_name: mongodb-kubernetes-operator-daily - name: periodic_build_readiness_probe commands: - - func: pipeline + - func: legacy_pipeline vars: image_name: readinessprobe-daily - name: periodic_build_version_upgrade_post_start_hook commands: - - func: pipeline + - func: legacy_pipeline vars: image_name: operator-version-upgrade-post-start-hook-daily diff --git a/.evergreen.yml b/.evergreen.yml index 749dc39bd..28dfc8b1a 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -283,7 +283,7 @@ tasks: - func: setup_building_host - func: quay_login - func: setup_docker_sbom - - func: pipeline + - func: legacy_pipeline vars: image_name: operator @@ -296,7 +296,7 @@ tasks: - func: setup_building_host - func: quay_login - func: setup_docker_sbom - - func: pipeline + - func: legacy_pipeline vars: image_name: init-appdb @@ -308,7 +308,7 @@ tasks: - func: setup_building_host - func: quay_login - func: setup_docker_sbom - - func: pipeline + - func: legacy_pipeline vars: image_name: init-database @@ -320,7 +320,7 @@ tasks: - func: setup_building_host - func: quay_login - func: setup_docker_sbom - - func: pipeline + - func: legacy_pipeline vars: image_name: init-ops-manager @@ -332,7 +332,7 @@ tasks: - func: setup_building_host - func: quay_login - func: setup_docker_sbom - - func: pipeline + - func: legacy_pipeline vars: image_name: agent @@ -345,7 +345,7 @@ tasks: - func: setup_building_host - func: quay_login - func: setup_docker_sbom - - func: pipeline + - func: legacy_pipeline vars: image_name: agent-pct include_tags: release @@ -390,7 +390,7 @@ tasks: commands: - func: clone - func: setup_building_host - - func: pipeline + - func: legacy_pipeline vars: image_name: agent-pct skip_tags: release @@ -405,7 +405,7 @@ tasks: commands: - func: clone - func: setup_building_host - - func: pipeline + - func: legacy_pipeline vars: image_name: agent-pct skip_tags: release @@ -549,7 +549,7 @@ tasks: - func: setup_building_host - func: quay_login - func: setup_docker_sbom - - func: pipeline + - func: legacy_pipeline vars: image_name: database @@ -568,7 +568,7 @@ tasks: - func: setup_building_host - func: quay_login - func: setup_docker_sbom - - func: pipeline + - func: legacy_pipeline vars: image_name: ops-manager include_tags: release diff --git a/.gitignore b/.gitignore index c5ca572c5..9e3cb309c 100644 --- a/.gitignore +++ b/.gitignore @@ -44,11 +44,6 @@ public/architectures/**/secrets/* docker/mongodb-kubernetes-appdb/content/readinessprobe mongodb-kubernetes -docker/mongodb-kubernetes-operator/Dockerfile -docker/mongodb-kubernetes-database/Dockerfile -docker/mongodb-enterprise-ops-manager/Dockerfile -docker/mongodb-kubernetes-init-database/Dockerfile -docker/mongodb-kubernetes-init-ops-manager/Dockerfile docker/mongodb-kubernetes-operator/content/mongodb-kubernetes-operator.tar docker/mongodb-kubernetes-tests/helm_chart/ docker/mongodb-kubernetes-tests/public/ diff --git a/docker/mongodb-agent-non-matrix/Dockerfile b/docker/mongodb-agent-non-matrix/Dockerfile index e1c1caff2..0677126fd 100644 --- a/docker/mongodb-agent-non-matrix/Dockerfile +++ b/docker/mongodb-agent-non-matrix/Dockerfile @@ -1,5 +1,14 @@ -ARG imagebase -FROM ${imagebase} as base +FROM scratch AS base + +ARG agent_version +ARG agent_distro +ARG tools_version +ARG tools_distro + +ADD https://mciuploads.s3.amazonaws.com/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-${agent_version}.${agent_distro}.tar.gz /data/mongodb-agent.tar.gz +ADD https://downloads.mongodb.org/tools/db/mongodb-database-tools-${tools_distro}-${tools_version}.tgz /data/mongodb-tools.tgz + +COPY ./docker/mongodb-kubernetes-init-database/content/LICENSE /data/LICENSE FROM registry.access.redhat.com/ubi9/ubi-minimal diff --git a/docker/mongodb-agent-non-matrix/README.md b/docker/mongodb-agent-non-matrix/README.md new file mode 100644 index 000000000..79dc0d2d5 --- /dev/null +++ b/docker/mongodb-agent-non-matrix/README.md @@ -0,0 +1,17 @@ +### Building locally + +For building the MongoDB Agent (non-static) image locally use the example command: + +TODO: What to do with label quay.expires-after=48h? +```bash +AGENT_VERSION="108.0.7.8810-1" +TOOLS_VERSION="100.12.0" +AGENT_DISTRO="rhel9_x86_64" +TOOLS_DISTRO="rhel93-x86_64" +docker buildx build --load --progress plain . -f docker/mongodb-agent/Dockerfile -t "mongodb-agent:${AGENT_VERSION}" \ + --build-arg version="${VERSION}" \ + --build-arg agent_version="${AGENT_VERSION}" \ + --build-arg tools_version="${TOOLS_VERSION}" \ + --build-arg agent_distro="${AGENT_DISTRO}" \ + --build-arg tools_distro="${TOOLS_DISTRO}" +``` diff --git a/docker/mongodb-agent/Dockerfile b/docker/mongodb-agent/Dockerfile index 08d8746d8..5ec4e127b 100644 --- a/docker/mongodb-agent/Dockerfile +++ b/docker/mongodb-agent/Dockerfile @@ -1,5 +1,40 @@ -ARG imagebase -FROM ${imagebase} as base +# the init database image gets supplied by pipeline.py and corresponds to the operator version we want to release +# the agent with. This enables us to release the agent for older operator. +ARG init_database_image +FROM ${init_database_image} AS init_database + +FROM public.ecr.aws/docker/library/golang:1.24 AS dependency_downloader + +WORKDIR /go/src/github.com/mongodb/mongodb-kubernetes/ + +COPY go.mod go.sum ./ + +RUN go mod download + +FROM public.ecr.aws/docker/library/golang:1.24 AS readiness_builder + +WORKDIR /go/src/github.com/mongodb/mongodb-kubernetes/ + +COPY --from=dependency_downloader /go/pkg /go/pkg +COPY . /go/src/github.com/mongodb/mongodb-kubernetes + +RUN CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -o /readinessprobe ./mongodb-community-operator/cmd/readiness/main.go +RUN CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -o /version-upgrade-hook ./mongodb-community-operator/cmd/versionhook/main.go + +FROM scratch AS base +ARG mongodb_tools_url_ubi +ARG mongodb_agent_url_ubi + +COPY --from=readiness_builder /readinessprobe /data/ +COPY --from=readiness_builder /version-upgrade-hook /data/ + +ADD ${mongodb_tools_url_ubi} /data/mongodb_tools_ubi.tgz +ADD ${mongodb_agent_url_ubi} /data/mongodb_agent_ubi.tgz + +COPY --from=init_database /probes/probe.sh /data/probe.sh +COPY --from=init_database /scripts/agent-launcher-lib.sh /data/ +COPY --from=init_database /scripts/agent-launcher.sh /data/ +COPY --from=init_database /licenses/LICENSE /data/ FROM registry.access.redhat.com/ubi9/ubi-minimal diff --git a/docker/mongodb-agent/README.md b/docker/mongodb-agent/README.md index 377f4b938..a447d60f0 100644 --- a/docker/mongodb-agent/README.md +++ b/docker/mongodb-agent/README.md @@ -1,4 +1,20 @@ # Mongodb-Agent The agent gets released in a matrix style with the init-database image, which gets tagged with the operator version. -This works by using the multi-stage pattern and build-args. First - retrieve the `init-database:` and retrieve the -binaries from there. Then we continue with the other steps to fully build the image. \ No newline at end of file +This works by using the multi-stage pattern and build-args. First - retrieve the `init-database:` and retrieve the +binaries from there. Then we continue with the other steps to fully build the image. + +### Building locally + +For building the MongoDB Agent image locally use the example command: + +```bash +VERSION="108.0.7.8810-1" +INIT_DATABASE_IMAGE="268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/mongodb-kubernetes-init-database:1.1.0" +MONGODB_TOOLS_URL_UBI="https://downloads.mongodb.org/tools/db/mongodb-database-tools-rhel93-x86_64-100.12.0.tgz" +MONGODB_AGENT_URL_UBI="https://mciuploads.s3.amazonaws.com/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-108.0.7.8810-1.rhel9_x86_64.tar.gz" +docker buildx build --load --progress plain . -f docker/mongodb-agent/Dockerfile -t "mongodb-agent:${VERSION}_1.1.0" \ + --build-arg version="${VERSION}" \ + --build-arg init_database_image="${INIT_DATABASE_IMAGE}" \ + --build-arg mongodb_tools_url_ubi="${MONGODB_TOOLS_URL_UBI}" \ + --build-arg mongodb_agent_url_ubi="${MONGODB_AGENT_URL_UBI}" +``` diff --git a/docker/mongodb-enterprise-ops-manager/Dockerfile b/docker/mongodb-enterprise-ops-manager/Dockerfile new file mode 100644 index 000000000..aa95b4bee --- /dev/null +++ b/docker/mongodb-enterprise-ops-manager/Dockerfile @@ -0,0 +1,95 @@ +# Build compilable stuff + +FROM public.ecr.aws/docker/library/golang:1.24 AS readiness_builder +COPY . /go/src/github.com/mongodb/mongodb-kubernetes +WORKDIR /go/src/github.com/mongodb/mongodb-kubernetes + +RUN CGO_ENABLED=0 go build -a -buildvcs=false -o /data/scripts/mmsconfiguration ./docker/mongodb-kubernetes-init-ops-manager/mmsconfiguration/edit_mms_configuration.go +RUN CGO_ENABLED=0 go build -a -buildvcs=false -o /data/scripts/backup-daemon-readiness-probe ./docker/mongodb-kubernetes-init-ops-manager/backupdaemon_readinessprobe/backupdaemon_readiness.go + +# Move binaries and scripts +FROM scratch AS base + +COPY --from=readiness_builder /data/scripts/mmsconfiguration /data/scripts/mmsconfiguration +COPY --from=readiness_builder /data/scripts/backup-daemon-readiness-probe /data/scripts/backup-daemon-readiness-probe + +# After v2.0, when non-Static Agent images will be removed, please ensure to copy those files +# into ./docker/mongodb-enterprise-ops-manager directory. Leaving it this way will make the maintenance easier. +COPY ./docker/mongodb-kubernetes-init-ops-manager/scripts/docker-entry-point.sh /data/scripts +COPY ./docker/mongodb-kubernetes-init-ops-manager/scripts/backup-daemon-liveness-probe.sh /data/scripts +COPY ./docker/mongodb-kubernetes-init-ops-manager/LICENSE /data/licenses/mongodb-enterprise-ops-manager + +FROM registry.access.redhat.com/ubi9/ubi-minimal + +ARG version +ARG om_download_url + +LABEL name="MongoDB Enterprise Ops Manager" \ + maintainer="support@mongodb.com" \ + vendor="MongoDB" \ + version=${version} \ + release="1" \ + summary="MongoDB Enterprise Ops Manager Image" \ + description="MongoDB Enterprise Ops Manager" + +ENV MMS_HOME=/mongodb-ops-manager +ENV MMS_PROP_FILE=${MMS_HOME}/conf/conf-mms.properties +ENV MMS_CONF_FILE=${MMS_HOME}/conf/mms.conf +ENV MMS_LOG_DIR=${MMS_HOME}/logs +ENV MMS_TMP_DIR=${MMS_HOME}/tmp + +EXPOSE 8080 + +# OpsManager docker image needs to have the MongoDB dependencies because the +# backup daemon is running its database locally + +# Replace libcurl-minimal and curl-minimal with the full versions +# https://bugzilla.redhat.com/show_bug.cgi?id=1994521 +RUN microdnf install -y libssh libpsl libbrotli \ + && microdnf download curl libcurl \ + && rpm -Uvh --nodeps --replacefiles "*curl*$( uname -i ).rpm" \ + && microdnf remove -y libcurl-minimal curl-minimal + +RUN microdnf install --disableplugin=subscription-manager -y \ + cyrus-sasl \ + cyrus-sasl-gssapi \ + cyrus-sasl-plain \ + krb5-libs \ + libpcap \ + lm_sensors-libs \ + net-snmp \ + net-snmp-agent-libs \ + openldap \ + openssl \ + tar \ + rpm-libs \ + net-tools \ + procps-ng \ + ncurses + +COPY --from=base /data/licenses /licenses/ +COPY --from=base /data/scripts /opt/scripts + +RUN curl --fail -L -o ops_manager.tar.gz ${om_download_url} \ + && tar -xzf ops_manager.tar.gz \ + && rm ops_manager.tar.gz \ + && mv mongodb-mms* "${MMS_HOME}" + +# permissions +RUN chmod -R 0777 "${MMS_LOG_DIR}" \ + && chmod -R 0777 "${MMS_TMP_DIR}" \ + && chmod -R 0775 "${MMS_HOME}/conf" \ + && chmod -R 0775 "${MMS_HOME}/jdk" \ + && mkdir "${MMS_HOME}/mongodb-releases/" \ + && chmod -R 0775 "${MMS_HOME}/mongodb-releases" \ + && chmod -R 0777 "${MMS_CONF_FILE}" \ + && chmod -R 0777 "${MMS_PROP_FILE}" + +# The "${MMS_HOME}/conf" will be populated by the docker-entry-point.sh. +# For now we need to move into the templates directory. +RUN cp -r "${MMS_HOME}/conf" "${MMS_HOME}/conf-template" + +USER 2000 + +# operator to change the entrypoint to: /mongodb-ops-manager/bin/mongodb-mms start_mms (or a wrapper around this) +ENTRYPOINT [ "sleep infinity" ] diff --git a/docker/mongodb-enterprise-ops-manager/Dockerfile.dcar b/docker/mongodb-enterprise-ops-manager/Dockerfile.dcar deleted file mode 100644 index 639c7930b..000000000 --- a/docker/mongodb-enterprise-ops-manager/Dockerfile.dcar +++ /dev/null @@ -1,25 +0,0 @@ -{% extends "Dockerfile.ubi" %} - - -{% block packages %} -RUN yum install --disableplugin=subscription-manager \ - cyrus-sasl \ - cyrus-sasl-gssapi \ - cyrus-sasl-plain \ - krb5-libs \ - libcurl \ - libpcap \ - lm_sensors-libs \ - net-snmp \ - net-snmp-agent-libs \ - openldap \ - openssl \ - rpm-libs \ - net-tools \ - procps-ng \ - ncurses -{% endblock %} - -{% block healthcheck %} -HEALTHCHECK --timeout=30s CMD ls /mongodb-ops-manager/bin/mongodb-mms || exit 1 -{% endblock %} diff --git a/docker/mongodb-enterprise-ops-manager/Dockerfile.plain b/docker/mongodb-enterprise-ops-manager/Dockerfile.plain new file mode 100644 index 000000000..717014b97 --- /dev/null +++ b/docker/mongodb-enterprise-ops-manager/Dockerfile.plain @@ -0,0 +1,84 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM registry.access.redhat.com/ubi9/ubi-minimal + + +LABEL name="MongoDB Enterprise Ops Manager" \ + maintainer="support@mongodb.com" \ + vendor="MongoDB" \ + version="8.0.7" \ + release="1" \ + summary="MongoDB Enterprise Ops Manager Image" \ + description="MongoDB Enterprise Ops Manager" + + +ENV MMS_HOME /mongodb-ops-manager +ENV MMS_PROP_FILE ${MMS_HOME}/conf/conf-mms.properties +ENV MMS_CONF_FILE ${MMS_HOME}/conf/mms.conf +ENV MMS_LOG_DIR ${MMS_HOME}/logs +ENV MMS_TMP_DIR ${MMS_HOME}/tmp + +EXPOSE 8080 + +# OpsManager docker image needs to have the MongoDB dependencies because the +# backup daemon is running its database locally + + +# Replace libcurl-minimal and curl-minimal with the full versions +# https://bugzilla.redhat.com/show_bug.cgi?id=1994521 +RUN microdnf install -y libssh libpsl libbrotli \ + && microdnf download curl libcurl \ + && rpm -Uvh --nodeps --replacefiles "*curl*$( uname -i ).rpm" \ + && microdnf remove -y libcurl-minimal curl-minimal + +RUN microdnf install --disableplugin=subscription-manager -y \ + cyrus-sasl \ + cyrus-sasl-gssapi \ + cyrus-sasl-plain \ + krb5-libs \ + libpcap \ + lm_sensors-libs \ + net-snmp \ + net-snmp-agent-libs \ + openldap \ + openssl \ + tar \ + rpm-libs \ + net-tools \ + procps-ng \ + ncurses + + +COPY --from=base /data/licenses /licenses/ + +COPY --from=base /data/scripts /opt/scripts + + + +RUN curl --fail -L -o ops_manager.tar.gz https://downloads.mongodb.com/on-prem-mms/tar/mongodb-mms-8.0.7.500.20250505T1426Z.tar.gz \ + && tar -xzf ops_manager.tar.gz \ + && rm ops_manager.tar.gz \ + && mv mongodb-mms* "${MMS_HOME}" + + +# permissions +RUN chmod -R 0777 "${MMS_LOG_DIR}" \ + && chmod -R 0777 "${MMS_TMP_DIR}" \ + && chmod -R 0775 "${MMS_HOME}/conf" \ + && chmod -R 0775 "${MMS_HOME}/jdk" \ + && mkdir "${MMS_HOME}/mongodb-releases/" \ + && chmod -R 0775 "${MMS_HOME}/mongodb-releases" \ + && chmod -R 0777 "${MMS_CONF_FILE}" \ + && chmod -R 0777 "${MMS_PROP_FILE}" + +# The "${MMS_HOME}/conf" will be populated by the docker-entry-point.sh. +# For now we need to move into the templates directory. +RUN cp -r "${MMS_HOME}/conf" "${MMS_HOME}/conf-template" + +USER 2000 + +# operator to change the entrypoint to: /mongodb-ops-manager/bin/mongodb-mms start_mms (or a wrapper around this) +ENTRYPOINT [ "sleep infinity" ] + + diff --git a/docker/mongodb-enterprise-ops-manager/LICENSE b/docker/mongodb-enterprise-ops-manager/LICENSE deleted file mode 100644 index dc71da876..000000000 --- a/docker/mongodb-enterprise-ops-manager/LICENSE +++ /dev/null @@ -1,3 +0,0 @@ -Usage of the MongoDB Enterprise Operator for Kubernetes indicates agreement with the MongoDB Customer Agreement. - -* https://www.mongodb.com/customer-agreement/ diff --git a/docker/mongodb-enterprise-ops-manager/README.md b/docker/mongodb-enterprise-ops-manager/README.md new file mode 100644 index 000000000..440e839bc --- /dev/null +++ b/docker/mongodb-enterprise-ops-manager/README.md @@ -0,0 +1,11 @@ +### Building locally + +For building the MongoDB Enterprise Ops Manager Docker image locally use the example command: + +```bash +VERSION="8.0.7" +OM_DOWNLOAD_URL="https://downloads.mongodb.com/on-prem-mms/tar/mongodb-mms-8.0.7.500.20250505T1426Z.tar.gz" +docker buildx build --load --progress plain . -f docker/mongodb-enterprise-ops-manager/Dockerfile -t "mongodb-enterprise-ops-manager:${VERSION}" \ + --build-arg version="${VERSION}" \ + --build-arg om_download_url="${OM_DOWNLOAD_URL}" +``` diff --git a/docker/mongodb-kubernetes-database/Dockerfile b/docker/mongodb-kubernetes-database/Dockerfile new file mode 100644 index 000000000..97fbda8d0 --- /dev/null +++ b/docker/mongodb-kubernetes-database/Dockerfile @@ -0,0 +1,71 @@ +FROM scratch AS base + +COPY ./docker/mongodb-kubernetes-database/LICENSE /data/licenses/mongodb-kubernetes-database + +FROM registry.access.redhat.com/ubi8/ubi-minimal + +ARG VERSION + +LABEL name="MongoDB Kubernetes Database" \ + version="${VERSION}" \ + summary="MongoDB Kubernetes Database Image" \ + description="MongoDB Kubernetes Database Image" \ + vendor="MongoDB" \ + release="1" \ + maintainer="support@mongodb.com" + +ENV MMS_HOME=/mongodb-automation +ENV MMS_LOG_DIR=/var/log/mongodb-mms-automation + +RUN microdnf update -y && rm -rf /var/cache/yum + +# these are the packages needed for the agent +RUN microdnf install -y --disableplugin=subscription-manager --setopt=install_weak_deps=0 nss_wrapper +RUN microdnf install -y --disableplugin=subscription-manager \ + hostname \ + procps + +# these are the packages needed for MongoDB +# (https://docs.mongodb.com/manual/tutorial/install-mongodb-enterprise-on-red-hat-tarball/ "RHEL/CentOS 8" tab) +RUN microdnf install -y --disableplugin=subscription-manager \ + cyrus-sasl \ + cyrus-sasl-gssapi \ + cyrus-sasl-plain \ + krb5-libs \ + libcurl \ + lm_sensors-libs \ + net-snmp \ + net-snmp-agent-libs \ + openldap \ + openssl \ + jq \ + tar \ + xz-libs \ + findutils + +RUN ln -s /usr/lib64/libsasl2.so.3 /usr/lib64/libsasl2.so.2 + +# Set the required perms +RUN mkdir -p "${MMS_LOG_DIR}" \ + && chmod 0775 "${MMS_LOG_DIR}" \ + && mkdir -p /var/lib/mongodb-mms-automation \ + && chmod 0775 /var/lib/mongodb-mms-automation \ + && mkdir -p /data \ + && chmod 0775 /data \ + && mkdir -p /journal \ + && chmod 0775 /journal \ + && mkdir -p "${MMS_HOME}" \ + && chmod -R 0775 "${MMS_HOME}" + +# USER needs to be set for this image to pass RedHat verification. Some customers have these requirements as well +# It does not matter what number it is, as long as it is set to something. +# However, OpenShift will run the container as a random user, +# and the number in this configuration is not relevant. +USER 2000 + +# The docker image doesn't have any scripts so by default does nothing +# The script will be copied in runtime from init containers and the operator is expected +# to override the COMMAND +ENTRYPOINT ["sleep infinity"] + +COPY --from=base /data/licenses/mongodb-kubernetes-database /licenses/mongodb-kubernetes-database diff --git a/docker/mongodb-kubernetes-database/Dockerfile.plain b/docker/mongodb-kubernetes-database/Dockerfile.plain new file mode 100644 index 000000000..ea7b4a8e7 --- /dev/null +++ b/docker/mongodb-kubernetes-database/Dockerfile.plain @@ -0,0 +1,87 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM registry.access.redhat.com/ubi8/ubi-minimal + + + +LABEL name="MongoDB Kubernetes Database" \ + version="1.1.0" \ + summary="MongoDB Kubernetes Database Image" \ + description="MongoDB Kubernetes Database Image" \ + vendor="MongoDB" \ + release="1" \ + maintainer="support@mongodb.com" + + + + + +ENV MMS_HOME /mongodb-automation +ENV MMS_LOG_DIR /var/log/mongodb-mms-automation + + + +RUN microdnf update -y && rm -rf /var/cache/yum + +# these are the packages needed for the agent +RUN microdnf install -y --disableplugin=subscription-manager --setopt=install_weak_deps=0 nss_wrapper +RUN microdnf install -y --disableplugin=subscription-manager \ + hostname \ + procps + + +# these are the packages needed for MongoDB +# (https://docs.mongodb.com/manual/tutorial/install-mongodb-enterprise-on-red-hat-tarball/ "RHEL/CentOS 8" tab) +RUN microdnf install -y --disableplugin=subscription-manager \ + cyrus-sasl \ + cyrus-sasl-gssapi \ + cyrus-sasl-plain \ + krb5-libs \ + libcurl \ + lm_sensors-libs \ + net-snmp \ + net-snmp-agent-libs \ + openldap \ + openssl \ + jq \ + tar \ + xz-libs \ + findutils + + + +RUN ln -s /usr/lib64/libsasl2.so.3 /usr/lib64/libsasl2.so.2 + + +# Set the required perms +RUN mkdir -p "${MMS_LOG_DIR}" \ + && chmod 0775 "${MMS_LOG_DIR}" \ + && mkdir -p /var/lib/mongodb-mms-automation \ + && chmod 0775 /var/lib/mongodb-mms-automation \ + && mkdir -p /data \ + && chmod 0775 /data \ + && mkdir -p /journal \ + && chmod 0775 /journal \ + && mkdir -p "${MMS_HOME}" \ + && chmod -R 0775 "${MMS_HOME}" + + + + +# USER needs to be set for this image to pass RedHat verification. Some customers have these requirements as well +# It does not matter what number it is, as long as it is set to something. +# However, OpenShift will run the container as a random user, +# and the number in this configuration is not relevant. +USER 2000 + + +# The docker image doesn't have any scripts so by default does nothing +# The script will be copied in runtime from init containers and the operator is expected +# to override the COMMAND +ENTRYPOINT ["sleep infinity"] + + +COPY --from=base /data/licenses/mongodb-kubernetes-database /licenses/mongodb-kubernetes-database + + diff --git a/docker/mongodb-kubernetes-database/README.md b/docker/mongodb-kubernetes-database/README.md index a6abf56a9..e7b937e0e 100644 --- a/docker/mongodb-kubernetes-database/README.md +++ b/docker/mongodb-kubernetes-database/README.md @@ -34,11 +34,12 @@ This image can't be built in any host, because it will require the use of a subs host, with subscription service enabled, is required. That's the reason behind using the Redhat build service to build this images with. -## Building the DCAR database image +### Building locally -The dcar image needs to be built manually. +For building the MongoDB Database image locally use the example command: ```bash -docker build . -t 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/usaf/mongodb-kubernetes-database:1.5.3 -docker push 268558157000.dkr.ecr.us-east-1.amazonaws.com/dev/usaf/mongodb-kubernetes-database:1.5.3 +VERSION="1.0.1" +docker buildx build --load --progress plain . -f docker/mongodb-kubernetes-database/Dockerfile -t "mongodb-kubernetes-database:${VERSION}" \ + --build-arg VERSION="${VERSION}" ``` diff --git a/docker/mongodb-kubernetes-init-appdb/Dockerfile b/docker/mongodb-kubernetes-init-appdb/Dockerfile new file mode 100644 index 000000000..ed0cea9dd --- /dev/null +++ b/docker/mongodb-kubernetes-init-appdb/Dockerfile @@ -0,0 +1,52 @@ +FROM public.ecr.aws/docker/library/golang:1.24 AS readiness_builder + +COPY . /go/src/github.com/mongodb/mongodb-kubernetes +WORKDIR /go/src/github.com/mongodb/mongodb-kubernetes +RUN CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -o /readinessprobe ./mongodb-community-operator/cmd/readiness/main.go +RUN CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -o /version-upgrade-hook ./mongodb-community-operator/cmd/versionhook/main.go + +FROM scratch AS base + +ARG mongodb_tools_url_ubi + +COPY --from=readiness_builder /readinessprobe /data/ +COPY --from=readiness_builder /version-upgrade-hook /data/version-upgrade-hook + +ADD ${mongodb_tools_url_ubi} /data/mongodb_tools_ubi.tgz + +COPY ./docker/mongodb-kubernetes-init-database/content/probe.sh /data/probe.sh + +COPY ./docker/mongodb-kubernetes-init-database/content/agent-launcher-lib.sh /data/scripts/ +COPY ./docker/mongodb-kubernetes-init-database/content/agent-launcher.sh /data/scripts/ + +COPY ./docker/mongodb-kubernetes-init-database/content/LICENSE /data/licenses/ + +FROM registry.access.redhat.com/ubi8/ubi-minimal + +ARG version +LABEL name="MongoDB Kubernetes Init AppDB" \ + version="mongodb-kubernetes-init-appdb-${version}" \ + summary="MongoDB Kubernetes AppDB Init Image" \ + description="Startup Scripts for MongoDB Enterprise Application Database for Ops Manager" \ + release="1" \ + vendor="MongoDB" \ + maintainer="support@mongodb.com" + +COPY --from=base /data/readinessprobe /probes/readinessprobe +COPY --from=base /data/probe.sh /probes/probe.sh +COPY --from=base /data/scripts/ /scripts/ +COPY --from=base /data/licenses /licenses/ +COPY --from=base /data/version-upgrade-hook /probes/version-upgrade-hook + +RUN microdnf -y update --nodocs \ + && microdnf -y install --nodocs tar gzip \ + && microdnf clean all + +COPY --from=base /data/mongodb_tools_ubi.tgz /tools/mongodb_tools.tgz + +RUN tar xfz /tools/mongodb_tools.tgz --directory /tools \ + && rm /tools/mongodb_tools.tgz + +USER 2000 + +ENTRYPOINT [ "/bin/cp", "-f", "-r", "/scripts/agent-launcher.sh", "/scripts/agent-launcher-lib.sh", "/probes/readinessprobe", "/probes/probe.sh", "/tools", "/opt/scripts/" ] diff --git a/docker/mongodb-kubernetes-init-appdb/Dockerfile.builder b/docker/mongodb-kubernetes-init-appdb/Dockerfile.builder new file mode 100644 index 000000000..69dc6d6af --- /dev/null +++ b/docker/mongodb-kubernetes-init-appdb/Dockerfile.builder @@ -0,0 +1,22 @@ +# Build compilable stuff + +FROM public.ecr.aws/docker/library/golang:1.24 as readiness_builder +COPY . /go/src/github.com/mongodb/mongodb-kubernetes +WORKDIR /go/src/github.com/mongodb/mongodb-kubernetes +RUN CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -o /readinessprobe ./mongodb-community-operator/cmd/readiness/main.go +RUN CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -o /version-upgrade-hook ./mongodb-community-operator/cmd/versionhook/main.go + +FROM scratch +ARG mongodb_tools_url_ubi + +COPY --from=readiness_builder /readinessprobe /data/ +COPY --from=readiness_builder /version-upgrade-hook /data/version-upgrade-hook + +ADD ${mongodb_tools_url_ubi} /data/mongodb_tools_ubi.tgz + +COPY ./docker/mongodb-kubernetes-init-database/content/probe.sh /data/probe.sh + +COPY ./docker/mongodb-kubernetes-init-database/content/agent-launcher-lib.sh /data/scripts/ +COPY ./docker/mongodb-kubernetes-init-database/content/agent-launcher.sh /data/scripts/ + +COPY ./docker/mongodb-kubernetes-init-database/content/LICENSE /data/licenses/ diff --git a/docker/mongodb-kubernetes-init-appdb/Dockerfile.plain b/docker/mongodb-kubernetes-init-appdb/Dockerfile.plain new file mode 100644 index 000000000..d0c5d967a --- /dev/null +++ b/docker/mongodb-kubernetes-init-appdb/Dockerfile.plain @@ -0,0 +1,35 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM registry.access.redhat.com/ubi8/ubi-minimal + +ARG version +LABEL name="MongoDB Kubernetes Init AppDB" \ + version="mongodb-kubernetes-init-appdb-${version}" \ + summary="MongoDB Kubernetes AppDB Init Image" \ + description="Startup Scripts for MongoDB Enterprise Application Database for Ops Manager" \ + release="1" \ + vendor="MongoDB" \ + maintainer="support@mongodb.com" + +COPY --from=base /data/readinessprobe /probes/readinessprobe +COPY --from=base /data/probe.sh /probes/probe.sh +COPY --from=base /data/scripts/ /scripts/ +COPY --from=base /data/licenses /licenses/ +COPY --from=base /data/version-upgrade-hook /probes/version-upgrade-hook + + +RUN microdnf -y update --nodocs \ + && microdnf -y install --nodocs tar gzip \ + && microdnf clean all + +COPY --from=base /data/mongodb_tools_ubi.tgz /tools/mongodb_tools.tgz + + +RUN tar xfz /tools/mongodb_tools.tgz --directory /tools \ + && rm /tools/mongodb_tools.tgz + +USER 2000 +ENTRYPOINT [ "/bin/cp", "-f", "-r", "/scripts/agent-launcher.sh", "/scripts/agent-launcher-lib.sh", "/probes/readinessprobe", "/probes/probe.sh", "/tools", "/opt/scripts/" ] + + diff --git a/docker/mongodb-kubernetes-init-appdb/Dockerfile.template b/docker/mongodb-kubernetes-init-appdb/Dockerfile.template new file mode 100644 index 000000000..3c0d45ee4 --- /dev/null +++ b/docker/mongodb-kubernetes-init-appdb/Dockerfile.template @@ -0,0 +1,42 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM {{ base_image }} + +ARG version + +{%- if is_appdb %} +LABEL name="MongoDB Kubernetes Init AppDB" \ + version="mongodb-kubernetes-init-appdb-${version}" \ + summary="MongoDB Kubernetes AppDB Init Image" \ + description="Startup Scripts for MongoDB Enterprise Application Database for Ops Manager" \ +{%- else %} +LABEL name="MongoDB Kubernetes Init Database" \ + version="mongodb-kubernetes-init-database-${version}" \ + summary="MongoDB Kubernetes Database Init Image" \ + description="Startup Scripts for MongoDB Enterprise Database" \ +{%- endif %} + release="1" \ + vendor="MongoDB" \ + maintainer="support@mongodb.com" + +COPY --from=base /data/readinessprobe /probes/readinessprobe +COPY --from=base /data/probe.sh /probes/probe.sh +COPY --from=base /data/scripts/ /scripts/ +COPY --from=base /data/licenses /licenses/ + +{%- if is_appdb %} +COPY --from=base /data/version-upgrade-hook /probes/version-upgrade-hook +{%- endif %} + +{% block mongodb_tools %} +{% endblock %} + +RUN tar xfz /tools/mongodb_tools.tgz --directory /tools \ + && rm /tools/mongodb_tools.tgz + +USER 2000 +ENTRYPOINT [ "/bin/cp", "-f", "-r", "/scripts/agent-launcher.sh", "/scripts/agent-launcher-lib.sh", "/probes/readinessprobe", "/probes/probe.sh", "/tools", "/opt/scripts/" ] + +{% block healthcheck %} +{% endblock %} diff --git a/docker/mongodb-kubernetes-init-appdb/Dockerfile.ubi_minimal b/docker/mongodb-kubernetes-init-appdb/Dockerfile.ubi_minimal new file mode 100644 index 000000000..b5400b147 --- /dev/null +++ b/docker/mongodb-kubernetes-init-appdb/Dockerfile.ubi_minimal @@ -0,0 +1,11 @@ +{% extends "Dockerfile.template" %} + +{% set base_image = "registry.access.redhat.com/ubi8/ubi-minimal" %} + +{% block mongodb_tools %} +RUN microdnf -y update --nodocs \ + && microdnf -y install --nodocs tar gzip \ + && microdnf clean all + +COPY --from=base /data/mongodb_tools_ubi.tgz /tools/mongodb_tools.tgz +{% endblock %} diff --git a/docker/mongodb-kubernetes-init-appdb/README.md b/docker/mongodb-kubernetes-init-appdb/README.md new file mode 100644 index 000000000..d49ca4b3a --- /dev/null +++ b/docker/mongodb-kubernetes-init-appdb/README.md @@ -0,0 +1,11 @@ +### Building locally + +For building the MongoDB Init AppDB image locally use the example command: + +```bash +VERSION="1.0.1" +MONGODB_TOOLS_URL_UBI="https://downloads.mongodb.org/tools/db/mongodb-database-tools-rhel93-x86_64-100.12.0.tgz" +docker buildx build --load --progress plain . -f docker/mongodb-kubernetes-init-appdb/Dockerfile -t "mongodb-kubernetes-init-appdb:${VERSION}" \ + --build-arg version="${VERSION}" \ + --build-arg mongodb_tools_url_ubi="${MONGODB_TOOLS_URL_UBI}" +``` diff --git a/docker/mongodb-kubernetes-init-database/Dockerfile b/docker/mongodb-kubernetes-init-database/Dockerfile new file mode 100644 index 000000000..6c861fb6a --- /dev/null +++ b/docker/mongodb-kubernetes-init-database/Dockerfile @@ -0,0 +1,50 @@ +FROM public.ecr.aws/docker/library/golang:1.24 AS readiness_builder + +COPY . /go/src/github.com/mongodb/mongodb-kubernetes +WORKDIR /go/src/github.com/mongodb/mongodb-kubernetes +RUN CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -o /readinessprobe ./mongodb-community-operator/cmd/readiness/main.go +RUN CGO_ENABLED=0 GOFLAGS=-buildvcs=false go build -o /version-upgrade-hook ./mongodb-community-operator/cmd/versionhook/main.go + +FROM scratch AS base + +ARG mongodb_tools_url_ubi + +COPY --from=readiness_builder /readinessprobe /data/ +COPY --from=readiness_builder /version-upgrade-hook /data/version-upgrade-hook + +ADD ${mongodb_tools_url_ubi} /data/mongodb_tools_ubi.tgz + +COPY ./docker/mongodb-kubernetes-init-database/content/probe.sh /data/probe.sh + +COPY ./docker/mongodb-kubernetes-init-database/content/agent-launcher-lib.sh /data/scripts/ +COPY ./docker/mongodb-kubernetes-init-database/content/agent-launcher.sh /data/scripts/ + +COPY ./docker/mongodb-kubernetes-init-database/content/LICENSE /data/licenses/ + +FROM registry.access.redhat.com/ubi8/ubi-minimal + +ARG version +LABEL name="MongoDB Kubernetes Init Database" \ + version="mongodb-kubernetes-init-database-${version}" \ + summary="MongoDB Kubernetes Database Init Image" \ + description="Startup Scripts for MongoDB Enterprise Database" \ + release="1" \ + vendor="MongoDB" \ + maintainer="support@mongodb.com" + +COPY --from=base /data/readinessprobe /probes/readinessprobe +COPY --from=base /data/probe.sh /probes/probe.sh +COPY --from=base /data/scripts/ /scripts/ +COPY --from=base /data/licenses /licenses/ + +RUN microdnf -y update --nodocs \ + && microdnf -y install --nodocs tar gzip \ + && microdnf clean all + +COPY --from=base /data/mongodb_tools_ubi.tgz /tools/mongodb_tools.tgz + +RUN tar xfz /tools/mongodb_tools.tgz --directory /tools \ + && rm /tools/mongodb_tools.tgz + +USER 2000 +ENTRYPOINT [ "/bin/cp", "-f", "-r", "/scripts/agent-launcher.sh", "/scripts/agent-launcher-lib.sh", "/probes/readinessprobe", "/probes/probe.sh", "/tools", "/opt/scripts/" ] diff --git a/docker/mongodb-kubernetes-init-database/Dockerfile.plain b/docker/mongodb-kubernetes-init-database/Dockerfile.plain new file mode 100644 index 000000000..ecf2e32ae --- /dev/null +++ b/docker/mongodb-kubernetes-init-database/Dockerfile.plain @@ -0,0 +1,34 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM registry.access.redhat.com/ubi8/ubi-minimal + +ARG version +LABEL name="MongoDB Kubernetes Init Database" \ + version="mongodb-kubernetes-init-database-${version}" \ + summary="MongoDB Kubernetes Database Init Image" \ + description="Startup Scripts for MongoDB Enterprise Database" \ + release="1" \ + vendor="MongoDB" \ + maintainer="support@mongodb.com" + +COPY --from=base /data/readinessprobe /probes/readinessprobe +COPY --from=base /data/probe.sh /probes/probe.sh +COPY --from=base /data/scripts/ /scripts/ +COPY --from=base /data/licenses /licenses/ + + +RUN microdnf -y update --nodocs \ + && microdnf -y install --nodocs tar gzip \ + && microdnf clean all + +COPY --from=base /data/mongodb_tools_ubi.tgz /tools/mongodb_tools.tgz + + +RUN tar xfz /tools/mongodb_tools.tgz --directory /tools \ + && rm /tools/mongodb_tools.tgz + +USER 2000 +ENTRYPOINT [ "/bin/cp", "-f", "-r", "/scripts/agent-launcher.sh", "/scripts/agent-launcher-lib.sh", "/probes/readinessprobe", "/probes/probe.sh", "/tools", "/opt/scripts/" ] + + diff --git a/docker/mongodb-kubernetes-init-database/README.md b/docker/mongodb-kubernetes-init-database/README.md new file mode 100644 index 000000000..0e6657531 --- /dev/null +++ b/docker/mongodb-kubernetes-init-database/README.md @@ -0,0 +1,11 @@ +### Building locally + +For building the MongoDB Init AppDB image locally use the example command: + +```bash +VERSION="1.0.1" +MONGODB_TOOLS_URL_UBI="https://downloads.mongodb.org/tools/db/mongodb-database-tools-rhel93-x86_64-100.12.0.tgz" +docker buildx build --load --progress plain . -f docker/mongodb-kubernetes-init-database/Dockerfile -t "mongodb-kubernetes-init-database:${VERSION}" \ + --build-arg version="${VERSION}" \ + --build-arg mongodb_tools_url_ubi="${MONGODB_TOOLS_URL_UBI}" +``` diff --git a/docker/mongodb-kubernetes-init-ops-manager/Dockerfile b/docker/mongodb-kubernetes-init-ops-manager/Dockerfile new file mode 100644 index 000000000..1229ec929 --- /dev/null +++ b/docker/mongodb-kubernetes-init-ops-manager/Dockerfile @@ -0,0 +1,31 @@ +FROM public.ecr.aws/docker/library/golang:1.24 AS base + +WORKDIR /go/src +ADD ./docker/mongodb-kubernetes-init-ops-manager . +RUN CGO_ENABLED=0 go build -a -buildvcs=false -o /data/scripts/mmsconfiguration ./mmsconfiguration +RUN CGO_ENABLED=0 go build -a -buildvcs=false -o /data/scripts/backup-daemon-readiness-probe ./backupdaemon_readinessprobe/ + +COPY ./docker/mongodb-kubernetes-init-ops-manager/scripts/docker-entry-point.sh /data/scripts/ +COPY ./docker/mongodb-kubernetes-init-ops-manager/scripts/backup-daemon-liveness-probe.sh /data/scripts/ +COPY ./docker/mongodb-kubernetes-init-ops-manager/LICENSE /data/licenses/mongodb-enterprise-ops-manager + +FROM registry.access.redhat.com/ubi9/ubi-minimal + +ARG version + +LABEL name="MongoDB Kubernetes Ops Manager Init" \ + maintainer="support@mongodb.com" \ + vendor="MongoDB" \ + version="mongodb-kubernetes-init-ops-manager-${version}" \ + release="1" \ + summary="MongoDB Kubernetes Ops Manager Init Image" \ + description="Startup Scripts for MongoDB Enterprise Ops Manager" + +COPY --from=base /data/scripts /scripts +COPY --from=base /data/licenses /licenses + +RUN microdnf -y update --nodocs \ + && microdnf clean all + +USER 2000 +ENTRYPOINT [ "/bin/cp", "-f", "/scripts/docker-entry-point.sh", "/scripts/backup-daemon-liveness-probe.sh", "/scripts/mmsconfiguration", "/scripts/backup-daemon-readiness-probe", "/opt/scripts/" ] diff --git a/docker/mongodb-kubernetes-init-ops-manager/Dockerfile.plain b/docker/mongodb-kubernetes-init-ops-manager/Dockerfile.plain new file mode 100644 index 000000000..f841b9e35 --- /dev/null +++ b/docker/mongodb-kubernetes-init-ops-manager/Dockerfile.plain @@ -0,0 +1,26 @@ +ARG imagebase +FROM ${imagebase} as base + +FROM registry.access.redhat.com/ubi9/ubi-minimal + +LABEL name="MongoDB Kubernetes Ops Manager Init" \ + maintainer="support@mongodb.com" \ + vendor="MongoDB" \ + version="mongodb-kubernetes-init-ops-manager-1.1.0" \ + release="1" \ + summary="MongoDB Kubernetes Ops Manager Init Image" \ + description="Startup Scripts for MongoDB Enterprise Ops Manager" + + +COPY --from=base /data/scripts /scripts +COPY --from=base /data/licenses /licenses + + +RUN microdnf -y update --nodocs \ + && microdnf clean all + + +USER 2000 +ENTRYPOINT [ "/bin/cp", "-f", "/scripts/docker-entry-point.sh", "/scripts/backup-daemon-liveness-probe.sh", "/scripts/mmsconfiguration", "/scripts/backup-daemon-readiness-probe", "/opt/scripts/" ] + + diff --git a/docker/mongodb-kubernetes-init-ops-manager/README.md b/docker/mongodb-kubernetes-init-ops-manager/README.md new file mode 100644 index 000000000..71d02da75 --- /dev/null +++ b/docker/mongodb-kubernetes-init-ops-manager/README.md @@ -0,0 +1,9 @@ +### Building locally + +For building the MongoDB Init Ops Manager image locally use the example command: + +```bash +VERSION="1.1.0" +docker buildx build --load --progress plain . -f docker/mongodb-kubernetes-init-ops-manager/Dockerfile -t "mongodb-kubernetes-init-ops-manager:${VERSION}" \ + --build-arg version="${VERSION}" +``` diff --git a/docker/mongodb-kubernetes-operator/Dockerfile b/docker/mongodb-kubernetes-operator/Dockerfile new file mode 100644 index 000000000..dcd3af35c --- /dev/null +++ b/docker/mongodb-kubernetes-operator/Dockerfile @@ -0,0 +1,72 @@ +FROM public.ecr.aws/docker/library/golang:1.24 AS builder + +ARG version +ARG log_automation_config_diff +ARG use_race + +COPY go.sum go.mod /go/src/github.com/mongodb/mongodb-kubernetes/ + +WORKDIR /go/src/github.com/mongodb/mongodb-kubernetes +RUN go mod download + +COPY . /go/src/github.com/mongodb/mongodb-kubernetes + +RUN go version +RUN git version +RUN mkdir /build && \ + if [ $use_race = "true" ]; then \ + echo "Building with race detector" && \ + CGO_ENABLED=1 go build -o /build/mongodb-kubernetes-operator \ + -buildvcs=false \ + -race \ + -ldflags=" -X github.com/mongodb/mongodb-kubernetes/pkg/util.OperatorVersion=${version} \ + -X github.com/mongodb/mongodb-kubernetes/pkg/util.LogAutomationConfigDiff=${log_automation_config_diff}"; \ + else \ + echo "Building without race detector" && \ + CGO_ENABLED=0 go build -o /build/mongodb-kubernetes-operator \ + -buildvcs=false \ + -ldflags="-s -w -X github.com/mongodb/mongodb-kubernetes/pkg/util.OperatorVersion=${version} \ + -X github.com/mongodb/mongodb-kubernetes/pkg/util.LogAutomationConfigDiff=${log_automation_config_diff}"; \ + fi + + +ADD https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 /usr/local/bin/jq +RUN chmod +x /usr/local/bin/jq + +RUN mkdir -p /data +RUN cat release.json | jq -r '.supportedImages."mongodb-agent" | { "supportedImages": { "mongodb-agent": . } }' > /data/om_version_mapping.json +RUN chmod +r /data/om_version_mapping.json + +FROM scratch AS base + +COPY --from=builder /build/mongodb-kubernetes-operator /data/ +COPY --from=builder /data/om_version_mapping.json /data/om_version_mapping.json + +ADD docker/mongodb-kubernetes-operator/licenses /data/licenses/ + +FROM registry.access.redhat.com/ubi9/ubi-minimal + +ARG version + +LABEL name="MongoDB Kubernetes Operator" \ + maintainer="support@mongodb.com" \ + vendor="MongoDB" \ + version="${version}" \ + release="1" \ + summary="MongoDB Kubernetes Operator Image" \ + description="MongoDB Kubernetes Operator Image" + +# Building an UBI-based image: https://red.ht/3n6b9y0 +RUN microdnf update \ + --disableplugin=subscription-manager \ + --disablerepo=* --enablerepo=ubi-9-appstream-rpms --enablerepo=ubi-9-baseos-rpms -y \ + && rm -rf /var/cache/yum +RUN microdnf install -y glibc-langpack-en + +COPY --from=base /data/mongodb-kubernetes-operator /usr/local/bin/mongodb-kubernetes-operator +COPY --from=base /data/om_version_mapping.json /usr/local/om_version_mapping.json +COPY --from=base /data/licenses /licenses/ + +USER 2000 + +ENTRYPOINT exec /usr/local/bin/mongodb-kubernetes-operator diff --git a/docker/mongodb-kubernetes-operator/Dockerfile.plain b/docker/mongodb-kubernetes-operator/Dockerfile.plain new file mode 100644 index 000000000..7466187f7 --- /dev/null +++ b/docker/mongodb-kubernetes-operator/Dockerfile.plain @@ -0,0 +1,38 @@ +# +# Base Template Dockerfile for Operator Image. +# + +ARG imagebase +FROM ${imagebase} as base + +FROM registry.access.redhat.com/ubi9/ubi-minimal + + +LABEL name="MongoDB Kubernetes Operator" \ + maintainer="support@mongodb.com" \ + vendor="MongoDB" \ + version="1.1.0" \ + release="1" \ + summary="MongoDB Kubernetes Operator Image" \ + description="MongoDB Kubernetes Operator Image" + + +# Building an UBI-based image: https://red.ht/3n6b9y0 +RUN microdnf update \ + --disableplugin=subscription-manager \ + --disablerepo=* --enablerepo=ubi-9-appstream-rpms --enablerepo=ubi-9-baseos-rpms -y \ + && rm -rf /var/cache/yum +RUN microdnf install -y glibc-langpack-en + + + + +COPY --from=base /data/mongodb-kubernetes-operator /usr/local/bin/mongodb-kubernetes-operator +COPY --from=base /data/om_version_mapping.json /usr/local/om_version_mapping.json +COPY --from=base /data/licenses /licenses/ + +USER 2000 + +ENTRYPOINT exec /usr/local/bin/mongodb-kubernetes-operator + + diff --git a/docker/mongodb-kubernetes-operator/README.md b/docker/mongodb-kubernetes-operator/README.md index 4dc971f03..8335c1d79 100644 --- a/docker/mongodb-kubernetes-operator/README.md +++ b/docker/mongodb-kubernetes-operator/README.md @@ -10,8 +10,16 @@ CGO_ENABLED=0 GOOS=linux GOFLAGS="-mod=vendor" go build -i -o mongodb-kubernetes ### Building the image +For building the MongoDB Init Ops Manager image locally use the example command: + ```bash -docker build -t mongodb-kubernetes-operator:0.1 . +VERSION="1.1.0" +LOG_AUTOMATION_CONFIG_DIFF="false" +USE_RACE="false" +docker buildx build --load --progress plain . -f docker/mongodb-kubernetes-operator/Dockerfile -t "mongodb-kubernetes-operator:${VERSION}" \ + --build-arg version="${VERSION}" \ + --build-arg log_automation_config_diff="${LOG_AUTOMATION_CONFIG_DIFF}" \ + --build-arg use_race="${USE_RACE}" ``` ### Running locally diff --git a/docker/mongodb-kubernetes-readinessprobe/Dockerfile b/docker/mongodb-kubernetes-readinessprobe/Dockerfile index 17c590526..a2f3159b4 100644 --- a/docker/mongodb-kubernetes-readinessprobe/Dockerfile +++ b/docker/mongodb-kubernetes-readinessprobe/Dockerfile @@ -1,6 +1,11 @@ -ARG imagebase -FROM ${imagebase} as base +FROM public.ecr.aws/docker/library/golang:1.24 AS builder + +WORKDIR /go/src +ADD . . + +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -a -o /data/scripts/readinessprobe ./mongodb-community-operator/cmd/readiness/main.go FROM registry.access.redhat.com/ubi9/ubi-minimal -COPY --from=base /probes/readinessprobe /probes/readinessprobe +COPY --from=builder /data/scripts/readinessprobe /probes/readinessprobe diff --git a/docker/mongodb-kubernetes-readinessprobe/README.md b/docker/mongodb-kubernetes-readinessprobe/README.md new file mode 100644 index 000000000..1dd56bae8 --- /dev/null +++ b/docker/mongodb-kubernetes-readinessprobe/README.md @@ -0,0 +1,10 @@ +### Building locally + +For building the readiness probe image locally use the example command: + +```bash +VERSION="1.0.22" +TARGETARCH="amd64" +docker buildx build --load --progress plain . -f docker/mongodb-kubernetes-readinessprobe/Dockerfile -t "mongodb-kubernetes-readinessprobe:${VERSION}" \ + --build-arg TARGETARCH="${TARGETARCH}" +``` diff --git a/docker/mongodb-kubernetes-tests/release.json b/docker/mongodb-kubernetes-tests/release.json new file mode 100644 index 000000000..4fdb45ec1 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/release.json @@ -0,0 +1,253 @@ +{ + "mongodbToolsBundle": { + "ubi": "mongodb-database-tools-rhel88-x86_64-100.12.0.tgz" + }, + "mongodbOperator": "1.1.0", + "initDatabaseVersion": "1.1.0", + "initOpsManagerVersion": "1.1.0", + "initAppDbVersion": "1.1.0", + "databaseImageVersion": "1.1.0", + "agentVersion": "108.0.2.8729-1", + "openshift": { + "minimumSupportedVersion": "4.6" + }, + "search": { + "community": { + "version": "1.47.0" + } + }, + "supportedImages": { + "readinessprobe": { + "ssdlc_name": "MongoDB Controllers for Kubernetes Readiness Probe", + "versions": [ + "1.0.22" + ], + "variants": [ + "ubi" + ] + }, + "operator-version-upgrade-post-start-hook": { + "ssdlc_name": "MongoDB Controllers for Kubernetes Operator Version Upgrade Hook", + "versions": [ + "1.0.9" + ], + "variants": [ + "ubi" + ] + }, + "ops-manager": { + "ssdlc_name": "MongoDB Controllers for Kubernetes Enterprise Ops Manager", + "versions": [ + "6.0.25", + "6.0.26", + "6.0.27", + "7.0.12", + "7.0.13", + "7.0.14", + "7.0.15", + "8.0.5", + "8.0.6", + "8.0.7" + ], + "variants": [ + "ubi" + ] + }, + "mongodb-kubernetes": { + "Description": "We support 3 last versions, see https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Operator+Support+Policy", + "ssdlc_name": "MongoDB Controllers for Kubernetes Operator", + "versions": [ + "1.0.0", + "1.0.1", + "1.1.0" + ], + "variants": [ + "ubi" + ] + }, + "mongodb-kubernetes-operator": { + "Description": "Community Operator daily rebuilds", + "ssdlc_name": "MongoDB Community Operator", + "versions": [ + "0.12.0", + "0.11.0", + "0.10.0", + "0.9.0", + "0.8.3", + "0.8.2", + "0.8.1", + "0.8.0", + "0.7.9", + "0.7.8", + "0.7.7", + "0.7.6" + ], + "variants": [ + "ubi" + ] + }, + "mongodb-agent": { + "Description": "Agents corresponding to OpsManager 5.x and 6.x series", + "ssdlc_name": "MongoDB Controllers for Kubernetes MongoDB Agent", + "Description for specific versions": { + "11.0.5.6963-1": "An upgraded version for OM 5.0 we use for Operator-only deployments", + "12.0.28.7763-1": "OM 6 basic version" + }, + "versions": [ + "108.0.2.8729-1" + ], + "opsManagerMapping": { + "Description": "These are the agents from which we start supporting static containers.", + "cloud_manager": "13.35.0.9498-1", + "cloud_manager_tools": "100.12.1", + "ops_manager": { + "6.0.25": { + "agent_version": "12.0.33.7866-1", + "tools_version": "100.10.0" + }, + "6.0.26": { + "agent_version": "12.0.34.7888-1", + "tools_version": "100.10.0" + }, + "6.0.27": { + "agent_version": "12.0.35.7911-1", + "tools_version": "100.10.0" + }, + "7.0.13": { + "agent_version": "107.0.13.8702-1", + "tools_version": "100.10.0" + }, + "7.0.14": { + "agent_version": "107.0.13.8702-1", + "tools_version": "100.10.0" + }, + "7.0.15": { + "agent_version": "107.0.15.8741-1", + "tools_version": "100.11.0" + }, + "8.0.5": { + "agent_version": "108.0.4.8770-1", + "tools_version": "100.11.0" + }, + "8.0.6": { + "agent_version": "108.0.6.8796-1", + "tools_version": "100.11.0" + }, + "8.0.7": { + "agent_version": "108.0.7.8810-1", + "tools_version": "100.12.0" + } + } + }, + "variants": [ + "ubi" + ] + }, + "init-ops-manager": { + "Description": "The lowest version corresponds to the lowest supported Operator version, see https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Operator+Support+Policy", + "ssdlc_name": "MongoDB Controllers for Kubernetes Init Ops Manager", + "versions": [ + "1.0.0", + "1.0.1", + "1.1.0" + ], + "variants": [ + "ubi" + ] + }, + "init-database": { + "Description": "The lowest version corresponds to the lowest supported Operator version, see https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Operator+Support+Policy", + "ssdlc_name": "MongoDB Controllers for Kubernetes Init Database", + "versions": [ + "1.0.0", + "1.0.1", + "1.1.0" + ], + "variants": [ + "ubi" + ] + }, + "init-appdb": { + "Description": "The lowest version corresponds to the lowest supported Operator version, see https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Operator+Support+Policy", + "ssdlc_name": "MongoDB Controllers for Kubernetes Init AppDB", + "versions": [ + "1.0.0", + "1.0.1", + "1.1.0" + ], + "variants": [ + "ubi" + ] + }, + "database": { + "Description": "The lowest version corresponds to the lowest supported Operator version, see https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Operator+Support+Policy", + "ssdlc_name": "MongoDB Controllers for Kubernetes Database", + "versions": [ + "1.0.0", + "1.0.1", + "1.1.0" + ], + "variants": [ + "ubi" + ] + }, + "mongodb-enterprise-server": { + "Description": "The lowest version corresponds to the lowest supported Operator version, see https://wiki.corp.mongodb.com/display/MMS/Kubernetes+Operator+Support+Policy", + "ssdlc_name": "MongoDB Enterprise Server", + "versions": [ + "4.4.0-ubi8", + "4.4.1-ubi8", + "4.4.2-ubi8", + "4.4.3-ubi8", + "4.4.4-ubi8", + "4.4.5-ubi8", + "4.4.6-ubi8", + "4.4.7-ubi8", + "4.4.8-ubi8", + "4.4.9-ubi8", + "4.4.10-ubi8", + "4.4.11-ubi8", + "4.4.12-ubi8", + "4.4.13-ubi8", + "4.4.14-ubi8", + "4.4.15-ubi8", + "4.4.16-ubi8", + "4.4.17-ubi8", + "4.4.18-ubi8", + "4.4.19-ubi8", + "4.4.20-ubi8", + "4.4.21-ubi8", + "5.0.0-ubi8", + "5.0.1-ubi8", + "5.0.2-ubi8", + "5.0.3-ubi8", + "5.0.4-ubi8", + "5.0.5-ubi8", + "5.0.6-ubi8", + "5.0.7-ubi8", + "5.0.8-ubi8", + "5.0.9-ubi8", + "5.0.10-ubi8", + "5.0.11-ubi8", + "5.0.12-ubi8", + "5.0.13-ubi8", + "5.0.14-ubi8", + "5.0.15-ubi8", + "5.0.16-ubi8", + "5.0.17-ubi8", + "5.0.18-ubi8", + "6.0.0-ubi8", + "6.0.1-ubi8", + "6.0.2-ubi8", + "6.0.3-ubi8", + "6.0.4-ubi8", + "6.0.5-ubi8", + "8.0.0-ubi8", + "8.0.0-ubi9" + ], + "variants": [ + "ubi" + ] + } + } +} diff --git a/docker/mongodb-kubernetes-upgrade-hook/Dockerfile b/docker/mongodb-kubernetes-upgrade-hook/Dockerfile index 362831582..5005f5801 100644 --- a/docker/mongodb-kubernetes-upgrade-hook/Dockerfile +++ b/docker/mongodb-kubernetes-upgrade-hook/Dockerfile @@ -1,6 +1,11 @@ -ARG imagebase -FROM ${imagebase} as base +FROM public.ecr.aws/docker/library/golang:1.24 AS builder + +WORKDIR /go/src +ADD . . + +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -a -o /data/scripts/version-upgrade-hook ./mongodb-community-operator/cmd/versionhook/main.go FROM registry.access.redhat.com/ubi9/ubi-minimal -COPY --from=base /version-upgrade-hook /version-upgrade-hook +COPY --from=builder /data/scripts/version-upgrade-hook /version-upgrade-hook diff --git a/docker/mongodb-kubernetes-upgrade-hook/README.md b/docker/mongodb-kubernetes-upgrade-hook/README.md new file mode 100644 index 000000000..9205118c2 --- /dev/null +++ b/docker/mongodb-kubernetes-upgrade-hook/README.md @@ -0,0 +1,10 @@ +### Building locally + +For building the readiness probe image locally use the example command: + +```bash +VERSION="1.0.9" +TARGETARCH="amd64" +docker buildx build --load --progress plain . -f docker/mongodb-kubernetes-upgrade-hook/Dockerfile -t "mongodb-kubernetes-upgrade-hook:${VERSION}" \ + --build-arg TARGETARCH="${TARGETARCH}" +``` diff --git a/ideal_release_flow.md b/ideal_release_flow.md new file mode 100644 index 000000000..6ce43a0fb --- /dev/null +++ b/ideal_release_flow.md @@ -0,0 +1,66 @@ +## Release from master + +```mermaid +%%{ + init: { + 'logLevel': 'debug', + 'theme': 'dark', + 'gitGraph': { + 'showBranches': true, + 'mainBranchName': 'master', + 'parallelCommits': 'true' + } + } +}%% +gitGraph + checkout master + commit id: "A1" tag:"v1.0.0" + commit id: "A2" + commit id: "A3" tag:"v1.1.0" + commit id: "A4" tag:"v1.2.0" + commit id: "A5" + commit id: "A6" + commit id: "A7" tag:"v2.0.0" + commit id: "A8" + commit id: "A9" tag:"v2.1.0" + commit id: "A10" + commit id: "A11" tag: "v3.0.0" +``` + +## Patching previous versions + +```mermaid +%%{ + init: { + 'logLevel': 'debug', + 'theme': 'dark', + 'gitGraph': { + 'showBranches': true, + 'mainBranchName': 'master', + 'parallelCommits': 'true' + } + } +}%% +gitGraph + checkout master + commit id: "A1" tag: "v1.0.0" + commit id: "A2" + commit id: "A3" tag: "v1.1.0" + commit id: "A4" tag: "v1.2.0" + branch release-1.x + commit id: "B1" tag: "v1.2.1" + commit id: "B2" + commit id: "B3" tag: "v1.2.2" + checkout master + commit id: "A5" + commit id: "A6" + commit id: "A7" tag:"v2.0.0" + commit id: "A8" + commit id: "A9" tag:"v2.1.0" + branch release-2.x + commit id: "C1" tag: "v2.1.1" + commit id: "C2" tag: "v2.1.2" + checkout master + commit id: "A10" + commit id: "A11" tag: "v3.0.0" +``` diff --git a/ideal_release_flow.mmd b/ideal_release_flow.mmd new file mode 100644 index 000000000..c883bac17 --- /dev/null +++ b/ideal_release_flow.mmd @@ -0,0 +1,28 @@ +%%{ + init: { + 'theme': 'dark', + 'logLevel': 'debug', + 'gitGraph': { + 'showBranches': true, + 'mainBranchName': 'master', + 'parallelCommits': true + }, + 'themeVariables': { + 'commitLabelColor': '#ffffff', + 'commitLabelBackground': '#333333' + } + } +}%% +gitGraph + checkout master + commit id: "A1" tag:"v1.0.0" + commit id: "A2" + commit id: "A3" tag:"v1.1.0" + commit id: "A4" tag:"v1.2.0" + commit id: "A5" + commit id: "A6" + commit id: "A7" tag:"v2.0.0" + commit id: "A8" + commit id: "A9" tag:"v2.1.0" + commit id: "A10" + commit id: "A11" tag: "v3.0.0" diff --git a/ideal_release_flow.png b/ideal_release_flow.png new file mode 100644 index 000000000..55deaaf52 Binary files /dev/null and b/ideal_release_flow.png differ diff --git a/ideal_release_flow_backport.mmd b/ideal_release_flow_backport.mmd new file mode 100644 index 000000000..3f378ea1e --- /dev/null +++ b/ideal_release_flow_backport.mmd @@ -0,0 +1,37 @@ +%%{ + init: { + 'logLevel': 'debug', + 'theme': 'dark', + 'gitGraph': { + 'showBranches': true, + 'mainBranchName': 'master', + 'parallelCommits': true + }, + 'themeVariables': { + 'commitLabelColor': '#ffffff', + 'commitLabelBackground': '#333333' + } + } +}%% +gitGraph + checkout master + commit id: "A1" tag: "v1.0.0" + commit id: "A2" + commit id: "A3" tag: "v1.1.0" + commit id: "A4" tag: "v1.2.0" + branch release-1.x + commit id: "B1" tag: "v1.2.1" + commit id: "B2" + commit id: "B3" tag: "v1.2.2" + checkout master + commit id: "A5" + commit id: "A6" + commit id: "A7" tag:"v2.0.0" + commit id: "A8" + commit id: "A9" tag:"v2.1.0" + branch release-2.x + commit id: "C1" tag: "v2.1.1" + commit id: "C2" tag: "v2.1.2" + checkout master + commit id: "A10" + commit id: "A11" tag: "v3.0.0" diff --git a/ideal_release_flow_backport.png b/ideal_release_flow_backport.png new file mode 100644 index 000000000..14ccde1fc Binary files /dev/null and b/ideal_release_flow_backport.png differ diff --git a/ideal_release_flow_versioning_complex.mmd b/ideal_release_flow_versioning_complex.mmd new file mode 100644 index 000000000..baec7cace --- /dev/null +++ b/ideal_release_flow_versioning_complex.mmd @@ -0,0 +1,27 @@ +%%{ + init: { + 'theme': 'dark', + 'logLevel': 'debug', + 'gitGraph': { + 'showBranches': true, + 'mainBranchName': 'master', + 'parallelCommits': true + }, + 'themeVariables': { + 'commitLabelColor': '#ffffff', + 'commitLabelBackground': '#333333' + } + } +}%% +gitGraph + checkout master + commit id: "A1" tag:"v1.0.0" + commit id: "A2: fix v1.0.1" + commit id: "A3: feature v1.1.0" + commit id: "A4: fix v1.1.0" + commit id: "A5: fix v1.1.0" tag:"v1.1.0" + commit id: "A6: feature v1.2.0" + commit id: "A7: feature v1.2.0" + commit id: "A8: breaking v2.0.0" + commit id: "A9: breaking v2.0.0" tag:"v2.0.0" + commit id: "A10: feature v2.1.0" diff --git a/ideal_release_flow_versioning_complex.png b/ideal_release_flow_versioning_complex.png new file mode 100644 index 000000000..71422202e Binary files /dev/null and b/ideal_release_flow_versioning_complex.png differ diff --git a/ideal_release_flow_versioning_easy.mmd b/ideal_release_flow_versioning_easy.mmd new file mode 100644 index 000000000..566baba33 --- /dev/null +++ b/ideal_release_flow_versioning_easy.mmd @@ -0,0 +1,26 @@ +%%{ + init: { + 'theme': 'dark', + 'logLevel': 'debug', + 'gitGraph': { + 'showBranches': true, + 'mainBranchName': 'master', + 'parallelCommits': true + }, + 'themeVariables': { + 'commitLabelColor': '#ffffff', + 'commitLabelBackground': '#333333' + } + } +}%% +gitGraph + checkout master + commit id: "A1" tag:"v1.0.0" + commit id: "A2: fix v1.0.1" + commit id: "A3: fix v1.0.1" + commit id: "A4: fix v1.0.1" tag:"v1.0.1" + commit id: "A5: feature v1.1.0" + commit id: "A6: feature v1.1.0" + commit id: "A7: feature v1.1.0" tag:"v1.1.0" + commit id: "A8: feature v1.2.0" tag:"v1.2.0" + commit id: "A9: feature v1.3.0" diff --git a/ideal_release_flow_versioning_easy.png b/ideal_release_flow_versioning_easy.png new file mode 100644 index 000000000..b5bf22186 Binary files /dev/null and b/ideal_release_flow_versioning_easy.png differ diff --git a/inventories/database.yaml b/inventories/database.yaml index 05d123f31..b179a30cf 100644 --- a/inventories/database.yaml +++ b/inventories/database.yaml @@ -11,6 +11,7 @@ images: stages: - name: database-build-context task_type: docker_build + tags: ["final_dockerfile"] dockerfile: Dockerfile.builder output: - registry: $(inputs.params.registry)/mongodb-kubernetes-database-context @@ -19,7 +20,7 @@ images: - name: init-appdb-template-ubi task_type: dockerfile_template distro: ubi - tags: ["ubi"] + tags: ["ubi", "final_dockerfile"] inputs: - version output: @@ -28,12 +29,11 @@ images: - name: database-build-ubi task_type: docker_build dockerfile: $(stages['init-appdb-template-ubi'].outputs[0].dockerfile) - tags: ["ubi"] + tags: ["ubi", "final_dockerfile"] buildargs: imagebase: $(inputs.params.registry)/mongodb-kubernetes-database-context:$(inputs.params.version_id) output: - - registry: $(inputs.params.registry)/mongodb-kubernetes-database - tag: $(inputs.params.version_id) + - dockerfile: $(functions.tempfile) - name: master-latest task_type: tag_image diff --git a/inventories/init_appdb.yaml b/inventories/init_appdb.yaml index 50d5d4199..228968135 100644 --- a/inventories/init_appdb.yaml +++ b/inventories/init_appdb.yaml @@ -6,13 +6,14 @@ images: - name: init-appdb vars: context: . - template_context: docker/mongodb-kubernetes-init-database + template_context: docker/mongodb-kubernetes-init-appdb platform: linux/amd64 stages: - name: init-appdb-build-context task_type: docker_build - dockerfile: docker/mongodb-kubernetes-init-database/Dockerfile.builder + tags: ["final_dockerfile"] + dockerfile: docker/mongodb-kubernetes-init-appdb/Dockerfile.builder buildargs: mongodb_tools_url_ubi: $(inputs.params.mongodb_tools_url_ubi) output: @@ -22,7 +23,7 @@ images: - name: init-appdb-template-ubi task_type: dockerfile_template template_file_extension: ubi_minimal - tags: ["ubi"] + tags: ["final_dockerfile"] inputs: - is_appdb output: diff --git a/inventories/init_database.yaml b/inventories/init_database.yaml index 57ab81679..15901536c 100644 --- a/inventories/init_database.yaml +++ b/inventories/init_database.yaml @@ -12,6 +12,7 @@ images: stages: - name: init-database-build-context task_type: docker_build + tags: ["final_dockerfile"] dockerfile: docker/mongodb-kubernetes-init-database/Dockerfile.builder buildargs: mongodb_tools_url_ubi: $(inputs.params.mongodb_tools_url_ubi) @@ -24,7 +25,7 @@ images: - name: init-database-template-ubi task_type: dockerfile_template template_file_extension: ubi_minimal - tags: ["ubi"] + tags: ["ubi", "final_dockerfile"] inputs: - is_appdb output: diff --git a/inventories/init_om.yaml b/inventories/init_om.yaml index f3d310470..c05d34c3c 100644 --- a/inventories/init_om.yaml +++ b/inventories/init_om.yaml @@ -11,6 +11,7 @@ images: stages: - name: init-ops-manager-build-context task_type: docker_build + tags: ["final_dockerfile"] dockerfile: Dockerfile.builder output: - registry: $(inputs.params.registry)/mongodb-kubernetes-init-ops-manager-context @@ -19,7 +20,7 @@ images: - name: init-ops-manager-template-ubi task_type: dockerfile_template template_file_extension: ubi_minimal - tags: ["ubi"] + tags: ["ubi", "final_dockerfile"] inputs: - version output: diff --git a/inventories/om.yaml b/inventories/om.yaml index e4daf3103..d9b802376 100644 --- a/inventories/om.yaml +++ b/inventories/om.yaml @@ -13,6 +13,7 @@ images: stages: - name: ops-manager-context task_type: docker_build + tags: ["final_dockerfile"] dockerfile: docker/mongodb-enterprise-ops-manager/Dockerfile.builder output: - registry: $(inputs.params.registry)/ops-manager-context @@ -21,7 +22,7 @@ images: - name: ops-manager-template-ubi task_type: dockerfile_template template_file_extension: ubi - tags: ["ubi"] + tags: ["ubi", "final_dockerfile"] inputs: - om_download_url - version diff --git a/inventory.yaml b/inventory.yaml index d4beb4137..7d96add38 100644 --- a/inventory.yaml +++ b/inventory.yaml @@ -39,6 +39,7 @@ images: - name: operator-template-ubi task_type: dockerfile_template + tags: ["final_dockerfile"] distro: ubi inputs: - version diff --git a/lib/sonar/sonar.py b/lib/sonar/sonar.py old mode 100644 new mode 100755 diff --git a/lib/sonar/test/test_final_dockerfiles.py b/lib/sonar/test/test_final_dockerfiles.py new file mode 100644 index 000000000..f4516baea --- /dev/null +++ b/lib/sonar/test/test_final_dockerfiles.py @@ -0,0 +1,101 @@ +from unittest import skip + +from ..sonar import process_image + + +@skip("This test case is only used to generate the final Dockerfile for ops-manager") +def test_build_om_dockerfile(): + process_image( + image_name="ops-manager", + skip_tags=["release"], + include_tags=["final_dockerfile"], + build_args={ + "registry": "localhost:5000", + "version": "8.0.7", + "om_download_url": "https://downloads.mongodb.com/on-prem-mms/tar/mongodb-mms-8.0.7.500.20250505T1426Z.tar.gz", + }, + build_options={}, + inventory="inventories/om.yaml", + ) + + +@skip("This test case is only used to generate the final Dockerfile for database") +def test_build_database_dockerfile(): + process_image( + image_name="database", + skip_tags=["release"], + include_tags=["final_dockerfile"], + build_args={ + "registry": "localhost:5000", + "version": "1.1.0", + }, + build_options={}, + inventory="inventories/database.yaml", + ) + + +@skip("This test case is only used to generate the final Dockerfile for init appdb") +def test_build_init_appdb_dockerfile(): + process_image( + image_name="init-appdb", + skip_tags=["release"], + include_tags=["final_dockerfile"], + build_args={ + "registry": "localhost:5000", + "version": "1.1.0", + "is_appdb": True, + "mongodb_tools_url_ubi": "https://downloads.mongodb.org/tools/db/mongodb-database-tools-rhel93-x86_64-100.12.0.tgz", + }, + build_options={}, + inventory="inventories/init_appdb.yaml", + ) + + +@skip("This test case is only used to generate the final Dockerfile for init database") +def test_build_init_database_dockerfile(): + process_image( + image_name="init-database", + skip_tags=["release"], + include_tags=["final_dockerfile"], + build_args={ + "registry": "localhost:5000", + "version": "1.1.0", + "is_appdb": False, + "mongodb_tools_url_ubi": "https://downloads.mongodb.org/tools/db/mongodb-database-tools-rhel93-x86_64-100.12.0.tgz", + }, + build_options={}, + inventory="inventories/init_database.yaml", + ) + + +@skip("This test case is only used to generate the final Dockerfile for init ops manager") +def test_build_init_ops_manager_dockerfile(): + process_image( + image_name="init-ops-manager", + skip_tags=["release"], + include_tags=["final_dockerfile"], + build_args={ + "registry": "localhost:5000", + "version": "1.1.0", + }, + build_options={}, + inventory="inventories/init_om.yaml", + ) + + +def test_build_operator_dockerfile(): + process_image( + image_name="mongodb-kubernetes", + skip_tags=["release"], + include_tags=["final_dockerfile"], + build_args={ + "version": "1.1.0", + "registry": "localhost:5000", + "release_version": "1.1.0", + "log_automation_config_diff": "false", + "use_race": "false", + "debug": False, + }, + build_options={}, + inventory="inventory.yaml", + ) diff --git a/requirements.txt b/requirements.txt index 44fa0c412..3bb2bcd9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,7 @@ pytest-mock==3.14.1 wrapt==1.17.2 botocore==1.38.23 boto3==1.38.23 +python-on-whales # from kubeobject freezegun==1.5.2 diff --git a/scripts/release/__init__.py b/scripts/release/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/release/atomic_pipeline.py b/scripts/release/atomic_pipeline.py new file mode 100755 index 000000000..915f7f086 --- /dev/null +++ b/scripts/release/atomic_pipeline.py @@ -0,0 +1,856 @@ +#!/usr/bin/env python3 + +"""This pipeline script knows about the details of our Docker images +and where to fetch and calculate parameters. It uses Sonar.py +to produce the final images.""" +import json +import os +import shutil +from concurrent.futures import ProcessPoolExecutor +from queue import Queue +from typing import Callable, Dict, List, Optional, Tuple, Union + +import requests +import semver +from opentelemetry import trace +from packaging.version import Version + + +from lib.base_logger import logger +from scripts.evergreen.release.agent_matrix import ( + get_supported_operator_versions, +) +from scripts.evergreen.release.images_signing import ( + mongodb_artifactory_login, + sign_image, + verify_signature, +) +from scripts.evergreen.release.sbom import generate_sbom, generate_sbom_for_cli + +from .build_configuration import BuildConfiguration +from .build_context import BuildScenario +from .build_images import process_image +from .optimized_operator_build import build_operator_image_fast + +TRACER = trace.get_tracer("evergreen-agent") +DEFAULT_NAMESPACE = "default" + +# TODO: rename architecture -> platform everywhere + +def make_list_of_str(value: Union[None, str, List[str]]) -> List[str]: + if value is None: + return [] + + if isinstance(value, str): + return [e.strip() for e in value.split(",")] + + return value + + +def get_tools_distro(tools_version: str) -> Dict[str, str]: + new_rhel_tool_version = "100.10.0" + default_distro = {"arm": "rhel90-aarch64", "amd": "rhel90-x86_64"} + if Version(tools_version) >= Version(new_rhel_tool_version): + return {"arm": "rhel93-aarch64", "amd": "rhel93-x86_64"} + return default_distro + + +def is_running_in_evg_pipeline(): + return os.getenv("RUNNING_IN_EVG", "") == "true" + + +def is_running_in_patch(): + is_patch = os.environ.get("is_patch") + return is_patch is not None and is_patch.lower() == "true" + + +def load_release_file() -> Dict: + with open("release.json") as release: + return json.load(release) + + +@TRACER.start_as_current_span("sonar_build_image") +def pipeline_process_image( + image_name: str, + dockerfile_path: str, + build_configuration: BuildConfiguration, + dockerfile_args: Dict[str, str] = None, + build_path: str = ".", + with_sbom: bool = True, +): + """Builds a Docker image with arguments defined in `args`.""" + span = trace.get_current_span() + span.set_attribute("mck.image_name", image_name) + if dockerfile_args: + span.set_attribute("mck.build_args", str(dockerfile_args)) + + # TODO use these? + build_options = { + # Will continue building an image if it finds an error. See next comment. + "continue_on_errors": True, + # But will still fail after all the tasks have completed + "fail_on_errors": True, + } + + logger.info(f"Dockerfile args: {dockerfile_args}, for image: {image_name}") + + if not dockerfile_args: + dockerfile_args = {} + logger.debug(f"Build args: {dockerfile_args}") + process_image( + image_name, + image_tag=build_configuration.version, + dockerfile_path=dockerfile_path, + dockerfile_args=dockerfile_args, + base_registry=build_configuration.base_registry, + platforms=build_configuration.platforms, + sign=build_configuration.sign, + build_path=build_path, + ) + + if with_sbom: + produce_sbom(dockerfile_args) + + +@TRACER.start_as_current_span("produce_sbom") +def produce_sbom(args): + span = trace.get_current_span() + if not is_running_in_evg_pipeline(): + logger.info("Skipping SBOM Generation (enabled only for EVG)") + return + + try: + image_pull_spec = args["quay_registry"] + args.get("ubi_suffix", "") + except KeyError: + logger.error(f"Could not find image pull spec. Args: {args}") + logger.error(f"Skipping SBOM generation") + return + + try: + image_tag = args["release_version"] + span.set_attribute("mck.release_version", image_tag) + except KeyError: + logger.error(f"Could not find image tag. Args: {args}") + logger.error(f"Skipping SBOM generation") + return + + image_pull_spec = f"{image_pull_spec}:{image_tag}" + print(f"Producing SBOM for image: {image_pull_spec} args: {args}") + + platform = "linux/amd64" + if "platform" in args: + if args["platform"] == "arm64": + platform = "linux/arm64" + elif args["platform"] == "amd64": + platform = "linux/amd64" + else: + # TODO: return here? + logger.error(f"Unrecognized architectures in {args}. Skipping SBOM generation") + + generate_sbom(image_pull_spec, platform) + + +def build_tests_image(build_configuration: BuildConfiguration): + """ + Builds image used to run tests. + """ + image_name = "mongodb-kubernetes-tests" + + # helm directory needs to be copied over to the tests docker context. + helm_src = "helm_chart" + helm_dest = "docker/mongodb-kubernetes-tests/helm_chart" + requirements_dest = "docker/mongodb-kubernetes-tests/requirements.txt" + public_src = "public" + public_dest = "docker/mongodb-kubernetes-tests/public" + + # Remove existing directories/files if they exist + shutil.rmtree(helm_dest, ignore_errors=True) + shutil.rmtree(public_dest, ignore_errors=True) + + # Copy directories and files (recursive copy) + shutil.copytree(helm_src, helm_dest) + shutil.copytree(public_src, public_dest) + shutil.copyfile("release.json", "docker/mongodb-kubernetes-tests/release.json") + shutil.copyfile("requirements.txt", requirements_dest) + + python_version = os.getenv("PYTHON_VERSION", "3.11") + if python_version == "": + raise Exception("Missing PYTHON_VERSION environment variable") + + buildargs = dict({"PYTHON_VERSION": python_version}) + + pipeline_process_image( + image_name, + dockerfile_path="Dockerfile", + build_configuration=build_configuration, + dockerfile_args=buildargs, + build_path="docker/mongodb-kubernetes-tests", + ) + + +def build_mco_tests_image(build_configuration: BuildConfiguration): + """ + Builds image used to run community tests. + """ + image_name = "mongodb-community-tests" + golang_version = os.getenv("GOLANG_VERSION", "1.24") + if golang_version == "": + raise Exception("Missing GOLANG_VERSION environment variable") + + buildargs = dict({"GOLANG_VERSION": golang_version}) + + pipeline_process_image( + image_name, + dockerfile_path="docker/mongodb-community-tests/Dockerfile", + build_configuration=build_configuration, + dockerfile_args=buildargs, + ) + + +def build_operator_image(build_configuration: BuildConfiguration): + """Calculates arguments required to build the operator image, and starts the build process.""" + # In evergreen, we can pass test_suffix env to publish the operator to a quay + # repository with a given suffix. + test_suffix = os.environ.get("test_suffix", "") + log_automation_config_diff = os.environ.get("LOG_AUTOMATION_CONFIG_DIFF", "false") + + args = { + "version": build_configuration.version, + "log_automation_config_diff": log_automation_config_diff, + "test_suffix": test_suffix, + "debug": build_configuration.debug, + } + + logger.info(f"Building Operator args: {args}") + + image_name = "mongodb-kubernetes" + build_image_generic( + image_name=image_name, + dockerfile_path="docker/mongodb-kubernetes-operator/Dockerfile", + build_configuration=build_configuration, + extra_args=args, + ) + + +def build_operator_image_patch(build_configuration: BuildConfiguration): + if not build_operator_image_fast(build_configuration): + build_operator_image(build_configuration) + + +def build_database_image(build_configuration: BuildConfiguration): + """ + Builds a new database image. + """ + release = load_release_file() + version = release["databaseImageVersion"] + args = {"version": build_configuration.version} + build_image_generic( + image_name="mongodb-kubernetes-database", + dockerfile_path="docker/mongodb-kubernetes-database/Dockerfile", + build_configuration=build_configuration, + extra_args=args, + ) + + +def build_CLI_SBOM(build_configuration: BuildConfiguration): + if not is_running_in_evg_pipeline(): + logger.info("Skipping SBOM Generation (enabled only for EVG)") + return + + if build_configuration.platforms is None or len(build_configuration.platforms) == 0: + architectures = ["linux/amd64", "linux/arm64", "darwin/arm64", "darwin/amd64"] + elif "arm64" in build_configuration.platforms: + architectures = ["linux/arm64", "darwin/arm64"] + elif "amd64" in build_configuration.platforms: + architectures = ["linux/amd64", "darwin/amd64"] + else: + logger.error(f"Unrecognized architectures {build_configuration.platforms}. Skipping SBOM generation") + return + + release = load_release_file() + version = release["mongodbOperator"] + + for architecture in architectures: + generate_sbom_for_cli(version, architecture) + + +def should_skip_arm64(): + """ + Determines if arm64 builds should be skipped based on environment. + Returns True if running in Evergreen pipeline as a patch. + """ + return is_running_in_evg_pipeline() and is_running_in_patch() + + +@TRACER.start_as_current_span("sign_image_in_repositories") +def sign_image_in_repositories(args: Dict[str, str], arch: str = None): + span = trace.get_current_span() + repository = args["quay_registry"] + args["ubi_suffix"] + tag = args["release_version"] + if arch: + tag = f"{tag}-{arch}" + + span.set_attribute("mck.tag", tag) + + sign_image(repository, tag) + verify_signature(repository, tag) + + +def find_om_in_releases(om_version: str, releases: Dict[str, str]) -> Optional[str]: + """ + There are a few alternatives out there that allow for json-path or xpath-type + traversal of Json objects in Python, I don't have time to look for one of + them now but I have to do at some point. + """ + for release in releases: + if release["version"] == om_version: + for platform in release["platform"]: + if platform["package_format"] == "deb" and platform["arch"] == "x86_64": + for package in platform["packages"]["links"]: + if package["name"] == "tar.gz": + return package["download_link"] + return None + + +def get_om_releases() -> Dict[str, str]: + """Returns a dictionary representation of the Json document holdin all the OM + releases. + """ + ops_manager_release_archive = ( + "https://info-mongodb-com.s3.amazonaws.com/com-download-center/ops_manager_release_archive.json" + ) + + return requests.get(ops_manager_release_archive).json() + + +def find_om_url(om_version: str) -> str: + """Gets a download URL for a given version of OM.""" + releases = get_om_releases() + + current_release = find_om_in_releases(om_version, releases["currentReleases"]) + if current_release is None: + current_release = find_om_in_releases(om_version, releases["oldReleases"]) + + if current_release is None: + raise ValueError("Ops Manager version {} could not be found".format(om_version)) + + return current_release + + +def build_init_om_image(build_configuration: BuildConfiguration): + release = load_release_file() + version = release["initOpsManagerVersion"] + args = {"version": build_configuration.version} + build_image_generic( + image_name="mongodb-kubernetes-init-ops-manager", + dockerfile_path="docker/mongodb-kubernetes-init-ops-manager/Dockerfile", + build_configuration=build_configuration, + extra_args=args, + ) + + +def build_om_image(build_configuration: BuildConfiguration): + # Make this a parameter for the Evergreen build + # https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds + om_version = os.environ.get("om_version") + if om_version is None: + raise ValueError("`om_version` should be defined.") + + om_download_url = os.environ.get("om_download_url", "") + if om_download_url == "": + om_download_url = find_om_url(om_version) + + args = { + "version": om_version, + "om_download_url": om_download_url, + } + + build_image_generic( + image_name="mongodb-enterprise-ops-manager-ubi", + dockerfile_path="docker/mongodb-enterprise-ops-manager/Dockerfile", + build_configuration=build_configuration, + extra_args=args, + ) + + +def build_image_generic( + image_name: str, + dockerfile_path: str, + build_configuration: BuildConfiguration, + extra_args: dict | None = None, + multi_arch_args_list: list[dict] | None = None, + is_multi_arch: bool = False, +): + """ + Build one or more architecture-specific images, then (optionally) + push a manifest and sign the result. + """ + + # 1) Defaults + registry = build_configuration.base_registry + args_list = multi_arch_args_list or [extra_args or {}] + version = args_list[0].get("version", "") + architectures = [args.get("architecture") for args in args_list] + + # 2) Build each arch + for base_args in args_list: + # merge in the registry without mutating caller’s dict + build_args = {**base_args, "quay_registry": registry} + logger.debug(f"Build args: {build_args}") + + for arch in architectures: + logger.debug(f"Building {image_name} for arch={arch}") + logger.debug(f"build image generic - registry={registry}") + pipeline_process_image( + image_name=image_name, + image_tag=version, + dockerfile_path=dockerfile_path, + dockerfile_args=build_args, + base_registry=registry, + platforms=arch, + sign=False, + with_sbom=False, + ) + + # 3) Multi-arch manifest + if is_multi_arch: + create_and_push_manifest(registry + "/" + image_name, version, architectures=architectures) + + # 4) Signing (only on real releases) + if build_configuration.sign: + sign_image(registry, version) + verify_signature(registry, version) + + +def build_init_appdb(build_configuration: BuildConfiguration): + release = load_release_file() + version = release["initAppDbVersion"] + base_url = "https://fastdl.mongodb.org/tools/db/" + mongodb_tools_url_ubi = "{}{}".format(base_url, release["mongodbToolsBundle"]["ubi"]) + args = {"version": build_configuration.version, "mongodb_tools_url_ubi": mongodb_tools_url_ubi} + build_image_generic( + image_name="mongodb-kubernetes-init-appdb", + dockerfile_path="docker/mongodb-kubernetes-init-appdb/Dockerfile", + build_configuration=build_configuration, + extra_args=args, + ) + + +# TODO: nam static: remove this once static containers becomes the default +def build_init_database(build_configuration: BuildConfiguration): + release = load_release_file() + version = release["initDatabaseVersion"] # comes from release.json + base_url = "https://fastdl.mongodb.org/tools/db/" + mongodb_tools_url_ubi = "{}{}".format(base_url, release["mongodbToolsBundle"]["ubi"]) + args = {"version": build_configuration.version, "mongodb_tools_url_ubi": mongodb_tools_url_ubi} + build_image_generic( + "mongodb-kubernetes-init-database", + "docker/mongodb-kubernetes-init-database/Dockerfile", + build_configuration=build_configuration, + extra_args=args, + ) + + +def build_community_image(build_configuration: BuildConfiguration, image_type: str): + """ + Builds image for community components (readiness probe, upgrade hook). + + Args: + build_configuration: The build configuration to use + image_type: Type of image to build ("readiness-probe" or "upgrade-hook") + """ + + if image_type == "readiness-probe": + image_name = "mongodb-kubernetes-readinessprobe" + dockerfile_path = "docker/mongodb-kubernetes-readinessprobe/Dockerfile" + elif image_type == "upgrade-hook": + image_name = "mongodb-kubernetes-operator-version-upgrade-post-start-hook" + dockerfile_path = "docker/mongodb-kubernetes-upgrade-hook/Dockerfile" + else: + raise ValueError(f"Unsupported image type: {image_type}") + + version = build_configuration.version + golang_version = os.getenv("GOLANG_VERSION", "1.24") + + # Use only amd64 if we should skip arm64 builds + if should_skip_arm64(): + platforms = ["linux/amd64"] + logger.info("Skipping ARM64 builds for community image as this is running in EVG pipeline as a patch") + else: + platforms = build_configuration.platforms or ["linux/amd64", "linux/arm64"] + + # Extract architectures from platforms for build args + architectures = [platform.split("/")[-1] for platform in platforms] + multi_arch_args_list = [] + + for arch in architectures: + arch_args = { + "version": version, + "GOLANG_VERSION": golang_version, + "architecture": arch, + "TARGETARCH": arch, + } + multi_arch_args_list.append(arch_args) + + # Create a copy of build_configuration with overridden platforms + from copy import copy + build_config_copy = copy(build_configuration) + build_config_copy.platforms = platforms + + build_image_generic( + image_name=image_name, + dockerfile_path=dockerfile_path, + build_configuration=build_config_copy, + multi_arch_args_list=multi_arch_args_list, + is_multi_arch=True, + ) + + +def build_readiness_probe_image(build_configuration: BuildConfiguration): + """ + Builds image used for readiness probe. + """ + build_community_image(build_configuration, "readiness-probe") + + +def build_upgrade_hook_image(build_configuration: BuildConfiguration): + """ + Builds image used for version upgrade post-start hook. + """ + build_community_image(build_configuration, "upgrade-hook") + + +def build_agent_pipeline( + build_configuration: BuildConfiguration, + image_version, + init_database_image, + mongodb_tools_url_ubi, + mongodb_agent_url_ubi: str, + agent_version, +): + version = f"{agent_version}_{image_version}" + + args = { + "version": version, + "agent_version": agent_version, + "ubi_suffix": "-ubi", + "release_version": image_version, + "init_database_image": init_database_image, + "mongodb_tools_url_ubi": mongodb_tools_url_ubi, + "mongodb_agent_url_ubi": mongodb_agent_url_ubi, + "quay_registry": build_configuration.base_registry, + } + + build_image_generic( + image_name="mongodb-agent-ubi", + dockerfile_path="docker/mongodb-agent/Dockerfile", + build_configuration=build_configuration, + extra_args=args, + ) + + +def build_multi_arch_agent_in_sonar( + build_configuration: BuildConfiguration, + image_version, + tools_version, +): + """ + Creates the multi-arch non-operator suffixed version of the agent. + This is a drop-in replacement for the agent + release from MCO. + This should only be called during releases. + Which will lead to a release of the multi-arch + images to quay and ecr. + """ + + logger.info(f"building multi-arch base image for: {image_version}") + args = { + "version": image_version, + "tools_version": tools_version, + } + + arch_arm = { + "agent_distro": "amzn2_aarch64", + "tools_distro": get_tools_distro(tools_version=tools_version)["arm"], + "architecture": "arm64", + } + arch_amd = { + "agent_distro": "rhel9_x86_64", + "tools_distro": get_tools_distro(tools_version=tools_version)["amd"], + "architecture": "amd64", + } + + new_rhel_tool_version = "100.10.0" + if Version(tools_version) >= Version(new_rhel_tool_version): + arch_arm["tools_distro"] = "rhel93-aarch64" + arch_amd["tools_distro"] = "rhel93-x86_64" + + joined_args = [args | arch_amd] + + # Only include arm64 if we shouldn't skip it + if not should_skip_arm64(): + joined_args.append(args | arch_arm) + + build_image_generic( + image_name="mongodb-agent-ubi", + dockerfile_path="docker/mongodb-agent-non-matrix/Dockerfile", + build_configuration=build_config_copy, + is_multi_arch=True, + multi_arch_args_list=joined_args, + ) + +# TODO: why versions are wrong -> 13.35.0.9498-1_13.35.0.9498-1_6874c19d2aab5d0007820c51 ; duplicate +# TODO: figure out why I hit toomanyrequests: Rate exceeded with the new pipeline +def build_agent_default_case(build_configuration: BuildConfiguration): + """ + Build the agent only for the latest operator for patches and operator releases. + + See more information in the function: build_agent_on_agent_bump + """ + release = load_release_file() + + # We need to release [all agents x latest operator] on operator releases + if build_configuration.scenario == BuildScenario.RELEASE: + agent_versions_to_build = gather_all_supported_agent_versions(release) + # We only need [latest agents (for each OM major version and for CM) x patch ID] for patches + else: + agent_versions_to_build = gather_latest_agent_versions(release) + + logger.info( + f"Building Agent versions: {agent_versions_to_build} for Operator versions: {build_configuration.version}" + ) + + tasks_queue = Queue() + max_workers = 1 + if build_configuration.parallel: + max_workers = None + if build_configuration.parallel_factor > 0: + max_workers = build_configuration.parallel_factor + with ProcessPoolExecutor(max_workers=max_workers) as executor: + logger.info(f"running with factor of {max_workers}") + for agent_version in agent_versions_to_build: + # We don't need to keep create and push the same image on every build. + # It is enough to create and push the non-operator suffixed images only during releases to ecr and quay. + # if build_configuration.is_release_step_executed() or build_configuration.all_agents: + # tasks_queue.put( + # executor.submit( + # build_multi_arch_agent_in_sonar, + # build_configuration, + # agent_version[0], + # agent_version[1], + # ) + # ) + _build_agent_operator( + agent_version, + build_configuration, + executor, + build_configuration.version, + tasks_queue, + build_configuration.scenario == BuildScenario.RELEASE, + ) + + queue_exception_handling(tasks_queue) + +# TODO: for now, release agents ECR release versions with image:version_version (duplicated) +def build_agent_on_agent_bump(build_configuration: BuildConfiguration): + """ + Build the agent matrix (operator version x agent version), triggered by PCT. + + We have three cases where we need to build the agent: + - e2e test runs + - operator releases + - OM/CM bumps via PCT + + We don’t require building a full matrix on e2e test runs and operator releases. + "Operator releases" and "e2e test runs" require only the latest operator x agents + + In OM/CM bumps, we release a new agent which we potentially require to release to older operators as well. + This function takes care of that. + """ + release = load_release_file() + is_release = build_configuration.is_release_step_executed() + + if build_configuration.all_agents: + # We need to release [all agents x latest operator] on operator releases to make e2e tests work + # This was changed previously in https://github.com/mongodb/mongodb-kubernetes/pull/3960 + agent_versions_to_build = gather_all_supported_agent_versions(release) + else: + # we only need to release the latest images, we don't need to re-push old images, as we don't clean them up anymore. + agent_versions_to_build = gather_latest_agent_versions(release) + + legacy_agent_versions_to_build = release["supportedImages"]["mongodb-agent"]["versions"] + + tasks_queue = Queue() + max_workers = 1 + if build_configuration.parallel: + max_workers = None + if build_configuration.parallel_factor > 0: + max_workers = build_configuration.parallel_factor + with ProcessPoolExecutor(max_workers=max_workers) as executor: + logger.info(f"running with factor of {max_workers}") + + # We need to regularly push legacy agents, otherwise ecr lifecycle policy will expire them. + # We only need to push them once in a while to ecr, so no quay required + if not is_release: + for legacy_agent in legacy_agent_versions_to_build: + tasks_queue.put( + executor.submit( + build_multi_arch_agent_in_sonar, + build_configuration, + legacy_agent, + # we assume that all legacy agents are build using that tools version + "100.9.4", + ) + ) + + for agent_version in agent_versions_to_build: + # We don't need to keep create and push the same image on every build. + # It is enough to create and push the non-operator suffixed images only during releases to ecr and quay. + if build_configuration.is_release_step_executed() or build_configuration.all_agents: + tasks_queue.put( + executor.submit( + build_multi_arch_agent_in_sonar, + build_configuration, + agent_version[0], + agent_version[1], + ) + ) + for operator_version in get_supported_operator_versions(): + logger.info(f"Building Agent versions: {agent_version} for Operator versions: {operator_version}") + _build_agent_operator( + agent_version, build_configuration, executor, operator_version, tasks_queue, is_release + ) + + queue_exception_handling(tasks_queue) + + +def queue_exception_handling(tasks_queue): + exceptions_found = False + for task in tasks_queue.queue: + if task.exception() is not None: + exceptions_found = True + logger.fatal(f"The following exception has been found when building: {task.exception()}") + if exceptions_found: + raise Exception( + f"Exception(s) found when processing Agent images. \nSee also previous logs for more info\nFailing the build" + ) + + +def _build_agent_operator( + agent_version: Tuple[str, str], + build_configuration: BuildConfiguration, + executor: ProcessPoolExecutor, + operator_version: str, + tasks_queue: Queue, + use_quay: bool = False, +): + agent_distro = "rhel9_x86_64" + tools_version = agent_version[1] + tools_distro = get_tools_distro(tools_version)["amd"] + image_version = f"{agent_version[0]}_{operator_version}" + mongodb_tools_url_ubi = ( + f"https://downloads.mongodb.org/tools/db/mongodb-database-tools-{tools_distro}-{tools_version}.tgz" + ) + mongodb_agent_url_ubi = f"https://mciuploads.s3.amazonaws.com/mms-automation/mongodb-mms-build-agent/builds/automation-agent/prod/mongodb-mms-automation-agent-{agent_version[0]}.{agent_distro}.tar.gz" + init_database_image = f"{build_configuration.base_registry}/mongodb-kubernetes-init-database:{operator_version}" + + tasks_queue.put( + executor.submit( + build_agent_pipeline, + build_configuration, + image_version, + init_database_image, + mongodb_tools_url_ubi, + mongodb_agent_url_ubi, + agent_version[0], + ) + ) + + +def gather_all_supported_agent_versions(release: Dict) -> List[Tuple[str, str]]: + # This is a list of a tuples - agent version and corresponding tools version + agent_versions_to_build = list() + agent_versions_to_build.append( + ( + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"], + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager_tools"], + ) + ) + for _, om in release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"].items(): + agent_versions_to_build.append((om["agent_version"], om["tools_version"])) + + # lets not build the same image multiple times + return sorted(list(set(agent_versions_to_build))) + + +def gather_latest_agent_versions(release: Dict) -> List[Tuple[str, str]]: + """ + This function is used when we release a new agent via OM bump. + That means we will need to release that agent with all supported operators. + Since we don’t want to release all agents again, we only release the latest, which will contain the newly added one + :return: the latest agent for each major version + """ + agent_versions_to_build = list() + agent_versions_to_build.append( + ( + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager"], + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["cloud_manager_tools"], + ) + ) + + latest_versions = {} + + for version in release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"].keys(): + parsed_version = semver.VersionInfo.parse(version) + major_version = parsed_version.major + if major_version in latest_versions: + latest_parsed_version = semver.VersionInfo.parse(str(latest_versions[major_version])) + latest_versions[major_version] = max(parsed_version, latest_parsed_version) + else: + latest_versions[major_version] = version + + for major_version, latest_version in latest_versions.items(): + agent_versions_to_build.append( + ( + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"][str(latest_version)][ + "agent_version" + ], + release["supportedImages"]["mongodb-agent"]["opsManagerMapping"]["ops_manager"][str(latest_version)][ + "tools_version" + ], + ) + ) + + # TODO: Remove this once we don't need to use OM 7.0.12 in the OM Multicluster DR tests + # https://jira.mongodb.org/browse/CLOUDP-297377 + agent_versions_to_build.append(("107.0.12.8669-1", "100.10.0")) + + return sorted(list(set(agent_versions_to_build))) + + +def get_builder_function_for_image_name() -> Dict[str, Callable]: + """Returns a dictionary of image names that can be built.""" + + image_builders = { + "cli": build_CLI_SBOM, + "test": build_tests_image, + "operator": build_operator_image, + "mco-test": build_mco_tests_image, + # TODO: add support to build this per patch + "readiness-probe": build_readiness_probe_image, + "upgrade-hook": build_upgrade_hook_image, + "operator-quick": build_operator_image_patch, + "database": build_database_image, + "agent-pct": build_agent_on_agent_bump, + "agent": build_agent_default_case, + # + # Init images + "init-appdb": build_init_appdb, + "init-database": build_init_database, + "init-ops-manager": build_init_om_image, + # + # Ops Manager image + "ops-manager": build_om_image, + } + + return image_builders diff --git a/scripts/release/build_configuration.py b/scripts/release/build_configuration.py new file mode 100644 index 000000000..b62994d0e --- /dev/null +++ b/scripts/release/build_configuration.py @@ -0,0 +1,21 @@ +from dataclasses import dataclass +from typing import List, Optional + +from .build_context import BuildScenario + + +@dataclass +class BuildConfiguration: + scenario: BuildScenario + version: str + base_registry: str + + parallel: bool = False + parallel_factor: int = 0 + platforms: Optional[List[str]] = None + sign: bool = False + all_agents: bool = False + debug: bool = True + + def is_release_step_executed(self) -> bool: + return self.scenario == BuildScenario.RELEASE diff --git a/scripts/release/build_context.py b/scripts/release/build_context.py new file mode 100644 index 000000000..7ed98976e --- /dev/null +++ b/scripts/release/build_context.py @@ -0,0 +1,81 @@ +import os +from dataclasses import dataclass +from enum import Enum +from typing import Optional + +from lib.base_logger import logger + + +class BuildScenario(str, Enum): + """Represents the context in which the build is running.""" + + RELEASE = "release" # Official release build from a git tag + PATCH = "patch" # CI build for a patch/pull request + MASTER = "master" # CI build from a merge to the master + DEVELOPMENT = "development" # Local build on a developer machine + + +def infer_scenario_from_environment() -> BuildScenario: + """Infer the build scenario from environment variables.""" + git_tag = os.getenv("triggered_by_git_tag") + is_patch = os.getenv("is_patch", "false").lower() == "true" + is_evg = os.getenv("RUNNING_IN_EVG", "false").lower() == "true" + patch_id = os.getenv("version_id") + + if git_tag: + scenario = BuildScenario.RELEASE + logger.info(f"Build scenario: {scenario} (git_tag: {git_tag})") + elif is_patch: + scenario = BuildScenario.PATCH + logger.info(f"Build scenario: {scenario} (patch_id: {patch_id})") + elif is_evg: + scenario = BuildScenario.MASTER # TODO: ultimately we won't have RELEASE variant and master will push to staging + logger.info(f"Build scenario: {scenario} (patch_id: {patch_id})") + else: + scenario = BuildScenario.DEVELOPMENT + logger.info(f"Build scenario: {scenario}") + + return scenario + + +@dataclass +class BuildContext: + """Define build parameters based on the build scenario.""" + + scenario: BuildScenario + git_tag: Optional[str] = None + patch_id: Optional[str] = None + signing_enabled: bool = False + multi_arch: bool = True + version: Optional[str] = None + + @classmethod + def from_scenario(cls, scenario: BuildScenario) -> "BuildContext": + """Create build context from a given scenario.""" + git_tag = os.getenv("triggered_by_git_tag") + patch_id = os.getenv("version_id") + signing_enabled = scenario == BuildScenario.RELEASE + + return cls( + scenario=scenario, + git_tag=git_tag, + patch_id=patch_id, + signing_enabled=signing_enabled, + version=git_tag or patch_id, + ) + + def get_version(self) -> str: + """Gets the version that will be used to tag the images.""" + if self.scenario == BuildScenario.RELEASE: + return self.git_tag + if self.patch_id: + return self.patch_id + return "latest" + + def get_base_registry(self) -> str: + """Get the base registry URL for the current scenario.""" + if self.scenario == BuildScenario.RELEASE: + return os.environ.get("STAGING_REPO_URL") + else: + return os.environ.get("BASE_REPO_URL") + diff --git a/scripts/release/build_images.py b/scripts/release/build_images.py index 6ccaf4a8d..92632e5f3 100644 --- a/scripts/release/build_images.py +++ b/scripts/release/build_images.py @@ -1 +1,180 @@ -# Methods responsible for building and pushing docker images. +# This file is the new Sonar +import base64 +import sys +from typing import Dict + +import python_on_whales +from python_on_whales.exceptions import DockerException +import time + +import boto3 +from botocore.exceptions import BotoCoreError, ClientError + +import docker +from lib.base_logger import logger +from lib.sonar.sonar import create_ecr_repository +from scripts.evergreen.release.images_signing import sign_image, verify_signature + + +# TODO use either from python_on_whales import docker to use buildx and build multi arch image at once +# or subprocess with cmd = [ "docker", "buildx", "build", "--platform", platforms ] +def ecr_login_boto3(region: str, account_id: str): + """ + Fetches an auth token from ECR via boto3 and logs + into the Docker daemon via the Docker SDK. + """ + registry = f"{account_id}.dkr.ecr.{region}.amazonaws.com" + # 1) get token + ecr = boto3.client("ecr", region_name=region) + try: + resp = ecr.get_authorization_token(registryIds=[account_id]) + except (BotoCoreError, ClientError) as e: + raise RuntimeError(f"Failed to fetch ECR token: {e}") + + auth_data = resp["authorizationData"][0] + token = auth_data["authorizationToken"] # base64 of "AWS:password" + username, password = base64.b64decode(token).decode().split(":", 1) + + # 2) docker login + client = docker.APIClient() # low-level client supports login() + login_resp = client.login(username=username, password=password, registry=registry, reauth=True) + # login_resp is a dict like {'Status': 'Login Succeeded'} + status = login_resp.get("Status", "") + if "Succeeded" not in status: + raise RuntimeError(f"Docker login failed: {login_resp}") + logger.debug(f"ECR login succeeded: {status}") + +# TODO: don't do it every time ? +def ensure_buildx_builder(builder_name: str = "multiarch") -> str: + """ + Ensures a Docker Buildx builder exists for multi-platform builds. + + :param builder_name: Name for the buildx builder + :return: The builder name that was created or reused + """ + docker = python_on_whales.docker + + try: + docker.buildx.create( + name=builder_name, + driver="docker-container", + use=True, + bootstrap=True, + ) + logger.info(f"Created new buildx builder: {builder_name}") + except DockerException as e: + if f'existing instance for "{builder_name}"' in str(e): + logger.info(f"Builder '{builder_name}' already exists – reusing it.") + # Make sure it's the current one: + docker.buildx.use(builder_name) + else: + # Some other failure happened + logger.error(f"Failed to create buildx builder: {e}") + raise + + return builder_name + + +def build_image(tag: str, dockerfile: str, path: str, args: Dict[str, str] = {}, push: bool = True, platforms: list[str] = None): + """ + Build a Docker image using python_on_whales and Docker Buildx for multi-architecture support. + + :param tag: Image tag (name:tag) + :param dockerfile: Name or relative path of the Dockerfile within `path` + :param path: Build context path (directory with your Dockerfile) + :param args: Build arguments dictionary + :param push: Whether to push the image after building + :param platforms: List of target platforms (e.g., ["linux/amd64", "linux/arm64"]) + """ + docker = python_on_whales.docker + + try: + # Convert build args to the format expected by python_on_whales + build_args = {k: str(v) for k, v in args.items()} if args else {} + + # Set default platforms if not specified + if platforms is None: + platforms = ["linux/amd64"] + + logger.info(f"Building image: {tag}") + logger.info(f"Platforms: {platforms}") + logger.info(f"Dockerfile: {dockerfile}") + logger.info(f"Build context: {path}") + logger.debug(f"Build args: {build_args}") + + # Use buildx for multi-platform builds + if len(platforms) > 1: + logger.info(f"Multi-platform build for {len(platforms)} architectures") + + # We need a special driver to handle multi platform builds + builder_name = ensure_buildx_builder("multiarch") + + # Build the image using buildx + docker.buildx.build( + context_path=path, + file=dockerfile, + tags=[tag], + platforms=platforms, + builder=builder_name, + build_args=build_args, + push=push, + pull=False, # Don't always pull base images + ) + + logger.info(f"Successfully built {'and pushed' if push else ''} {tag}") + + except Exception as e: + logger.error(f"Failed to build image {tag}: {e}") + raise RuntimeError(f"Failed to build image {tag}: {str(e)}") + + + +def process_image( + image_name: str, + image_tag: str, + dockerfile_path: str, + dockerfile_args: Dict[str, str], + base_registry: str, + platforms: list[str] = None, + sign: bool = False, + build_path: str = ".", + push: bool = True, +): + # Login to ECR using boto3 + ecr_login_boto3(region="us-east-1", account_id="268558157000") + + # Helper to automatically create registry with correct name + should_create_repo = False + if should_create_repo: + repo_to_create = "julienben/staging-temp/" + image_name + logger.debug(f"repo_to_create: {repo_to_create}") + create_ecr_repository(repo_to_create) + logger.info(f"Created repository {repo_to_create}") + + # Set default platforms if none provided TODO: remove from here and do it at higher level later + if platforms is None: + platforms = ["linux/amd64"] + + # Build image with multi-platform support + docker_registry = f"{base_registry}/{image_name}" + image_full_uri = f"{docker_registry}:{image_tag}" + + logger.info(f"Building image: {image_full_uri}") + logger.info(f"Target platforms: {platforms}") + logger.info(f"Using Dockerfile at: {dockerfile_path}, and build path: {build_path}") + logger.debug(f"Build args: {dockerfile_args}") + + # Use new build_image function with buildx multi-platform support + build_image( + tag=image_full_uri, + dockerfile=dockerfile_path, + path=build_path, + args=dockerfile_args, + push=push, + platforms=platforms + ) + + if sign: + logger.info("Signing image") + sign_image(docker_registry, image_tag) + verify_signature(docker_registry, image_tag) diff --git a/scripts/release/main.py b/scripts/release/main.py new file mode 100644 index 000000000..55b65efb0 --- /dev/null +++ b/scripts/release/main.py @@ -0,0 +1,203 @@ +import argparse +import os +import sys +import time +from typing import Callable, Dict, Iterable, List, Optional + +from opentelemetry import context, trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter as OTLPSpanGrpcExporter, +) +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import ( + SynchronousMultiSpanProcessor, + TracerProvider, +) +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags + +from lib.base_logger import logger +from scripts.evergreen.release.images_signing import mongodb_artifactory_login +from scripts.release.atomic_pipeline import ( + build_agent_default_case, + build_agent_on_agent_bump, + build_CLI_SBOM, + build_database_image, + build_init_appdb, + build_init_database, + build_init_om_image, + build_mco_tests_image, + build_om_image, + build_operator_image, + build_operator_image_patch, + build_readiness_probe_image, + build_tests_image, + build_upgrade_hook_image, +) +from scripts.release.build_configuration import BuildConfiguration +from scripts.release.build_context import ( + BuildContext, + BuildScenario, +) + +""" +The goal of main.py, build_configuration.py and build_context.py is to provide a single source of truth for the build +configuration. All parameters that depend on the the build environment (local dev, evg, etc) should be resolved here and +not in the pipeline. +""" + + +def get_builder_function_for_image_name() -> Dict[str, Callable]: + """Returns a dictionary of image names that can be built.""" + + image_builders = { + "cli": build_CLI_SBOM, + "test": build_tests_image, + "operator": build_operator_image, + "mco-test": build_mco_tests_image, + # TODO: add support to build this per patch + "readiness-probe": build_readiness_probe_image, + "upgrade-hook": build_upgrade_hook_image, + "operator-quick": build_operator_image_patch, + "database": build_database_image, + "agent-pct": build_agent_on_agent_bump, + "agent": build_agent_default_case, + # + # Init images + "init-appdb": build_init_appdb, + "init-database": build_init_database, + "init-ops-manager": build_init_om_image, + # + # Ops Manager image + "ops-manager": build_om_image, + } + + return image_builders + + +def build_image(image_name: str, build_configuration: BuildConfiguration): + """Builds one of the supported images by its name.""" + get_builder_function_for_image_name()[image_name](build_configuration) + + +def _setup_tracing(): + trace_id = os.environ.get("otel_trace_id") + parent_id = os.environ.get("otel_parent_id") + endpoint = os.environ.get("otel_collector_endpoint") + if any(value is None for value in [trace_id, parent_id, endpoint]): + logger.info("tracing environment variables are missing, not configuring tracing") + return + logger.info(f"parent_id is {parent_id}") + logger.info(f"trace_id is {trace_id}") + logger.info(f"endpoint is {endpoint}") + span_context = SpanContext( + trace_id=int(trace_id, 16), + span_id=int(parent_id, 16), + is_remote=False, + # Magic number needed for our OTEL collector + trace_flags=TraceFlags(0x01), + ) + ctx = trace.set_span_in_context(NonRecordingSpan(span_context)) + context.attach(ctx) + sp = SynchronousMultiSpanProcessor() + span_processor = BatchSpanProcessor( + OTLPSpanGrpcExporter( + endpoint=endpoint, + ) + ) + sp.add_span_processor(span_processor) + resource = Resource(attributes={SERVICE_NAME: "evergreen-agent"}) + provider = TracerProvider(resource=resource, active_span_processor=sp) + trace.set_tracer_provider(provider) + + +def main(): + + _setup_tracing() + parser = argparse.ArgumentParser(description="Build container images.") + parser.add_argument("image", help="Image to build.") # Required + parser.add_argument("--parallel", action="store_true", help="Build images in parallel.") + parser.add_argument("--debug", action="store_true", help="Enable debug logging.") + parser.add_argument("--sign", action="store_true", help="Sign images.") + parser.add_argument( + "--scenario", + choices=list(BuildScenario), + help=f"Override the build scenario instead of inferring from environment. Options: release, patch, master, development", + ) + # Override arguments for build context and configuration + parser.add_argument( + "--platform", + default="linux/amd64", + help="Target platforms for multi-arch builds (comma-separated). Example: linux/amd64,linux/arm64. Defaults to linux/amd64.", + ) + parser.add_argument( + "--version", + help="Override the version/tag instead of resolving from build scenario", + ) + parser.add_argument( + "--registry", + help="Override the base registry instead of resolving from build scenario", + ) + + # Agent specific arguments + parser.add_argument( + "--all-agents", + action="store_true", + help="Build all agent variants instead of only the latest.", + ) + parser.add_argument( + "--parallel-factor", + default=0, + type=int, + help="Number of builds to run in parallel, defaults to number of cores", + ) + + args = parser.parse_args() + + build_config = build_config_from_args(args) + logger.info(f"Building image: {args.image}") + logger.info(f"Build configuration: {build_config}") + + build_image(args.image, build_config) + + +def build_config_from_args(args): + # Validate that the image name is supported + supported_images = get_builder_function_for_image_name().keys() + if args.image not in supported_images: + logger.error(f"Unsupported image '{args.image}'. Supported images: {', '.join(supported_images)}") + sys.exit(1) + + # Parse platform argument (comma-separated) + platforms = [p.strip() for p in args.platform.split(",")] + SUPPORTED_PLATFORMS = ["linux/amd64", "linux/arm64"] + if any(p not in SUPPORTED_PLATFORMS for p in platforms): + logger.error(f"Unsupported platform in '{args.platform}'. Supported platforms: {', '.join(SUPPORTED_PLATFORMS)}") + sys.exit(1) + + # Centralized configuration management with overrides + build_scenario = args.scenario or infer_scenario_from_environment() + build_context = BuildContext.from_scenario(build_scenario) + + # Resolve final values with overrides + scenario = args.scenario or build_context.scenario + version = args.version or build_context.get_version() + registry = args.registry or build_context.get_base_registry() + sign = args.sign or build_context.signing_enabled + all_agents = args.all_agents or bool(os.environ.get("all_agents", False)) + + return BuildConfiguration( + scenario=scenario, + version=version, + base_registry=registry, + parallel=args.parallel, + debug=args.debug, # TODO: is debug used ? + platforms=platforms, + sign=sign, + all_agents=all_agents, + parallel_factor=args.parallel_factor, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/optimized_operator_build.py b/scripts/release/optimized_operator_build.py new file mode 100644 index 000000000..c59e3c003 --- /dev/null +++ b/scripts/release/optimized_operator_build.py @@ -0,0 +1,87 @@ +import os +import subprocess +import tarfile +from datetime import datetime, timedelta, timezone + +import docker +from lib.base_logger import logger +from scripts.release.build_configuration import BuildConfiguration + + +def copy_into_container(client, src, dst): + """Copies a local file into a running container.""" + + os.chdir(os.path.dirname(src)) + srcname = os.path.basename(src) + with tarfile.open(src + ".tar", mode="w") as tar: + tar.add(srcname) + + name, dst = dst.split(":") + container = client.containers.get(name) + + with open(src + ".tar", "rb") as fd: + container.put_archive(os.path.dirname(dst), fd.read()) + + +def build_operator_image_fast(build_configuration: BuildConfiguration) -> bool: + """This function builds the operator locally and pushed into an existing + Docker image. This is the fastest way I could image we can do this.""" + + client = docker.from_env() + # image that we know is where we build operator. + image_repo = build_configuration.base_registry + "/" + build_configuration.image_type + "/mongodb-kubernetes" + image_tag = "latest" + repo_tag = image_repo + ":" + image_tag + + logger.debug(f"Pulling image: {repo_tag}") + try: + image = client.images.get(repo_tag) + except docker.errors.ImageNotFound: + logger.debug("Operator image does not exist locally. Building it now") + return False + + logger.debug("Done") + too_old = datetime.now() - timedelta(hours=3) + image_timestamp = datetime.fromtimestamp( + image.history()[0]["Created"] + ) # Layer 0 is the latest added layer to this Docker image. [-1] is the FROM layer. + + if image_timestamp < too_old: + logger.info("Current operator image is too old, will rebuild it completely first") + return False + + container_name = "mongodb-enterprise-operator" + operator_binary_location = "/usr/local/bin/mongodb-kubernetes-operator" + try: + client.containers.get(container_name).remove() + logger.debug(f"Removed {container_name}") + except docker.errors.NotFound: + pass + + container = client.containers.run(repo_tag, name=container_name, entrypoint="sh", detach=True) + + logger.debug("Building operator with debugging symbols") + subprocess.run(["make", "manager"], check=True, stdout=subprocess.PIPE) + logger.debug("Done building the operator") + + copy_into_container( + client, + os.getcwd() + "/docker/mongodb-kubernetes-operator/content/mongodb-kubernetes-operator", + container_name + ":" + operator_binary_location, + ) + + # Commit changes on disk as a tag + container.commit( + repository=image_repo, + tag=image_tag, + ) + # Stop this container so we can use it next time + container.stop() + container.remove() + + logger.info("Pushing operator to {}:{}".format(image_repo, image_tag)) + client.images.push( + repository=image_repo, + tag=image_tag, + ) + return True