diff --git a/Jenkinsfile b/Jenkinsfile index 60a51b3fcac..744867e5a87 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,7 +42,7 @@ // no debian/ support yet /* groovylint-disable-next-line CompileStatic */ -packageBuildingPipelineDAOSTest(['distros' : ['centos7', 'el8', 'el9', 'leap15'], +packageBuildingPipelineDAOSTest(['distros' : ['el8', 'el9', 'leap15'], 'publish_branch': 'daos_adio-rpm', 'make args' : 'CHROOT=true -f Makefile-rpm.mk', 'add_make_targets': 'romio-tarball', diff --git a/mpich.rpmlintrc b/mpich.rpmlintrc index a752d60d7b6..e58238ea5f9 100644 --- a/mpich.rpmlintrc +++ b/mpich.rpmlintrc @@ -8,3 +8,4 @@ addFilter('mpich.src: E: invalid-spec-name') addFilter('mpich-doc.noarch: E: devel-dependency mpich-devel') addFilter('mpich.x86_64: E: standard-dir-owned-by-package /usr/share/doc') addFilter('mpich-devel.x86_64: E: rpath-in-buildconfig /usr/lib64/mpi/gcc/mpich/lib64/pkgconfig/mpich.pc lines 12') +addFilter('E: spelling-error') diff --git a/packaging/Dockerfile.centos.7 b/packaging/Dockerfile.centos.7 index cdfb7f69784..189ea1e4c61 100644 --- a/packaging/Dockerfile.centos.7 +++ b/packaging/Dockerfile.centos.7 @@ -5,9 +5,31 @@ # # Pull base image -FROM centos:7 +FROM centos:centos7 LABEL maintainer="daos@daos.groups.io" +# Use local repo server if present +ARG REPO_FILE_URL +RUN set -e; \ + if [ -n "$REPO_FILE_URL" ]; then \ + cd /etc/yum.repos.d/ && \ + curl -k -f -o daos_ci-centos7-artifactory.repo.tmp \ + "$REPO_FILE_URL"daos_ci-centos7-artifactory.repo && \ + for file in *.repo; do \ + true > $file; \ + done; \ + mv daos_ci-centos7-artifactory.repo{.tmp,}; \ + fi; \ + yum -y install dnf; \ + yum clean all; \ + dnf --disablerepo \*epel\* -y install epel-release \ + dnf-plugins-core; \ + if [ -n "$REPO_FILE_URL" ]; then \ + dnf -y --quiet config-manager --disable epel; \ + fi; \ + dnf -y update epel-release; \ + dnf -y clean all + # use same UID as host and default value of 1000 if not specified ARG UID=1000 @@ -15,9 +37,9 @@ ARG UID=1000 #Nothing to do for CentOS # Install basic tools -RUN yum install -y epel-release -RUN yum install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ - git python-srpm-macros dnf +RUN dnf install -y epel-release +RUN dnf install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \ + git python-srpm-macros dnf && dnf -y clean all # Add build user (to keep rpmbuild happy) ENV USER build diff --git a/packaging/Dockerfile.coverity b/packaging/Dockerfile.coverity index 7eed2c35d58..f8171c92a75 100755 --- a/packaging/Dockerfile.coverity +++ b/packaging/Dockerfile.coverity @@ -1,12 +1,13 @@ # # Copyright 2018-2020, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build for a Coverity scan. # # Pull base image FROM fedora:latest -MAINTAINER daos-stack +LABEL maintainer="daos-stack "" # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index c8bc1a48790..4d8f243ddb9 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -1,11 +1,12 @@ # -# Copyright 2018-2023 Intel Corporation +# Copyright 2018-2024 Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an RPM # # Pull base image -ARG FVERSION=38 +ARG FVERSION=latest FROM fedora:$FVERSION # Needed for later use of FVERSION ARG FVERSION @@ -13,29 +14,37 @@ LABEL maintainer="daos@daos.groups.io" # Use local repo server if present ARG REPO_FILE_URL -RUN if [ -n "$REPO_FILE_URL" ]; then \ - cd /etc/yum.repos.d/ && \ - curl -f -o daos_ci-fedora-artifactory.repo.tmp \ - "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \ - rm -f *.repo && \ - mv daos_ci-fedora-artifactory.repo{.tmp,}; \ - fi +ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME +# script to install OS updates basic tools and daos dependencies +# COPY ./utils/scripts/install-fedora.sh /tmp/install.sh +# script to setup local repo if available +COPY ./packaging/scripts/repo-helper-fedora.sh /tmp/repo-helper.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh # Install basic tools -RUN dnf -y install mock make \ - rpm-build createrepo rpmlint redhat-lsb-core git \ - python-srpm-macros rpmdevtools +RUN dnf -y install mock make \ + rpm-build createrepo rpmlint git \ + python-srpm-macros rpmdevtools && \ + dnf -y clean all # use same UID as host and default value of 1000 if not specified ARG UID=1000 # Add build user (to keep rpmbuild happy) -ENV USER build -ENV PASSWD build -RUN useradd -u $UID -ms /bin/bash $USER -RUN echo "$USER:$PASSWD" | chpasswd +ENV USER=build +ENV PASSWD=build # add the user to the mock group so it can run mock -RUN usermod -a -G mock $USER +RUN if [ $UID != 0 ]; then \ + useradd -u $UID -ms /bin/bash $USER; \ + echo "$USER:$PASSWD" | chpasswd; \ + usermod -a -G mock $USER; \ + mkdir -p /var/cache/mock; \ + chown $USER:root /var/cache/mock; \ + fi ARG CB0 RUN dnf -y upgrade && \ @@ -45,15 +54,24 @@ RUN dnf -y upgrade && \ # https://github.com/rpm-software-management/rpmlint/pull/795 in it # But make sure to patch after dnf upgrade so that an upgraded rpmlint # RPM doesn't wipe out our patch -COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch . +# Ditto for the patch to zero and display ccache stats +# https://github.com/rpm-software-management/mock/pull/1299 +ARG PACKAGINGDIR=packaging +COPY ${PACKAGINGDIR}/*.patch ./ RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') && \ if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then \ - if ! patch -p1; then \ + if ! patch -p1 < $OLDPWD/rpmlint--ignore-unused-rpmlintrc.patch; then \ exit 1; \ fi; \ rm -f rpmlint/__pycache__/{cli,lint}.*.pyc; \ - fi) < rpmlint--ignore-unused-rpmlintrc.patch; \ - rm -f rpmlint--ignore-unused-rpmlintrc.patch + fi; \ + if ! grep _ccachePostBuildHook mockbuild/plugins/ccache.py; then \ + if ! patch -p3 < $OLDPWD/ccache-stats.patch; then \ + exit 1; \ + fi; \ + rm -f mockbuild/plugins/__pycache__/ccache.*.pyc; \ + fi); \ + rm -f rpmlint--ignore-unused-rpmlintrc.patch ccache-stats.patch # show the release that was built ARG CACHEBUST diff --git a/packaging/Dockerfile.ubuntu b/packaging/Dockerfile.ubuntu new file mode 100644 index 00000000000..1e3813d58a6 --- /dev/null +++ b/packaging/Dockerfile.ubuntu @@ -0,0 +1,51 @@ +# Keep Dockerfile.ubuntu the same as this file until all packaging +# jobs are fixed to have a Dockerfile.ubuntu, and then the common +# Jenkinsfile will be changed to use Dockerfile.ubuntu. +# +# Copyright 2019-2021, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP +# +# 'recipe' for Docker to build an Debian package +# +# Pull base image +ARG BASE_DISTRO=ubuntu:20.04 +FROM $BASE_DISTRO +LABEL org.opencontainers.image.authors="daos@daos.groups.io" +# Needed for later use of BASE_DISTRO +ARG BASE_DISTRO + +ARG REPO_FILE_URL +ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME +# script to setup local repo if available +COPY ./scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh + +# Install basic tools +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + autoconf bash ca-certificates curl debhelper dh-make \ + dpkg-dev dh-python doxygen gcc git git-buildpackage \ + javahelper locales make patch pbuilder pkg-config \ + python3-dev python3-distro python3-distutils rpm scons wget \ + cmake valgrind rpmdevtools + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 + +# Add build user (to keep chrootbuild happy) +ENV USER=build +RUN useradd -u $UID -ms /bin/bash $USER + +# need to run the build command as root, as it needs to chroot +RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ + fi; \ + echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ + chmod 0440 /etc/sudoers.d/build; \ + visudo -c; \ + sudo -l -U build diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index ec76bfd1086..bd5363b5982 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -1,57 +1,42 @@ +# Keep Dockerfile.ubuntu the same as this file until all packaging +# jobs are fixed to have a Dockerfile.ubuntu, and then the common +# Jenkinsfile will be changed to use Dockerfile.ubuntu. # # Copyright 2019-2021, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an Debian package # # Pull base image -FROM ubuntu:20.04 +ARG BASE_DISTRO=ubuntu:20.04 +FROM $BASE_DISTRO LABEL org.opencontainers.image.authors="daos@daos.groups.io" - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - curl gpg +# Needed for later use of BASE_DISTRO +ARG BASE_DISTRO ARG REPO_FILE_URL -RUN if [ -n "$REPO_FILE_URL" ]; then \ - cd /etc/apt/sources.list.d && \ - curl -f -o daos_ci-ubuntu20.04-artifactory.list.tmp \ - "$REPO_FILE_URL"daos_ci-ubuntu20.04-artifactory.list && \ - true > ../sources.list && \ - mv daos_ci-ubuntu20.04-artifactory.list.tmp \ - daos_ci-ubuntu20.04-artifactory.list; \ - fi; \ - cd -; \ - curl -f -O "$REPO_FILE_URL"esad_repo.key; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import esad_repo.key; \ - mkdir -p /usr/local/share/keyrings/; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/daos-stack-public.gpg; \ - rm ./temp-keyring.gpg; \ - url_prefix=https://downloads.linux.hpe.com/SDR/; \ - for url in hpPublicKey2048.pub \ - hpPublicKey2048_key1.pub \ - hpePublicKey2048_key1.pub; do \ - curl -f -O "$url_prefix$url"; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import "$(basename $url)"; \ - done; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/hpe-sdr-public.gpg; \ - rm ./temp-keyring.gpg +ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME +# script to setup local repo if available +COPY ./packaging/scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh -# Install basic tools +# Install basic tools - rpmdevtools temporary commented out. RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ dpkg-dev dh-python doxygen gcc git git-buildpackage \ javahelper locales make patch pbuilder pkg-config \ - python3-dev python3-distro python3-distutils rpm scons wget \ - cmake valgrind rpmdevtools + python3-dev python3-distro python3-distutils rpm scons sudo \ + wget cmake valgrind # rpmdevtools # use same UID as host and default value of 1000 if not specified ARG UID=1000 # Add build user (to keep chrootbuild happy) -ENV USER build +ENV USER=build RUN useradd -u $UID -ms /bin/bash $USER # need to run the build command as root, as it needs to chroot @@ -59,7 +44,7 @@ RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ fi; \ echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ - echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ chmod 0440 /etc/sudoers.d/build; \ visudo -c; \ diff --git a/packaging/Dockerfile.ubuntu.rolling b/packaging/Dockerfile.ubuntu.rolling index 02aca458445..ed5d3633cdc 100644 --- a/packaging/Dockerfile.ubuntu.rolling +++ b/packaging/Dockerfile.ubuntu.rolling @@ -1,11 +1,12 @@ # # Copyright 2019, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an Debian package # # Pull base image FROM ubuntu:rolling -Maintainer daos-stack +LABEL org.opencontainers.image.authors="daos@daos.groups.io" # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/Makefile_distro_vars.mk b/packaging/Makefile_distro_vars.mk index 6a7f88b6072..79ae211e895 100644 --- a/packaging/Makefile_distro_vars.mk +++ b/packaging/Makefile_distro_vars.mk @@ -83,6 +83,22 @@ DISTRO_VERSION ?= $(VERSION_ID) ORIG_TARGET_VER := 15.4 SED_EXPR := 1p endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.5-x86_64) +VERSION_ID := 15.5 +DISTRO_ID := sl15.5 +DISTRO_BASE := LEAP_15 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 15.5 +SED_EXPR := 1p +endif +ifeq ($(CHROOT_NAME),opensuse-leap-15.6-x86_64) +VERSION_ID := 15.6 +DISTRO_ID := sl15.6 +DISTRO_BASE := LEAP_15 +DISTRO_VERSION ?= $(VERSION_ID) +ORIG_TARGET_VER := 15.6 +SED_EXPR := 1p +endif endif ifeq ($(ID),centos) ID = el diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 6f4b16fee04..75038c1178f 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -35,7 +35,11 @@ TEST_PACKAGES ?= ${NAME} # unfortunately we cannot always name the repo the same as the project REPO_NAME ?= $(NAME) +ifneq ($(CI_PR_REPOS),) +PR_REPOS ?= $(CI_PR_REPOS) +else PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos: *\(.*\)/\1/p') +endif LEAP_15_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p') EL_7_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p') EL_8_PR_REPOS ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p') @@ -54,6 +58,7 @@ RPM_BUILD_OPTIONS := $(BUILD_DEFINES) GIT_DIFF_EXCLUDES := $(PATCH_EXCLUDE_FILES:%=':!%') endif +FVERSION ?= latest COMMON_RPM_ARGS := --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES) SPEC := $(shell if [ -f $(NAME)-$(DISTRO_BASE).spec ]; then echo $(NAME)-$(DISTRO_BASE).spec; else echo $(NAME).spec; fi) VERSION = $(eval VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p'))$(VERSION) @@ -163,7 +168,7 @@ endif $(notdir $(SOURCE) $(OTHER_SOURCES) $(REAL_SOURCE)): $(SPEC) $(CALLING_MAKEFILE) # TODO: need to clean up old ones - $(SPECTOOL) -g $(SPEC) + $(SPECTOOL) $(COMMON_RPM_ARGS) -g $(SPEC) $(DEB_TOP)/%: % | $(DEB_TOP)/ @@ -363,12 +368,14 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE) LOCAL_REPOS='$(LOCAL_REPOS)' \ ARTIFACTORY_URL="$(ARTIFACTORY_URL)" \ DISTRO_VERSION="$(DISTRO_VERSION)" \ + PACKAGE="$(NAME)" \ TARGET="$<" \ packaging/rpm_chrootbuild endif podman_chrootbuild: if ! podman build --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \ + --build-arg FVERSION=$(FVERSION) \ -t $(subst +,-,$(CHROOT_NAME))-chrootbuild \ -f packaging/Dockerfile.mockbuild .; then \ echo "Container build failed"; \ @@ -386,7 +393,9 @@ podman_chrootbuild: exit 1; \ fi; \ rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm | \ - grep -v -e debuginfo -e debugsource -e src.rpm)' + grep -v -e debuginfo -e debugsource -e src.rpm)'; then \ + exit 1; \ + fi docker_chrootbuild: if ! $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild \ @@ -419,6 +428,8 @@ packaging_check: --exclude libfabric.spec \ --exclude Makefile \ --exclude README.md \ + --exclude SECURITY.md \ + --exclude LICENSE \ --exclude _topdir \ --exclude \*.tar.\* \ --exclude \*.code-workspace \ diff --git a/packaging/debian_chrootbuild b/packaging/debian_chrootbuild index cc2cc96d8b0..5cb2943f2ed 100755 --- a/packaging/debian_chrootbuild +++ b/packaging/debian_chrootbuild @@ -2,17 +2,35 @@ set -uex +: "${REPO_FILE_URL:=}" +: "${HTTPS_PROXY:=}" + +# Currently not fully working behind a proxy if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then - echo "MIRRORSITE=${ARTIFACTORY_URL}artifactory/ubuntu-proxy" | sudo tee /root/.pbuilderrc + pbuilderrc="./pbuilder_rc.txt" + rm -f "$pbuilderrc" + if [ -n "${HTTPS_PROXY}" ]; then + echo "export http_proxy=\"${HTTPS_PROXY}\"" >> "$pbuilderrc" + else + echo "MIRRORSITE=${ARTIFACTORY_URL}/ubuntu-proxy/ubuntu" > "$pbuilderrc" + fi + #if [ -n "$REPO_FILE_URL" ]; then + # direct="${REPO_FILE_URL##*//}" + # direct="${direct%%/*}" + # echo "no_proxy=\"${direct}\"" >> "$pbuilderrc" + #fi + # shellcheck disable=SC2002 + cat "$pbuilderrc" | sudo tee /root/.pbuilderrc fi # shellcheck disable=SC2086 sudo pbuilder create \ --extrapackages "gnupg ca-certificates" \ - $DISTRO_ID_OPT + $DISTRO_ID_OPT || true # Ignore error status for now. repo_args="" repos_added=() +# currently a bit broken, pbuilder will not accept user provided CAs. for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do branch="master" build_number="lastSuccessfulBuild" @@ -32,31 +50,34 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do repo_args="$repo_args|deb [trusted=yes] ${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/$branch/$build_number/artifact/artifacts/$DISTRO/ ./" done -repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | - sed -e 's/#.*//' -e '/ubuntu-proxy/d' -e '/^$/d' -e '/^$/d' \ - -e 's/signed-by=.*\.gpg/trusted=yes/' | - sed -e ':a; N; $!ba; s/\n/|/g')" -for repo in $JOB_REPOS; do - repo_name=${repo##*://} - repo_name=${repo_name//\//_} - if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then - # don't add duplicates, first found wins - continue - fi - repos_added+=("$repo_name") - repo_args+="|deb ${repo} $VERSION_CODENAME main" -done -# NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. -# After the build is updated to use 22.04, which supports go >= 1.18, it -# should no longer be needed. -repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu $VERSION_CODENAME main" -echo "$repo_args" -if [ "$repo_args" = "|" ]; then - repo_args="" -else - #repo_args="--othermirror"${repo_args#|}\"" - repo_args="${repo_args#|}" -fi +# currently broken, builder will not accept internal certs. +# repo_args+="|$(curl -sSf "$REPO_FILE_URL"daos_ci-"$DISTRO"-artifactory.list | +# sed -e 's/#.*//' -e '/ubuntu-proxy/d' -e '/^$/d' -e '/^$/d' \ +# -e 's/signed-by=.*\.gpg/trusted=yes/' | +# sed -e ':a; N; $!ba; s/\n/|/g')" +#for repo in $JOB_REPOS; do +# repo_name=${repo##*://} +# repo_name=${repo_name//\//_} +# if [[ " ${repos_added[*]} " = *\ ${repo_name}\ * ]]; then +# # don't add duplicates, first found wins +# continue +# fi +# repos_added+=("$repo_name") +# repo_args+="|deb ${repo} $VERSION_CODENAME main" +#done + +## NB: This PPA is needed to support modern go toolchains on ubuntu 20.04. +## After the build is updated to use 22.04, which supports go >= 1.18, it +## should no longer be needed. +# currently broken - claim is public key not available. +#repo_args="$repo_args|deb [trusted=yes] https://ppa.launchpadcontent.net/longsleep/golang-backports/ubuntu $VERSION_CODENAME main" +#echo "$repo_args" +#if [ "$repo_args" = "|" ]; then +# repo_args="" +#else +# #repo_args="--othermirror"${repo_args#|}\"" +# repo_args="${repo_args#|}" +#fi cd "$DEB_TOP" # shellcheck disable=SC2086 sudo pbuilder update --override-config $DISTRO_ID_OPT ${repo_args:+--othermirror "$repo_args"} diff --git a/packaging/get_base_branch b/packaging/get_base_branch new file mode 100755 index 00000000000..75eb90baadf --- /dev/null +++ b/packaging/get_base_branch @@ -0,0 +1,22 @@ +#!/bin/bash + +# find the base branch of the current branch + +set -eux -o pipefail +IFS=' ' read -r -a add_bases <<< "${1:-}" +origin="${ORIGIN:-origin}" +mapfile -t all_bases < <(echo "master" + git branch -r | sed -ne "/^ $origin\\/release\\/[0-9]/s/^ $origin\\///p") +all_bases+=("${add_bases[@]}") +TARGET="master" +min_diff=-1 +for base in "${all_bases[@]}"; do + git rev-parse --verify "$origin/$base" &> /dev/null || continue + commits_ahead=$(git log --oneline "$origin/$base..HEAD" | wc -l) + if [ "$min_diff" -eq -1 ] || [ "$min_diff" -gt "$commits_ahead" ]; then + TARGET="$base" + min_diff=$commits_ahead + fi +done +echo "$TARGET" +exit 0 diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index d122e0e219a..8d37abc7dbf 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -2,12 +2,34 @@ set -uex -original_cfg_file="/etc/mock/$CHROOT_NAME.cfg" -cfg_file=mock.cfg +: "${HTTPS_PROXY:=}" +: "${REPO_FILE_URL:=}" +: "${ARCH:=$(arch)}" +: "${REPOSITORY_NAME:=artifactory}" -cp "$original_cfg_file" "$cfg_file" +cp /etc/mock/"$CHROOT_NAME".cfg mock.cfg -if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then +# Enable mock ccache plugin +cat <> mock.cfg +config_opts['plugin_conf']['ccache_enable'] = True +config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/" +EOF + + +# Optionally add a proxy to mock +if [ -n "$HTTPS_PROXY" ];then + yum_proxy="http://${HTTPS_PROXY##*//}" + echo "config_opts['https_proxy'] = '$yum_proxy'" >> mock.cfg +fi + +# No proxy for local mirrors +if [ -n "$REPO_FILE_URL" ]; then + direct="${REPO_FILE_URL##*//}" + direct="${direct%%/*}" + echo "config_opts['no_proxy'] = '${direct}'" >> mock.cfg +fi + +if [[ $CHROOT_NAME == *"epel-8-${ARCH}" ]]; then cat <> mock.cfg config_opts['module_setup_commands'] = [ ('enable', 'javapackages-tools:201801'), @@ -17,12 +39,12 @@ EOF fi # Use dnf on CentOS 7 -if [[ $CHROOT_NAME == *epel-7-x86_64 ]]; then +if [[ $CHROOT_NAME == *"epel-7-$ARCH" ]]; then MOCK_OPTIONS="--dnf --no-bootstrap-chroot${MOCK_OPTIONS:+ }$MOCK_OPTIONS" fi # Allow BR: foo-devel < 1.2 to work when foo-devel-1.3 is actually available -cat <> "$cfg_file" +cat <> mock.cfg config_opts['dnf.conf'] += """ [main] best=0 @@ -33,7 +55,7 @@ EOF repo_adds=() repo_dels=() -echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file" +echo -e "config_opts['yum.conf'] += \"\"\"\n" >> mock.cfg if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then repo_dels+=("--disablerepo=\*") @@ -56,8 +78,8 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then REPO_FILE_URL="file://$(readlink -e "$REPO_FILES_PR")/" fi fi - curl -sSf "${REPO_FILE_URL}daos_ci-$DISTRO"-mock-artifactory.repo >> "$cfg_file" - repo_adds+=("--enablerepo *-artifactory") + curl -sSf "$REPO_FILE_URL"daos_ci-"${CHROOT_NAME%-*}".repo >> mock.cfg + repo_adds+=("--enablerepo *-${REPOSITORY_NAME}") fi fi @@ -81,9 +103,9 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do repo_adds+=("--enablerepo $repo:${branch//[@\/]/_}:$build_number") echo -e "[$repo:${branch//[@\/]/_}:$build_number]\n\ name=$repo:${branch//[@\/]/_}:$build_number\n\ -baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\ +baseurl=${ARTIFACTS_URL:-${JENKINS_URL:-https://build.hpdd.intel.com/}job/}daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\ enabled=1\n\ -gpgcheck=False\n" >> "$cfg_file" +gpgcheck=False\n" >> mock.cfg done for repo in $JOB_REPOS; do repo_name=${repo##*://} @@ -97,29 +119,46 @@ for repo in $JOB_REPOS; do echo -e "[${repo_name//[@\/]/_}]\n\ name=${repo_name}\n\ baseurl=${repo//\//%2F}\n\ -enabled=1\n" >> "$cfg_file" +enabled=1\n" >> mock.cfg done -echo "\"\"\"" >> "$cfg_file" +echo "\"\"\"" >> mock.cfg if [ -n "$DISTRO_VERSION" ]; then releasever_opt=("--config-opts=releasever=$DISTRO_VERSION") fi -bs_dir=/scratch/mock/cache/"${CHROOT_NAME}"-bootstrap -if ls -l /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz; then - mkdir -p "/var/cache/mock/${CHROOT_NAME}-bootstrap" +bs_dir=/scratch/mock/cache/"${CHROOT_NAME}"-bootstrap-$(id -u) +if ls -l "$bs_dir"/root_cache/cache.tar.gz; then + mkdir -p "/var/cache/mock/${CHROOT_NAME}-bootstrap/" flock "$bs_dir" -c "cp -a $bs_dir/root_cache /var/cache/mock/${CHROOT_NAME}-bootstrap" fi +if ls -l "$bs_dir/ccache-$CHROOT_NAME-$PACKAGE".tar.gz; then + flock "$bs_dir" -c "tar -C / -xzf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz" +fi -# shellcheck disable=SC2086 -eval mock -r "$cfg_file" ${repo_dels[*]} ${repo_adds[*]} --disablerepo=\*-debug* \ - "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET" +rc=0 +# shellcheck disable=SC2086,SC2048 +if ! eval time mock -r mock.cfg ${repo_dels[*]} ${repo_adds[*]} --no-clean \ + --disablerepo=\*-debug* ${releasever_opt[*]} $MOCK_OPTIONS \ + $RPM_BUILD_OPTIONS "$TARGET"; then + rc=${PIPESTATUS[0]} +fi -date -if ls -l /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz && - [ -d /scratch/ ]; then - mkdir -p /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/ - if ! cmp /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz "$bs_dir"/root_cache/cache.tar.gz; then - flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/" +# Save the ccache +if [ -d /scratch/mock ]; then + mkdir -p "$bs_dir"/ + if ! flock "$bs_dir" -c "tar -czf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz /var/cache/mock/${CHROOT_NAME}/ccache"; then + echo "Failed to save ccache. Plowing onward." + echo "I am $(id)" + fi + if ls -l /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz; then + if ! cmp /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz "$bs_dir"/root_cache/cache.tar.gz; then + if ! flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/"; then + echo "Failed to save root_cache. Plowing onward." + echo "I am $(id)" + fi + fi fi fi + +exit "$rc" diff --git a/packaging/scripts/repo-helper-fedora.sh b/packaging/scripts/repo-helper-fedora.sh new file mode 100644 index 00000000000..6c8375e3805 --- /dev/null +++ b/packaging/scripts/repo-helper-fedora.sh @@ -0,0 +1,79 @@ +#!/bin/bash +set -uex + +# This script is used by Dockerfiles to optionally use +# a local repository instead of a distro provided repository. + +: "${REPO_FILE_URL:=}" +: "${DAOS_LAB_CA_FILE_URL:=}" +: "${FVERSION:=latest}" +: "${REPOSITORY_NAME:=artifactory}" +: "${archive:=}" +if [ "$FVERSION" != "latest" ]; then + if [ "$FVERSION" != "42" ]; then + if [ "$FVERSION" != "41" ]; then + archive="-archive" + fi + fi +fi + +# shellcheck disable=SC2120 +disable_repos () { + local repos_dir="$1" + shift + local save_repos + IFS=" " read -r -a save_repos <<< "${*:-} daos_ci-fedora${archive}-${REPOSITORY_NAME}" + if [ -n "$REPO_FILE_URL" ]; then + pushd "$repos_dir" + local repo + for repo in "${save_repos[@]}"; do + mv "$repo".repo{,.tmp} + done + for file in *.repo; do + true > "$file" + done + for repo in "${save_repos[@]}"; do + mv "$repo".repo{.tmp,} + done + popd + fi +} + +# Use local repo server if present +install_curl() { + : +} + +# Use local repo server if present +install_optional_ca() { + ca_storage="/etc/pki/ca-trust/source/anchors/" + if [ -n "$DAOS_LAB_CA_FILE_URL" ]; then + curl -k --noproxy '*' -sSf -o "${ca_storage}lab_ca_file.crt" \ + "$DAOS_LAB_CA_FILE_URL" + update-ca-trust + fi +} + +# Use local repo server if present +# if a local repo server is present and the distro repo server can not +# be reached, have to bootstrap in an environment to get curl installed +# to then install the pre-built repo file. + +if [ -n "$REPO_FILE_URL" ]; then + install_curl + install_optional_ca + mkdir -p /etc/yum.repos.d + pushd /etc/yum.repos.d/ + curl -k --noproxy '*' -sSf \ + -o "daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" \ + "${REPO_FILE_URL}daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" + disable_repos /etc/yum.repos.d/ + popd +fi +dnf -y install dnf-plugins-core +# This does not work in fedora/41 anymore -- needs investigation +# dnf -y config-manager --save --setopt=assumeyes=True +# dnf config-manager --save --setopt=install_weak_deps=False +dnf clean all + +disable_repos /etc/yum.repos.d/ "${save_repos[@]}" diff --git a/packaging/scripts/repo-helper-ubuntu.sh b/packaging/scripts/repo-helper-ubuntu.sh new file mode 100644 index 00000000000..20220e982c7 --- /dev/null +++ b/packaging/scripts/repo-helper-ubuntu.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -uex + +# This script is used by Dockerfiles to optionally use +# a local repository instead of a distro provided repository. +# It will also optionally allow running a /tmp/install script +# for custom packages if present. + +: "${REPO_FILE_URL:=}" +: "${HTTPS_PROXY:=}" +: "${DAOS_LAB_CA_FILE_URL:=}" +: "${REPOSITORY_NAME:=artifactory}" + +disable_repos () { + if [ -e /etc/apt/sources.list.d/ubuntu.sources ];then + mv /etc/apt/sources.list.d/ubuntu.sources \ + etc/apt/sources.list.d/ubuntu.sources.disabled + elif [ -e /etc/apt/sources.list ];then + mv /etc/apt/sources.list \ + etc/apt/sources.list.disabled + fi +} + +# Use local repo server if present +install_curl() { + + if command -v curl; then + echo "found curl!" + return + else + apt-get update + apt-get install curl ca-certificates gpg gpg-agent \ + software-properties-common + fi + + if command -v wget; then + echo "found wget!" + return + fi + # If we don't find one of these, we are basically sunk for using + # a local repository mirror. +} + +# Use local repo server if present +install_optional_ca() { + ca_storage="/usr/local/share/ca-certificates/" + if [ -n "$DAOS_LAB_CA_FILE_URL" ]; then + curl -k --noproxy '*' -sSf -o "${ca_storage}lab_ca_file.crt" \ + "$DAOS_LAB_CA_FILE_URL" + update-ca-certificates + fi +} + +echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/no-prompt +echo "APT::Install-Recommends \"false\";" > /etc/apt/apt.conf.d/no-recommends +if [ -n "$HTTPS_PROXY" ];then + apt_proxy="http://${HTTPS_PROXY##*//}" + echo "Acquire::http::Proxy \"$apt_proxy\";" > \ + /etc/apt/apt.conf.d/local_proxy + if [ -n "$REPO_FILE_URL" ]; then + direct="${REPO_FILE_URL##*//}" + direct="${direct%%/*}" + echo "Acquire::http::Proxy { $direct DIRECT; };" >> \ + /etc/apt/apt.conf.d/local_proxy + fi +fi + +# Use local repo server if present +# if a local repo server is present and the distro repo server can not +# be reached, have to bootstrap in an environment to get curl installed +# to then install the pre-built repo file. +DISTRO_VERSION="${BASE_DISTRO##*:}" +if [ -n "$REPO_FILE_URL" ]; then + install_curl + install_optional_ca + # Ubuntu local repo mirror is not working + # curl -k --noproxy '*' -sSf \ + # -o "daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" \ + # "${REPO_FILE_URL}daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" + # disable_repos + # temp hack until we can debug the Ubuntu repos. + REPO_BASE="${REPO_FILE_URL%repo-files/}" + curl -k --noproxy '*' -sSf \ + -o "rpmdevtools_8.10-10_amd64.deb" \ + "${REPO_BASE}daos-stack-deps-ubuntu-${DISTRO_VERSION}-x86_64-stable-local/pool/rpmdevtools_8.10-10_amd64.deb" + apt-get install "./rpmdevtools_8.10-10_amd64.deb" + mkdir -p /usr/local/share/keyrings/ + curl --noproxy '*' -sSf -O "${REPO_FILE_URL}esad_repo.key" + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import esad_repo.key + gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ + --output /usr/local/share/keyrings/daos-stack-public.gpg +fi + +apt-get update +apt-get upgrade +# add-apt-repository ppa:longsleep/golang-backports +apt-get update +if [ -e /tmp/install.sh ]; then + chmod +x /tmp/install.sh + /tmp/install.sh +fi +apt-get clean all