Linstor migration

.github/workflows/go_lint.yaml

@@ -18,16 +18,21 @@ jobs:
       - name: Setup Go environment
         uses: actions/setup-go@v5
         with:
-          go-version: '1.22'
+          go-version: "1.22"
 
       - name: Install golangci-lint
         run: go install github.com/golangci/golangci-lint/cmd/[email protected]
 
       - name: Run Go lint
-        run: |
+        run: | # TODO remove < if [ "$dir" == "images/linstor-csi/src" ]; then > as soon as migration is done
           basedir=$(pwd)
           failed='false'
           for dir in $(find images -type d); do
+            if [ "$dir" == "images/linstor-csi/src" ]; then
+              echo "Skipping linter in $dir"
+              continue
+            fi
+
             if ls $dir/go.mod &> /dev/null; then
               echo "Running linter in $dir"
               cd $dir
@@ -41,4 +46,4 @@ jobs:
           done
           if [ $failed == 'true' ]; then
             exit 1
-          fi
+          fi

.github/workflows/go_tests.yaml

@@ -21,11 +21,16 @@ jobs:
           go-version: '1.22'
 
       - name: Run Go tests
+        # TODO remove < if [ "$dir" == "images/linstor-csi/src" ]; then > as soon as migration is done
         run: |
           basedir=$(pwd)
           failed='false'
           for dir in $(find images -type d); do
             if ls $dir/*_test.go &> /dev/null; then
+              if [ "$dir" == "images/linstor-csi/src" ]; then
+                echo "Skipping tests in $dir"
+                continue
+              fi
               echo "Running tests in $dir"
               cd $dir
               go test -v

.golangci.yaml

@@ -16,8 +16,14 @@ linters-settings:
       - prefix(sds-replicated-volume-controller)
       - prefix(webhooks)
   errcheck:
-    ignore: fmt:.*,[rR]ead|[wW]rite|[cC]lose,io:Copy
-
+    exclude-functions:
+      - fmt.Println
+      - fmt.Printf
+      - fmt.Errorf
+      - io.Copy
+      - io.Read
+      - io.Write
+      - io.Close
 linters:
   disable-all: true
   enable:

.werf/consts.yaml

@@ -0,0 +1,40 @@
+# base images
+{{- $_ := set $ "BASE_ALPINE_DEV"  "registry.deckhouse.io/base_images/dev-alpine:3.16.3@sha256:c706fa83cc129079e430480369a3f062b8178cac9ec89266ebab753a574aca8e" }}
+{{- $_ := set $ "BASE_ALT_P11"     "registry.deckhouse.io/base_images/alt:p11@sha256:c396cd7348a48f9236413e2ef5569223c15e554c0a3ca37f9d92fb787d4f1893" }}
+{{- $_ := set $ "BASE_GOLANG_1_22" "registry.deckhouse.io/base_images/golang:1.22.7-bullseye@sha256:e5dc67bf84590c008338a0e30f56a6ed2092a38e0d2895c797dd501db73a2330" }}
+{{- $_ := set $ "BASE_PYTHON"      "registry.deckhouse.io/base_images/python:3.7.16-alpine3.16@sha256:054c898ee5eacb0b3d85bdb603d6229b93619964cc01be5274acdf3e451e5ef8" }}
+{{- $_ := set $ "BASE_SCRATCH"     "registry.deckhouse.io/base_images/scratch@sha256:653ae76965c98c8cd1c8c9ff7725316d2983986f896655b30e0f44d2f8b2dd7e" }}
+{{- $_ := set $ "BASE_UBUNTU"      "registry.deckhouse.io/base_images/ubuntu:jammy-20240808@sha256:e20b137325a45b9fe9f87ed718799a0728edabe05e88585f371e6864994cf0bc" }}
+
+
+# component versions
+{{- $versions := dict }}
+{{- $_ := set $versions "DRBD" "9.2.10" }}
+{{- $_ := set $versions "DRBD_REACTOR" "1.4.0" }}
+{{- $_ := set $versions "DRBD_UTILS" "9.28.0" }}
+{{- $_ := set $versions "LINSTOR_AFFINITY_CONTROLLER" "0.3.0" }}
+{{- $_ := set $versions "LINSTOR_API_PY" "1.19.0" }}
+{{- $_ := set $versions "LINSTOR_CLIENT" "1.19.0" }}
+{{- $_ := set $versions "LINSTOR_COMMON" "999c3c0793376f88b891eb7ef3dc8f8bc9aa26ec" }}
+{{- $_ := set $versions "LINSTOR_CSI" "98544cadb6d111d27a86a11ec07de91b99704b82" }}
+{{- $_ := set $versions "LINSTOR_K8S_AWAIT_ELECTION" "0.3.1" }}
+{{- $_ := set $versions "LINSTOR_SCHEDULER_EXTENDER" "0.3.2" }}
+# the closest version I found for the stork's module version v1.4.1-0.20220512171133-b99428ee1ddf which used in linstor-scheduler-extender v0.3.2
+# https://github.com/libopenstorage/stork/pull/1097/commits
+{{- $_ := set $versions "LINSTOR_SCHEDULER_STORK" "2.11.5" }}
+{{- $_ := set $versions "LINSTOR_SERVER" "1.24.2" }}
+# We are using Gradle version 6.9.3 because building Linstor Controller with version 7+ completes unsuccesfully with error:
+# Could not set unknown property 'release' for object of type org.gradle.api.tasks.compile.CompileOptions.
+{{- $_ := set $versions "LINSTOR_SERVER_GRADLE" "6.9.3" }}
+{{- $_ := set $versions "LINSTOR_SERVER_PIRAEUS" "20e96f83f52631dc06d011b7f96293d4026236a1" }}
+{{- $_ := set $versions "LINSTOR_WAIT_UNTIL" "0.2.1" }}
+{{- $_ := set $versions "SEMVER_TOOL" "3.4.0" }}
+{{- $_ := set $versions "SPAAS" "7bef2e7976a455550bce2533487c635f20390ccf" }}
+{{- $_ := set $versions "THIN_SEND_RECV" "1.0.1" }}
+
+{{- $_ := set $ "VERSIONS" $versions }}
+
+
+# custom constants
+{{- $_ := set $ "BUILD_PACKAGES" "build-essential rpm-build rpm-macros-intro-conflicts sudo git" }}
+

.werf/images.yaml

@@ -1,7 +1,7 @@
 {{- $ImagesBuildFiles := .Files.Glob "images/*/{Dockerfile,werf.inc.yaml}" }}
 
 {{- range $path, $content := $ImagesBuildFiles  }}
-  {{ $ctx := (dict "ImageName" ($path | split "/")._1) }}
+  {{ $ctx := (dict "ImageName" ($path | split "/")._1 "Root" $ "Versions" $.VERSIONS) }}
 ---
   {{- /* For Dockerfile just render it from the folder. */ -}}
   {{- if not (regexMatch "/werf.inc.yaml$" $path) }}

.werf/python-deps.yaml

@@ -2,12 +2,15 @@
 ---
 image: python-dependencies
 from: registry.deckhouse.io/base_images/alpine:3.16.3
-fromCacheVersion: 20240901190343
+
 git:
   - add: /
     to: /
     includePaths:
       - lib/python
+    stageDependencies:
+      setup:
+        - lib/python/requirements.txt
 shell:
   beforeInstall:
     - apk add --no-cache python3 py3-pip

.werf/release.yaml

@@ -1,7 +1,8 @@
 # Release image, stored in your.registry.io/modules/<module-name>/release:<semver>
 ---
-artifact: release-channel-version-artifact
+image: release-channel-version-artifact
 from: registry.deckhouse.io/base_images/alpine:3.16.3
+final: false
 shell:
   beforeInstall:
     - apk add --no-cache curl
@@ -14,7 +15,7 @@ shell:
 image: release-channel-version
 from: registry.deckhouse.io/base_images/scratch@sha256:b054705fcc9f2205777d80a558d920c0b4209efdc3163c22b5bfcb5dda1db5fc
 import:
-  - artifact: release-channel-version-artifact
+  - image: release-channel-version-artifact
     add: /
     to: /
     after: install

.werf/utils.yaml

@@ -0,0 +1,208 @@
+{{- /*
+  utils:checkout-git clone a specified Git repo and its submodules
+  Parameters (* for required):
+    * .folder     - Folder path to which clone the repo
+    * .repo       - Git repo URL
+    * .branch     - Name of branch/commit/tag to clone
+      .submodules - String with submodules and their's branches like: "submodule1-name=submodule1-branch submodule2-name="
+                    If branch is not set then branch will use from .git-submodules file
+  Example:
+  {{- $ctx := dict }}
+  {{- $_ := set $ctx "repo" "https://github.com/LINBIT/linstor-server" }}
+  {{- $_ := set $ctx "branch" (printf "v%s" $linstor_version) }}
+  {{- $_ := set $ctx "folder" "/linstor-server" }}
+  {{- $_ := set $ctx "submodules" (printf "linstor-common=%s" $common_submodule_version) }}
+  {{- include "utils:checkout-git" $ctx | nindent 2 }}
+*/}}
+
+{{ define "utils:checkout-git" }}
+  - 'echo ">>> utils:checkout-git <<< folder={{ .folder }} repo={{ .repo }} branch={{ .branch }}"'
+  # Clone just the repository's .git folder
+  - git clone --no-checkout --depth 1 --branch {{ .branch }} {{ .repo }} /git-repo.tmp
+  # Move the .git folder to the directory with the files.
+  - mv /git-repo.tmp/.git {{ .folder }}/
+  # Delete the temporary directory
+  - rmdir /git-repo.tmp
+  - cd {{ .folder }}
+    # git thinks all files are deleted, this reverts the state of the repo to HEAD.
+    # WARNING: any local changes to the files will be lost.
+  - git reset --hard HEAD
+  - git config --global url."https://github.com/".insteadOf git://github.com/
+  - git config --global user.email "[email protected]"
+  - git config --global user.name "deckhouse"
+  - git config --global advice.detachedHead false
+  # cleanup all submodules directories, checked out by werf
+  # script from https://stackoverflow.com/a/53899440
+  - |
+    SUBMODULES_BRANCHES=()
+    # .submodules is a string like "submodule1-name=submodule1-branch submodule2-name="
+    # If branch is not set then using branch from .git-submodules file
+    {{- range $submod := $.submodules | default "" | splitList " " }}
+    SUBMODULES_BRANCHES+=( {{ $submod }} )
+    {{- end }}
+
+    echo "Processing submodules: ${SUBMODULES_BRANCHES[@]}"
+    git -C "{{ .folder }}" config -f .gitmodules --get-regexp '^submodule\..*\.path$' |
+        while read -r KEY MODULE_PATH
+        do
+            # If the module's path exists, remove it.
+            # This is done b/c the module's path is currently
+            # not a valid git repo and adding the submodule will cause an error.
+            [ -d "${MODULE_PATH}" ] && rm -rf "${MODULE_PATH}"
+
+            NAME="$(echo "${KEY}" | sed 's/^submodule\.\(.*\)\.path$/\1/')"
+
+            url_key="$(echo "${KEY}" | sed 's/\.path$/.url/')"
+            branch_key="$(echo "${KEY}" | sed 's/\.path$/.branch/')"
+
+            URL="$(git config -f .gitmodules --get "${url_key}")"
+            BRANCH="$(git config -f .gitmodules --get "${branch_key}" || echo "master")"
+
+            git -C "{{ .folder }}" submodule add --force -b "${BRANCH}" --name "${NAME}" -- "${URL}" "${MODULE_PATH}"
+
+            for submod in "${SUBMODULES_BRANCHES[@]}"; do
+              submod_name=${submod%=*}
+              submod_branch=${submod#*=}
+
+              if [ -n "${submod_branch}" -a "${NAME}"=="${submod_name}" ]; then
+                echo "Checkout branch/commit '${submod_branch}' for submodule '${NAME}'"
+                git -C "{{ .folder }}" submodule set-branch --branch ${submod_branch} -- ${MODULE_PATH}
+                git -C "{{ .folder }}/${MODULE_PATH}" checkout -f ${submod_branch}
+              fi
+            done
+        done
+    # DO NOT do submodule update, or it revert all changes for submodule's branches back :-(
+    # git -C "{{ .folder }}" submodule update --init --recursive
+
+    if ! tmp=$(git diff --name-status HEAD 2>&1) || test -n "$tmp" ; then
+      echo "Auto-commit changes in .gitmodules"
+      git -C "{{ .folder }}" commit -am "Auto-commit changes in .gitmodules"
+    fi
+{{ end }}
+
+{{- /*
+  utils:build-deb-pkg builds a Debian/Ubuntu .deb package from current folder
+  Requirements: build-essential, debhelper and other build tools should be already installed in build container
+  Parameters (* for required):
+    * .name       - Name of deb package
+    * .version    - Desired version of deb package
+      .gen_path   - Path to files generated with 'make debrelease'. If not specified, then use current directory
+
+  Example:
+  {{- $ctx := dict }}
+  {{- $_ := set $ctx "name" "linstor-server" }}
+  {{- $_ := set $ctx "version" $linstor_version }}
+  {{- include "utils:build-deb-pkg" $ctx | nindent 2 }}
+*/}}
+{{ define "utils:build-deb-pkg" }}
+  - 'echo ">>> utils:build-deb-pkg <<< name={{ .name }} version={{ .version }} pwd=$(pwd)"'
+  - make debrelease VERSION={{ .version }}
+  - mv {{ printf "%s/" (.gen_path | default ".") }}{{ .name }}-{{ .version }}.tar.gz ../{{ .name }}_$(echo {{ .version }} | tr '-' '~').orig.tar.gz
+  - tar -C / -xvf ../{{ .name }}_$(echo {{ .version }} | tr '-' '~').orig.tar.gz
+  - cd /{{ .name }}-{{ .version }}
+  - dpkg-buildpackage -us -uc
+{{ end }}
+
+{{- /*
+  utils:build-rpm-pkg builds a ALT Linux .rpm package from current folder
+  Requirements: build-essential, sudo, hasher and other build tools should be already installed in build container
+  Note: You must execute 'utils:prepare-rpm-build' before try to build RPM with this one macros!
+  Parameters (* for required):
+    * .name          - Name of RPM package
+    * .version       - Desired version of RPM package
+    * .spec          - Path to RPM's .spec file
+      .gen_path      - Path to files generated with 'make debrelease'. If not specified, then use current directory
+      .tarball_cmd   - Command to make tarball (Default: 'make release')
+      .rpmbuild_args - Additional arguments for rpmbuild command (like "--define \\'some_macro some_value\\'")
+      .arch          - To which CPU arch .rpm package will be builded (Default: noarch)
+      .no_debug_pkg  - if 'true' (default) then remove debug symbols and packages from results
+
+  Example:
+  {{- $ctx := dict }}
+  {{- $_ := set $ctx "name" "linstor-server" }}
+  {{- $_ := set $ctx "version" $linstor_version }}
+  {{- $_ := set $ctx "spec" "./linstor.spec" }}
+  {{- include "utils:build-rpm-pkg" $ctx | nindent 2 }}
+*/}}
+
+{{ define "utils:build-rpm-pkg" }}
+  - 'echo ">>> utils:build-rpm-pkg <<< name={{ .name }} version={{ .version }} spec_file={{ .spec }} pwd=$(pwd)"'
+  - |
+    if ! id "rpmbuilder" >/dev/null 2>&1; then
+      echo 'User "rpmbuilder" not found! You must execute "utils:prepare-rpm-build" first'
+      exit 5
+    fi
+  - |
+    TARBALL_CMD="make release"
+    if [ "{{ .tarball_cmd | default "" }}" != "" ]; then
+      TARBALL_CMD="{{ .tarball_cmd }}"
+      echo "Set TARBALL_CMD to '${TARBALL_CMD}'"
+    fi
+  - |
+    RPMBUILD_ARGS=()
+    if [ "{{ .rpmbuild_args | default "" }}" != "" ]; then
+      IFS=' ' read -a RPMBUILD_ARGS <<< "{{ .rpmbuild_args }}"
+      echo "Set RPMBUILD_ARGS to '${RPMBUILD_ARGS[@]}'"
+    fi
+  - |
+    ARCH="noarch"
+    if [ "{{ .arch | default "" }}" != "" ]; then
+      ARCH="{{ .arch }}"
+      echo "Set ARCH to '${ARCH}'"
+    fi
+  - |
+    NO_DEBUG_PKG="true"
+    if [ "{{ .no_debug_pkg | default "true" }}" == "false" ]; then
+      NO_DEBUG_PKG="false"
+      echo "Set NO_DEBUG_PKG to '${NO_DEBUG_PKG}'"
+    fi
+  - |
+    if [ "${NO_DEBUG_PKG}" == "true" ]; then
+      # add special arguments to rpmbuild
+      # https://stackoverflow.com/a/66681875
+      RPMBUILD_ARGS+=( "--define 'debug_package %{nil}'" )
+    fi
+
+  - chown -R rpmbuilder:rpmbuilder .
+  # we need tarball first
+  - sudo -u rpmbuilder ${TARBALL_CMD} KEEPNAME=1 VERSION={{ .version }}
+  - mv {{ printf "%s/" (.gen_path | default ".") }}{{ .name }}-{{ .version }}.tar.gz /home/rpmbuilder/RPM/SOURCES/
+  - mv {{ .spec }} /home/rpmbuilder/RPM/SPECS
+  - cd /home/rpmbuilder/RPM/SPECS
+  - SPEC_FILE=$(basename "{{ .spec }}")
+  - |
+    if [ "{{ .spec_patch | default "" }}" != "" ]; then
+      echo "Patching spec file with '{{ .spec_patch }}'"
+      {{ .spec_patch | default "" }}
+    fi
+
+  # TODO: instead of rpmbuild try use hasher (https://www.altlinux.org/Hasher)?
+  - RPM_CMD="rpmbuild -v -ba --clean ${RPMBUILD_ARGS[@]} $SPEC_FILE"
+  - 'echo "Building RPM: sudo -H -u rpmbuilder -- sh -c ${RPM_CMD}"'
+  - sudo -H -u rpmbuilder -- sh -c "${RPM_CMD}"
+  - mv /home/rpmbuilder/RPM/RPMS/${ARCH}/*.rpm /
+  - |
+    if [ "${NO_DEBUG_PKG}" == "true" ]; then
+      echo "Remove all debuginfo packages because NO_DEBUG_PKG is 'true'"
+      rm -f /*-debuginfo-*.rpm
+    fi
+{{ end }}
+
+{{- /*
+  utils:prepare-rpm-build prepares build environment for building RPM packages
+  Requirements: build-essential, hasher and other build tools should be already installed in build container
+  Parameters (* for required):
+    None
+
+  Example:
+  {{- $ctx := dict }}
+  {{- include "utils:prepare-rpm-build" $ctx | nindent 2 }}
+*/}}
+{{ define "utils:prepare-rpm-build" }}
+  - 'echo ">>> utils:prepare-rpm-build <<< name={{ .name }} version={{ .version }} pwd=$(pwd)"'
+  - groupadd -f -g 744 rpmbuilder && useradd -m -g rpmbuilder -u 744 rpmbuilder
+  - sed -i "s/# root ALL=(ALL:ALL) ALL/root ALL=(ALL:ALL) ALL/" /etc/sudoers
+  - echo 'Defaults env_keep += "SPEC_FILE"' >> /etc/sudoers
+  - mkdir -p /home/rpmbuilder/RPM/{BUILD,RPMS,SOURCES,SPECS,SRPMS}
+  - chown -R 744:744 /home/rpmbuilder
+{{ end }}