diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 92aafee..2e512e5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,28 +18,10 @@ jobs: run: sudo apt update || apt update; sudo apt-get install -y python3 python3-pip || apt-get install -y python3 python3-pip; - pip3 install virtualenv; - python3 -m virtualenv $GITHUB_WORKSPACE/.venv/; - source $GITHUB_WORKSPACE/.venv/bin/activate && python package.py; + ./build.sh - name: Get the version id: get_version run: echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/} - - name: Get PBS binaries - id: get-pbs-binaries - run: - mkdir rpms/; - curl -L -k -o rpms/cyclecloud_api-8.3.1-py2.py3-none-any.whl https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//cyclecloud_api-8.3.1-py2.py3-none-any.whl; - curl -L -k -o rpms/hwloc-libs-1.11.9-3.el8.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//hwloc-libs-1.11.9-3.el8.x86_64.rpm; - curl -L -k -o rpms/openpbs-client-20.0.1-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//openpbs-client-20.0.1-0.x86_64.rpm; - curl -L -k -o rpms/openpbs-client-22.05.11-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//openpbs-client-22.05.11-0.x86_64.rpm; - curl -L -k -o rpms/openpbs-execution-20.0.1-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//openpbs-execution-20.0.1-0.x86_64.rpm; - curl -L -k -o rpms/openpbs-execution-22.05.11-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//openpbs-execution-22.05.11-0.x86_64.rpm; - curl -L -k -o rpms/openpbs-server-20.0.1-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//openpbs-server-20.0.1-0.x86_64.rpm; - curl -L -k -o rpms/openpbs-server-22.05.11-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//openpbs-server-22.05.11-0.x86_64.rpm; - curl -L -k -o rpms/pbspro-client-18.1.4-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//pbspro-client-18.1.4-0.x86_64.rpm; - curl -L -k -o rpms/pbspro-debuginfo-18.1.4-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//pbspro-debuginfo-18.1.4-0.x86_64.rpm; - curl -L -k -o rpms/pbspro-execution-18.1.4-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//pbspro-execution-18.1.4-0.x86_64.rpm; - curl -L -k -o rpms/pbspro-server-18.1.4-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//pbspro-server-18.1.4-0.x86_64.rpm; - name: Create Release id: create_release @@ -60,140 +42,140 @@ jobs: GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: dist/cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.version }}.tar.gz + asset_path: blobs/cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.version }}.tar.gz asset_name: cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.version }}.tar.gz asset_content_type: application/gzip - name: Upload cyclecloud_api-8.3.1-py2.py3-none-any.whl; - id: upload-1 + id: upload-0 uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rpms/cyclecloud_api-8.3.1-py2.py3-none-any.whl + asset_path: blobs/cyclecloud_api-8.3.1-py2.py3-none-any.whl asset_name: cyclecloud_api-8.3.1-py2.py3-none-any.whl; asset_content_type: application/octet-stream - name: Upload hwloc-libs-1.11.9-3.el8.x86_64.rpm; - id: upload-2 + id: upload-1 uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rpms/hwloc-libs-1.11.9-3.el8.x86_64.rpm + asset_path: blobs/hwloc-libs-1.11.9-3.el8.x86_64.rpm asset_name: hwloc-libs-1.11.9-3.el8.x86_64.rpm; asset_content_type: application/octet-stream - name: Upload openpbs-client-20.0.1-0.x86_64.rpm; - id: upload-3 + id: upload-2 uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rpms/openpbs-client-20.0.1-0.x86_64.rpm + asset_path: blobs/openpbs-client-20.0.1-0.x86_64.rpm asset_name: openpbs-client-20.0.1-0.x86_64.rpm; asset_content_type: application/octet-stream - name: Upload openpbs-client-22.05.11-0.x86_64.rpm; - id: upload-4 + id: upload-3 uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rpms/openpbs-client-22.05.11-0.x86_64.rpm + asset_path: blobs/openpbs-client-22.05.11-0.x86_64.rpm asset_name: openpbs-client-22.05.11-0.x86_64.rpm; asset_content_type: application/octet-stream - name: Upload openpbs-execution-20.0.1-0.x86_64.rpm; - id: upload-5 + id: upload-4 uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rpms/openpbs-execution-20.0.1-0.x86_64.rpm + asset_path: blobs/openpbs-execution-20.0.1-0.x86_64.rpm asset_name: openpbs-execution-20.0.1-0.x86_64.rpm; asset_content_type: application/octet-stream - name: Upload openpbs-execution-22.05.11-0.x86_64.rpm; - id: upload-6 + id: upload-5 uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rpms/openpbs-execution-22.05.11-0.x86_64.rpm + asset_path: blobs/openpbs-execution-22.05.11-0.x86_64.rpm asset_name: openpbs-execution-22.05.11-0.x86_64.rpm; asset_content_type: application/octet-stream - name: Upload openpbs-server-20.0.1-0.x86_64.rpm; - id: upload-7 + id: upload-6 uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rpms/openpbs-server-20.0.1-0.x86_64.rpm + asset_path: blobs/openpbs-server-20.0.1-0.x86_64.rpm asset_name: openpbs-server-20.0.1-0.x86_64.rpm; asset_content_type: application/octet-stream - name: Upload openpbs-server-22.05.11-0.x86_64.rpm; - id: upload-8 + id: upload-7 uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rpms/openpbs-server-22.05.11-0.x86_64.rpm + asset_path: blobs/openpbs-server-22.05.11-0.x86_64.rpm asset_name: openpbs-server-22.05.11-0.x86_64.rpm; asset_content_type: application/octet-stream - name: Upload pbspro-client-18.1.4-0.x86_64.rpm; - id: upload-9 + id: upload-8 uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rpms/pbspro-client-18.1.4-0.x86_64.rpm + asset_path: blobs/pbspro-client-18.1.4-0.x86_64.rpm asset_name: pbspro-client-18.1.4-0.x86_64.rpm; asset_content_type: application/octet-stream - name: Upload pbspro-debuginfo-18.1.4-0.x86_64.rpm; - id: upload-10 + id: upload-9 uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rpms/pbspro-debuginfo-18.1.4-0.x86_64.rpm + asset_path: blobs/pbspro-debuginfo-18.1.4-0.x86_64.rpm asset_name: pbspro-debuginfo-18.1.4-0.x86_64.rpm; asset_content_type: application/octet-stream - name: Upload pbspro-execution-18.1.4-0.x86_64.rpm; - id: upload-11 + id: upload-10 uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rpms/pbspro-execution-18.1.4-0.x86_64.rpm + asset_path: blobs/pbspro-execution-18.1.4-0.x86_64.rpm asset_name: pbspro-execution-18.1.4-0.x86_64.rpm; asset_content_type: application/octet-stream - name: Upload pbspro-server-18.1.4-0.x86_64.rpm; - id: upload-12 + id: upload-11 uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rpms/pbspro-server-18.1.4-0.x86_64.rpm + asset_path: blobs/pbspro-server-18.1.4-0.x86_64.rpm asset_name: pbspro-server-18.1.4-0.x86_64.rpm; asset_content_type: application/octet-stream diff --git a/.gitignore b/.gitignore index ee67f93..0174ffc 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ build blobs/** dist/** .env +libs/** +venv/** diff --git a/BUILDING.md b/BUILDING.md new file mode 100644 index 0000000..bfe122f --- /dev/null +++ b/BUILDING.md @@ -0,0 +1,42 @@ +# Test Azure CycleCloud OpenPBS Project Changes + +Test CycleCloud OpenPBS changes by creating new OpenPBS clusters. + +## Prerequisites +Install the [Azure CycleCloud CLI](https://learn.microsoft.com/azure/cyclecloud/how-to/install-cyclecloud-cli?view=cyclecloud-8) and confirm it is connected to your CycleCloud instance by running the following command. The expected output is `CycleCloud is configured properly`. +```bash +cyclecloud initialize +``` +Install a container runtime like [Docker](https://www.docker.com/) or [Podman](https://podman.io/) on your system. + +## 1. Upload to Your Storage Locker +1. Clone the `cyclecloud-pbspro` repository and make your desired changes. +2. From the root of the repository, run the following command to prepare [project blobs](https://learn.microsoft.com/azure/cyclecloud/how-to/storage-blobs?view=cyclecloud-8_) and generate `release.yml`. +```bash +./build.sh +``` +3. Run the following command then copy the name of the locker you would like to upload project blobs to. +```bash +cyclecloud locker list +``` +4. Run the following command to upload project blobs to your locker. +```bash +cyclecloud project upload "LOCKER_NAME" +``` +Replace `LOCKER_NAME` with the name of your locker. + +## 2. Edit Your Cluster Template and Deploy a Cluster +1. Update the openpbs template to point to your changes by running the following commands. +```bash +cp templates/openpbs.txt templates/openpbs-test.txt +sed -i -e 's/\(\[*cluster-init[^]]*\)\]/\1:RELEASE_VERSION]/' -e 's/cyclecloud\/pbspro/pbspro/g' templates/openpbs-test.txt +``` +Replace `RELEASE_VERSION` with the cyclecloud-pbspro release version (ex: `2.0.24`) + +2. Import the template by running the following command. +```bash +cyclecloud import_template -f templates/openpbs-test.txt -c openpbs OPENPBS_PREVIEW +``` +Replace `OPENPBS_PREVIEW` with the desired name for your new cluster type. + +3. Using the CycleCloud UI, create a new cluster and select `OPENPBS_PREVIEW` as the scheduler. \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..237e089 --- /dev/null +++ b/build.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# create a new venv if it does not exist, or is older than 7 days +if [ ! $(find . -path ./venv/created -mtime -7) ]; then + rm -rf venv + python3 -m venv venv + source venv/bin/activate + pip install setuptools + touch venv/created +else + source venv/bin/activate +fi + +python package.py + +if [[ -z "$GITHUB_REF" ]]; then + echo "Generating release.yml..." + python generate_release_yaml.py > .github/workflows/release.yml +fi diff --git a/generate_release_yaml.py b/generate_release_yaml.py index 42387f4..cba208e 100644 --- a/generate_release_yaml.py +++ b/generate_release_yaml.py @@ -1,12 +1,6 @@ from subprocess import check_output -from configparser import ConfigParser import os - -RELEASE_URL = ( - "https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins/" -) -RELEASE_URL = RELEASE_URL.rstrip("/") + "/" - +from util import get_blobs BASE_TEMPLATE = """# Based on example from https://github.com/actions/upload-release-asset on: @@ -28,23 +22,16 @@ run: sudo apt update || apt update; sudo apt-get install -y python3 python3-pip || apt-get install -y python3 python3-pip; - pip3 install virtualenv; - python3 -m virtualenv $GITHUB_WORKSPACE/.venv/; - source $GITHUB_WORKSPACE/.venv/bin/activate && python package.py; + ./build.sh - name: Get the version id: get_version run: echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/} - - name: Get PBS binaries - id: get-pbs-binaries - run: - mkdir rpms/; - %(get_pbs_binaries)s - name: Create Release id: create_release uses: actions/create-release@v1 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: tag_name: ${{ github.ref }} release_name: Release ${{ github.ref }} @@ -56,42 +43,33 @@ id: upload-release-asset uses: actions/upload-release-asset@v1 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: dist/cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.version }}.tar.gz + asset_path: blobs/cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.version }}.tar.gz asset_name: cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.version }}.tar.gz asset_content_type: application/gzip %(upload_steps)s """ -DOWNLOAD_LINE = f"curl -L -k -o rpms/%(fname)s {RELEASE_URL}/%(fname)s;" - UPLOAD_TEMPLATE = """ - name: Upload %(fname)s; id: upload-%(index)s uses: actions/upload-release-asset@v1 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.ACTION_PAT }} with: upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: rpms/%(fname)s + asset_path: blobs/%(fname)s asset_name: %(fname)s; asset_content_type: application/octet-stream""" cwd = os.path.abspath(os.path.dirname(__file__)) os.chdir(cwd) -parser = ConfigParser() -parser.read("project.ini") -blobs = [x.strip() for x in parser.get("blobs", "Files").split(",")] - -get_pbs_binaries = [] +blobs = get_blobs() upload_steps = [] for n, fname in enumerate(blobs): - if "cyclecloud" in fname and "-pkg" in fname: - continue - get_pbs_binaries.append(DOWNLOAD_LINE % {"fname": fname}) upload_steps.append(UPLOAD_TEMPLATE % {"fname": fname, "index": n}) -print(BASE_TEMPLATE % {"get_pbs_binaries": "\n ".join(get_pbs_binaries), "upload_steps": "\n".join(upload_steps)}) +print(BASE_TEMPLATE % {"upload_steps": "\n".join(upload_steps)}) diff --git a/package.py b/package.py index b1d5903..b80de8d 100644 --- a/package.py +++ b/package.py @@ -9,6 +9,7 @@ from argparse import Namespace from subprocess import check_call from typing import Dict, List, Optional +from util import download_release_files SCALELIB_VERSION = "1.0.5" CYCLECLOUD_API_VERSION = "8.3.1" @@ -30,7 +31,7 @@ def build_sdist() -> str: return fname -def get_cycle_libs(args: Namespace) -> List[str]: +def get_cycle_packages(args: Namespace) -> List[str]: ret = [build_sdist()] scalelib_file = "cyclecloud-scalelib-{}.tar.gz".format(SCALELIB_VERSION) @@ -44,26 +45,26 @@ def get_cycle_libs(args: Namespace) -> List[str]: cyclecloud_api_file: (args.cyclecloud_api, cyclecloud_api_url), } - for lib_file in to_download: - arg_override, url = to_download[lib_file] + for pkg_file in to_download: + arg_override, url = to_download[pkg_file] if arg_override: if not os.path.exists(arg_override): print(arg_override, "does not exist", file=sys.stderr) sys.exit(1) fname = os.path.basename(arg_override) orig = os.path.abspath(arg_override) - dest = os.path.abspath(os.path.join("libs", fname)) + dest = os.path.abspath(os.path.join("blobs" if pkg_file == cyclecloud_api_file else "libs", fname)) if orig != dest: shutil.copyfile(orig, dest) ret.append(fname) else: - dest = os.path.join("libs", lib_file) - check_call(["curl", "-L", "-k", "-s", "-f", "-o", dest, url]) - ret.append(lib_file) - print("Downloaded", lib_file, "to") + dest = os.path.join("blobs" if pkg_file == cyclecloud_api_file else "libs", pkg_file) + check_call(["curl", "-L", "-s", "-f", "-z", dest, "-o", dest, url]) - return ret + ret.append(pkg_file) + print("Downloaded", pkg_file, "to", dest) + return ret def execute() -> None: expected_cwd = os.path.abspath(os.path.dirname(__file__)) @@ -71,6 +72,9 @@ def execute() -> None: if not os.path.exists("libs"): os.makedirs("libs") + + if not os.path.exists("blobs"): + os.makedirs("blobs") argument_parser = argparse.ArgumentParser( "Builds CycleCloud GridEngine project with all dependencies.\n" @@ -80,7 +84,7 @@ def execute() -> None: argument_parser.add_argument("--cyclecloud-api", default=None) args = argument_parser.parse_args() - cycle_libs = get_cycle_libs(args) + cycle_packages = get_cycle_packages(args) parser = configparser.ConfigParser() ini_path = os.path.abspath("project.ini") @@ -92,11 +96,8 @@ def execute() -> None: if not version: raise RuntimeError("Missing [project] -> version in {}".format(ini_path)) - if not os.path.exists("dist"): - os.makedirs("dist") - tf = tarfile.TarFile.gzopen( - "dist/cyclecloud-pbspro-pkg-{}.tar.gz".format(version), "w" + "blobs/cyclecloud-pbspro-pkg-{}.tar.gz".format(version), "w" ) build_dir = tempfile.mkdtemp("cyclecloud-pbspro") @@ -113,8 +114,8 @@ def _add(name: str, path: Optional[str] = None, mode: Optional[int] = None) -> N tf.addfile(tarinfo, fr) packages = [] - for dep in cycle_libs: - dep_path = os.path.abspath(os.path.join("libs", dep)) + for dep in cycle_packages: + dep_path = os.path.abspath(os.path.join("blobs" if "cyclecloud_api" in dep else "libs", dep)) _add("packages/" + dep, dep_path) packages.append(dep_path) @@ -157,6 +158,9 @@ def _add(name: str, path: Optional[str] = None, mode: Optional[int] = None) -> N _add("autoscale_hook.py", "pbspro/conf/autoscale_hook.py") _add("logging.conf", "pbspro/conf/logging.conf") + print("Downloading release files...") + download_release_files() + if __name__ == "__main__": execute() diff --git a/util.py b/util.py new file mode 100644 index 0000000..d5ef50c --- /dev/null +++ b/util.py @@ -0,0 +1,25 @@ +from configparser import ConfigParser +import os +from subprocess import run + +RELEASE_URL = ( + "https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins" + ) +RELEASE_URL = RELEASE_URL.rstrip("/") + "/" + +def download_release_files(): + blobs = get_blobs() + for _, fname in enumerate(blobs): + if fname == "cyclecloud_api": + continue + + url = os.path.join(RELEASE_URL, fname) + run(["curl", "-L", "-C", "-", "-s", "-O", url], cwd="blobs", check=True) + +def get_blobs(): + parser = ConfigParser() + parser.read("project.ini") + blobs = [x.strip() for x in parser.get("blobs", "Files").split(",") if "cyclecloud-pbspro-pkg" not in x] + + return blobs +