Skip to content

Commit e0c8260

Browse files
emilylo3wolfgang-desalvador
emilylo3
authored and
wolfgang-desalvador
committed
Add build script and documentation (Azure#76)
build.sh sets up a venv, runs package.py, and overwrites release.yml
1 parent 016b9e7 commit e0c8260

File tree

7 files changed

+143
-91
lines changed

7 files changed

+143
-91
lines changed

‎.github/workflows/release.yml‎

Lines changed: 26 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -18,28 +18,10 @@ jobs:
1818
run:
1919
sudo apt update || apt update;
2020
sudo apt-get install -y python3 python3-pip || apt-get install -y python3 python3-pip;
21-
pip3 install virtualenv;
22-
python3 -m virtualenv $GITHUB_WORKSPACE/.venv/;
23-
source $GITHUB_WORKSPACE/.venv/bin/activate && pip3 install setuptools==69.0.2 && python package.py;
21+
./build.sh
2422
- name: Get the version
2523
id: get_version
2624
run: echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/}
27-
- name: Get PBS binaries
28-
id: get-pbs-binaries
29-
run:
30-
mkdir rpms/;
31-
curl -L -k -o rpms/cyclecloud_api-8.3.1-py2.py3-none-any.whl https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//cyclecloud_api-8.3.1-py2.py3-none-any.whl;
32-
curl -L -k -o rpms/hwloc-libs-1.11.9-3.el8.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//hwloc-libs-1.11.9-3.el8.x86_64.rpm;
33-
curl -L -k -o rpms/openpbs-client-20.0.1-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//openpbs-client-20.0.1-0.x86_64.rpm;
34-
curl -L -k -o rpms/openpbs-client-22.05.11-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//openpbs-client-22.05.11-0.x86_64.rpm;
35-
curl -L -k -o rpms/openpbs-execution-20.0.1-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//openpbs-execution-20.0.1-0.x86_64.rpm;
36-
curl -L -k -o rpms/openpbs-execution-22.05.11-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//openpbs-execution-22.05.11-0.x86_64.rpm;
37-
curl -L -k -o rpms/openpbs-server-20.0.1-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//openpbs-server-20.0.1-0.x86_64.rpm;
38-
curl -L -k -o rpms/openpbs-server-22.05.11-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//openpbs-server-22.05.11-0.x86_64.rpm;
39-
curl -L -k -o rpms/pbspro-client-18.1.4-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//pbspro-client-18.1.4-0.x86_64.rpm;
40-
curl -L -k -o rpms/pbspro-debuginfo-18.1.4-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//pbspro-debuginfo-18.1.4-0.x86_64.rpm;
41-
curl -L -k -o rpms/pbspro-execution-18.1.4-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//pbspro-execution-18.1.4-0.x86_64.rpm;
42-
curl -L -k -o rpms/pbspro-server-18.1.4-0.x86_64.rpm https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins//pbspro-server-18.1.4-0.x86_64.rpm;
4325

4426
- name: Create Release
4527
id: create_release
@@ -60,140 +42,140 @@ jobs:
6042
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
6143
with:
6244
upload_url: ${{ steps.create_release.outputs.upload_url }}
63-
asset_path: dist/cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.version }}.tar.gz
45+
asset_path: blobs/cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.version }}.tar.gz
6446
asset_name: cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.version }}.tar.gz
6547
asset_content_type: application/gzip
6648

6749

6850
- name: Upload cyclecloud_api-8.3.1-py2.py3-none-any.whl;
69-
id: upload-1
51+
id: upload-0
7052
uses: actions/upload-release-asset@v1
7153
env:
7254
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
7355
with:
7456
upload_url: ${{ steps.create_release.outputs.upload_url }}
75-
asset_path: rpms/cyclecloud_api-8.3.1-py2.py3-none-any.whl
57+
asset_path: blobs/cyclecloud_api-8.3.1-py2.py3-none-any.whl
7658
asset_name: cyclecloud_api-8.3.1-py2.py3-none-any.whl;
7759
asset_content_type: application/octet-stream
7860

7961
- name: Upload hwloc-libs-1.11.9-3.el8.x86_64.rpm;
80-
id: upload-2
62+
id: upload-1
8163
uses: actions/upload-release-asset@v1
8264
env:
8365
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
8466
with:
8567
upload_url: ${{ steps.create_release.outputs.upload_url }}
86-
asset_path: rpms/hwloc-libs-1.11.9-3.el8.x86_64.rpm
68+
asset_path: blobs/hwloc-libs-1.11.9-3.el8.x86_64.rpm
8769
asset_name: hwloc-libs-1.11.9-3.el8.x86_64.rpm;
8870
asset_content_type: application/octet-stream
8971

9072
- name: Upload openpbs-client-20.0.1-0.x86_64.rpm;
91-
id: upload-3
73+
id: upload-2
9274
uses: actions/upload-release-asset@v1
9375
env:
9476
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
9577
with:
9678
upload_url: ${{ steps.create_release.outputs.upload_url }}
97-
asset_path: rpms/openpbs-client-20.0.1-0.x86_64.rpm
79+
asset_path: blobs/openpbs-client-20.0.1-0.x86_64.rpm
9880
asset_name: openpbs-client-20.0.1-0.x86_64.rpm;
9981
asset_content_type: application/octet-stream
10082

10183
- name: Upload openpbs-client-22.05.11-0.x86_64.rpm;
102-
id: upload-4
84+
id: upload-3
10385
uses: actions/upload-release-asset@v1
10486
env:
10587
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
10688
with:
10789
upload_url: ${{ steps.create_release.outputs.upload_url }}
108-
asset_path: rpms/openpbs-client-22.05.11-0.x86_64.rpm
90+
asset_path: blobs/openpbs-client-22.05.11-0.x86_64.rpm
10991
asset_name: openpbs-client-22.05.11-0.x86_64.rpm;
11092
asset_content_type: application/octet-stream
11193

11294
- name: Upload openpbs-execution-20.0.1-0.x86_64.rpm;
113-
id: upload-5
95+
id: upload-4
11496
uses: actions/upload-release-asset@v1
11597
env:
11698
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
11799
with:
118100
upload_url: ${{ steps.create_release.outputs.upload_url }}
119-
asset_path: rpms/openpbs-execution-20.0.1-0.x86_64.rpm
101+
asset_path: blobs/openpbs-execution-20.0.1-0.x86_64.rpm
120102
asset_name: openpbs-execution-20.0.1-0.x86_64.rpm;
121103
asset_content_type: application/octet-stream
122104

123105
- name: Upload openpbs-execution-22.05.11-0.x86_64.rpm;
124-
id: upload-6
106+
id: upload-5
125107
uses: actions/upload-release-asset@v1
126108
env:
127109
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
128110
with:
129111
upload_url: ${{ steps.create_release.outputs.upload_url }}
130-
asset_path: rpms/openpbs-execution-22.05.11-0.x86_64.rpm
112+
asset_path: blobs/openpbs-execution-22.05.11-0.x86_64.rpm
131113
asset_name: openpbs-execution-22.05.11-0.x86_64.rpm;
132114
asset_content_type: application/octet-stream
133115

134116
- name: Upload openpbs-server-20.0.1-0.x86_64.rpm;
135-
id: upload-7
117+
id: upload-6
136118
uses: actions/upload-release-asset@v1
137119
env:
138120
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
139121
with:
140122
upload_url: ${{ steps.create_release.outputs.upload_url }}
141-
asset_path: rpms/openpbs-server-20.0.1-0.x86_64.rpm
123+
asset_path: blobs/openpbs-server-20.0.1-0.x86_64.rpm
142124
asset_name: openpbs-server-20.0.1-0.x86_64.rpm;
143125
asset_content_type: application/octet-stream
144126

145127
- name: Upload openpbs-server-22.05.11-0.x86_64.rpm;
146-
id: upload-8
128+
id: upload-7
147129
uses: actions/upload-release-asset@v1
148130
env:
149131
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
150132
with:
151133
upload_url: ${{ steps.create_release.outputs.upload_url }}
152-
asset_path: rpms/openpbs-server-22.05.11-0.x86_64.rpm
134+
asset_path: blobs/openpbs-server-22.05.11-0.x86_64.rpm
153135
asset_name: openpbs-server-22.05.11-0.x86_64.rpm;
154136
asset_content_type: application/octet-stream
155137

156138
- name: Upload pbspro-client-18.1.4-0.x86_64.rpm;
157-
id: upload-9
139+
id: upload-8
158140
uses: actions/upload-release-asset@v1
159141
env:
160142
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
161143
with:
162144
upload_url: ${{ steps.create_release.outputs.upload_url }}
163-
asset_path: rpms/pbspro-client-18.1.4-0.x86_64.rpm
145+
asset_path: blobs/pbspro-client-18.1.4-0.x86_64.rpm
164146
asset_name: pbspro-client-18.1.4-0.x86_64.rpm;
165147
asset_content_type: application/octet-stream
166148

167149
- name: Upload pbspro-debuginfo-18.1.4-0.x86_64.rpm;
168-
id: upload-10
150+
id: upload-9
169151
uses: actions/upload-release-asset@v1
170152
env:
171153
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
172154
with:
173155
upload_url: ${{ steps.create_release.outputs.upload_url }}
174-
asset_path: rpms/pbspro-debuginfo-18.1.4-0.x86_64.rpm
156+
asset_path: blobs/pbspro-debuginfo-18.1.4-0.x86_64.rpm
175157
asset_name: pbspro-debuginfo-18.1.4-0.x86_64.rpm;
176158
asset_content_type: application/octet-stream
177159

178160
- name: Upload pbspro-execution-18.1.4-0.x86_64.rpm;
179-
id: upload-11
161+
id: upload-10
180162
uses: actions/upload-release-asset@v1
181163
env:
182164
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
183165
with:
184166
upload_url: ${{ steps.create_release.outputs.upload_url }}
185-
asset_path: rpms/pbspro-execution-18.1.4-0.x86_64.rpm
167+
asset_path: blobs/pbspro-execution-18.1.4-0.x86_64.rpm
186168
asset_name: pbspro-execution-18.1.4-0.x86_64.rpm;
187169
asset_content_type: application/octet-stream
188170

189171
- name: Upload pbspro-server-18.1.4-0.x86_64.rpm;
190-
id: upload-12
172+
id: upload-11
191173
uses: actions/upload-release-asset@v1
192174
env:
193175
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
194176
with:
195177
upload_url: ${{ steps.create_release.outputs.upload_url }}
196-
asset_path: rpms/pbspro-server-18.1.4-0.x86_64.rpm
178+
asset_path: blobs/pbspro-server-18.1.4-0.x86_64.rpm
197179
asset_name: pbspro-server-18.1.4-0.x86_64.rpm;
198180
asset_content_type: application/octet-stream
199181

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,5 @@ build
2020
blobs/**
2121
dist/**
2222
.env
23+
libs/**
24+
venv/**

‎BUILDING.md‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Test Azure CycleCloud OpenPBS Project Changes
2+
3+
Test CycleCloud OpenPBS changes by creating new OpenPBS clusters.
4+
5+
## Prerequisites
6+
Install the [Azure CycleCloud CLI](https://learn.microsoft.com/azure/cyclecloud/how-to/install-cyclecloud-cli?view=cyclecloud-8) and confirm it is connected to your CycleCloud instance by running the following command. The expected output is `CycleCloud is configured properly`.
7+
```bash
8+
cyclecloud initialize
9+
```
10+
Install a container runtime like [Docker](https://www.docker.com/) or [Podman](https://podman.io/) on your system.
11+
12+
## 1. Upload to Your Storage Locker
13+
1. Clone the `cyclecloud-pbspro` repository and make your desired changes.
14+
2. From the root of the repository, run the following command to prepare [project blobs](https://learn.microsoft.com/azure/cyclecloud/how-to/storage-blobs?view=cyclecloud-8_) and generate `release.yml`.
15+
```bash
16+
./build.sh
17+
```
18+
3. Run the following command then copy the name of the locker you would like to upload project blobs to.
19+
```bash
20+
cyclecloud locker list
21+
```
22+
4. Run the following command to upload project blobs to your locker.
23+
```bash
24+
cyclecloud project upload "LOCKER_NAME"
25+
```
26+
Replace `LOCKER_NAME` with the name of your locker.
27+
28+
## 2. Edit Your Cluster Template and Deploy a Cluster
29+
1. Update the openpbs template to point to your changes by running the following commands.
30+
```bash
31+
cp templates/openpbs.txt templates/openpbs-test.txt
32+
sed -i -e 's/\(\[*cluster-init[^]]*\)\]/\1:RELEASE_VERSION]/' -e 's/cyclecloud\/pbspro/pbspro/g' templates/openpbs-test.txt
33+
```
34+
Replace `RELEASE_VERSION` with the cyclecloud-pbspro release version (ex: `2.0.24`)
35+
36+
2. Import the template by running the following command.
37+
```bash
38+
cyclecloud import_template -f templates/openpbs-test.txt -c openpbs OPENPBS_PREVIEW
39+
```
40+
Replace `OPENPBS_PREVIEW` with the desired name for your new cluster type.
41+
42+
3. Using the CycleCloud UI, create a new cluster and select `OPENPBS_PREVIEW` as the scheduler.

‎build.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
# create a new venv if it does not exist, or is older than 7 days
4+
if [ ! $(find . -path ./venv/created -mtime -7) ]; then
5+
rm -rf venv
6+
python3 -m venv venv
7+
source venv/bin/activate
8+
pip install setuptools
9+
touch venv/created
10+
else
11+
source venv/bin/activate
12+
fi
13+
14+
python package.py
15+
16+
if [[ -z "$GITHUB_REF" ]]; then
17+
echo "Generating release.yml..."
18+
python generate_release_yaml.py > .github/workflows/release.yml
19+
fi

‎generate_release_yaml.py‎

Lines changed: 9 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,6 @@
11
from subprocess import check_output
2-
from configparser import ConfigParser
32
import os
4-
5-
RELEASE_URL = (
6-
"https://github.com/Azure/cyclecloud-pbspro/releases/download/2023-03-29-bins/"
7-
)
8-
RELEASE_URL = RELEASE_URL.rstrip("/") + "/"
9-
3+
from util import get_blobs
104

115
BASE_TEMPLATE = """# Based on example from https://github.com/actions/upload-release-asset
126
on:
@@ -28,23 +22,16 @@
2822
run:
2923
sudo apt update || apt update;
3024
sudo apt-get install -y python3 python3-pip || apt-get install -y python3 python3-pip;
31-
pip3 install virtualenv;
32-
python3 -m virtualenv $GITHUB_WORKSPACE/.venv/;
33-
source $GITHUB_WORKSPACE/.venv/bin/activate && python package.py;
25+
./build.sh
3426
- name: Get the version
3527
id: get_version
3628
run: echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/}
37-
- name: Get PBS binaries
38-
id: get-pbs-binaries
39-
run:
40-
mkdir rpms/;
41-
%(get_pbs_binaries)s
4229
4330
- name: Create Release
4431
id: create_release
4532
uses: actions/create-release@v1
4633
env:
47-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
34+
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
4835
with:
4936
tag_name: ${{ github.ref }}
5037
release_name: Release ${{ github.ref }}
@@ -56,42 +43,33 @@
5643
id: upload-release-asset
5744
uses: actions/upload-release-asset@v1
5845
env:
59-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
46+
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
6047
with:
6148
upload_url: ${{ steps.create_release.outputs.upload_url }}
62-
asset_path: dist/cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.version }}.tar.gz
49+
asset_path: blobs/cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.version }}.tar.gz
6350
asset_name: cyclecloud-pbspro-pkg-${{ steps.get_version.outputs.version }}.tar.gz
6451
asset_content_type: application/gzip
6552
6653
%(upload_steps)s
6754
"""
6855

69-
DOWNLOAD_LINE = f"curl -L -k -o rpms/%(fname)s {RELEASE_URL}/%(fname)s;"
70-
7156
UPLOAD_TEMPLATE = """
7257
- name: Upload %(fname)s;
7358
id: upload-%(index)s
7459
uses: actions/upload-release-asset@v1
7560
env:
76-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
61+
GITHUB_TOKEN: ${{ secrets.ACTION_PAT }}
7762
with:
7863
upload_url: ${{ steps.create_release.outputs.upload_url }}
79-
asset_path: rpms/%(fname)s
64+
asset_path: blobs/%(fname)s
8065
asset_name: %(fname)s;
8166
asset_content_type: application/octet-stream"""
8267
cwd = os.path.abspath(os.path.dirname(__file__))
8368
os.chdir(cwd)
8469

85-
parser = ConfigParser()
86-
parser.read("project.ini")
87-
blobs = [x.strip() for x in parser.get("blobs", "Files").split(",")]
88-
89-
get_pbs_binaries = []
70+
blobs = get_blobs()
9071
upload_steps = []
9172
for n, fname in enumerate(blobs):
92-
if "cyclecloud" in fname and "-pkg" in fname:
93-
continue
94-
get_pbs_binaries.append(DOWNLOAD_LINE % {"fname": fname})
9573
upload_steps.append(UPLOAD_TEMPLATE % {"fname": fname, "index": n})
9674

97-
print(BASE_TEMPLATE % {"get_pbs_binaries": "\n ".join(get_pbs_binaries), "upload_steps": "\n".join(upload_steps)})
75+
print(BASE_TEMPLATE % {"upload_steps": "\n".join(upload_steps)})

0 commit comments

Comments
 (0)