Skip to content

Commit 1a41750

Browse files
wyliwyli
authored
pip install, CI/CD enhancements (#1033)
* CI CD enhancements Signed-off-by: Wenqi Li <[email protected]> * remove temp tests Signed-off-by: Wenqi Li <[email protected]> * improves error msg for python versions Signed-off-by: Wenqi Li <[email protected]>
1 parent fa521a9 commit 1a41750

File tree

10 files changed

+163
-83
lines changed

10 files changed

+163
-83
lines changed

‎.github/workflows/cron.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ jobs:
3535
echo $CUDA_VISIBLE_DEVICES
3636
python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
3737
python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
38-
./runtests.sh --coverage
38+
BUILD_MONAI=1 ./runtests.sh --coverage
3939
coverage xml
4040
- name: Upload coverage
4141
uses: codecov/codecov-action@v1
@@ -58,7 +58,7 @@ jobs:
5858
python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
5959
python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
6060
ngc --version
61-
./runtests.sh --coverage --pytype
61+
BUILD_MONAI=1 ./runtests.sh --coverage --pytype
6262
coverage xml
6363
- name: Upload coverage
6464
uses: codecov/codecov-action@v1

‎.github/workflows/pythonapp.yml

+58-5
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,9 @@ jobs:
8484
cat "requirements-dev.txt"
8585
python -m pip install -r requirements-dev.txt
8686
python -m pip list
87-
SKIP_MONAI_BUILD=1 python setup.py develop
87+
python setup.py develop
8888
python setup.py develop --uninstall
89-
python setup.py develop # compile the cpp extensions
89+
BUILD_MONAI=1 python setup.py develop # compile the cpp extensions
9090
shell: bash
9191
- name: Run quick tests (CPU ${{ runner.os }})
9292
run: |
@@ -95,6 +95,56 @@ jobs:
9595
env:
9696
QUICKTEST: True
9797

98+
min-dep-py3: # min dependencies installed
99+
runs-on: ${{ matrix.os }}
100+
strategy:
101+
fail-fast: false
102+
matrix:
103+
os: [windows-latest, macOS-latest, ubuntu-latest]
104+
timeout-minutes: 20
105+
steps:
106+
- uses: actions/checkout@v2
107+
- name: Set up Python 3.x
108+
uses: actions/setup-python@v1
109+
with:
110+
python-version: '3.x'
111+
- name: Prepare pip wheel
112+
run: |
113+
which python
114+
python -m pip install --upgrade pip wheel
115+
- name: cache weekly timestamp
116+
id: pip-cache
117+
run: |
118+
echo "::set-output name=datew::$(date '+%Y-%V')"
119+
echo "::set-output name=dir::$(pip cache dir)"
120+
shell: bash
121+
- name: cache for pip
122+
uses: actions/cache@v2
123+
id: cache
124+
with:
125+
path: ${{ steps.pip-cache.outputs.dir }}
126+
key: ${{ matrix.os }}-latest-pip-${{ steps.pip-cache.outputs.datew }}
127+
- if: runner.os == 'windows'
128+
name: Install torch cpu from pytorch.org (Windows only)
129+
run: |
130+
python -m pip install torch==1.4 -f https://download.pytorch.org/whl/cpu/torch_stable.html
131+
- name: Install the dependencies
132+
run: |
133+
# min. requirements for windows instances
134+
python -m pip install torch==1.4
135+
python -c "f=open('requirements-dev.txt', 'r'); txt=f.readlines(); f.close(); print(txt); f=open('requirements-dev.txt', 'w'); f.writelines(txt[1:5]); f.close()"
136+
cat "requirements-dev.txt"
137+
python -m pip install -r requirements-dev.txt
138+
python -m pip list
139+
BUILD_MONAI=0 python setup.py develop # no compile of extensions
140+
shell: bash
141+
- name: Run quick tests (CPU ${{ runner.os }})
142+
run: |
143+
python -c 'import torch; print(torch.__version__); print(torch.rand(5,3))'
144+
python -m tests.min_tests
145+
env:
146+
QUICKTEST: True
147+
98148
GPU-quick-py3: # GPU with full dependencies
99149
strategy:
100150
matrix:
@@ -170,11 +220,11 @@ jobs:
170220
run: |
171221
python -m pip list
172222
nvidia-smi
173-
export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
223+
export CUDA_VISIBLE_DEVICES=$(coverage run -m tests.utils)
174224
echo $CUDA_VISIBLE_DEVICES
175225
python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
176226
python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
177-
./runtests.sh --quick
227+
BUILD_MONAI=1 ./runtests.sh --quick
178228
if [ ${{ matrix.environment }} == "PT16+CUDA110" ]; then
179229
# test the clang-format tool downloading once
180230
coverage run -m tests.clang_format_utils
@@ -274,4 +324,7 @@ jobs:
274324
- name: Make html
275325
run: |
276326
cd docs/
277-
make html
327+
make clean
328+
make html 2>&1 | tee tmp_log
329+
if [[ $(grep -c "^WARNING:" tmp_log) != 0 ]]; then echo "found warnings"; exit 1; fi
330+
shell: bash

‎.github/workflows/setupapp.yml

+13-58
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ jobs:
4343
echo $CUDA_VISIBLE_DEVICES
4444
python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
4545
python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
46-
./runtests.sh --coverage
46+
BUILD_MONAI=1 ./runtests.sh --coverage
4747
coverage xml
4848
- name: Upload coverage
4949
uses: codecov/codecov-action@v1
@@ -83,65 +83,15 @@ jobs:
8383
run: |
8484
python -m pip list
8585
python -c 'import torch; print(torch.__version__); print(torch.rand(5,3))'
86-
./runtests.sh --quick
86+
BUILD_MONAI=1 ./runtests.sh --quick
8787
coverage xml
8888
- name: Upload coverage
8989
uses: codecov/codecov-action@v1
9090
with:
9191
fail_ci_if_error: false
9292
file: ./coverage.xml
9393

94-
min-dep-py3: # min dependencies installed
95-
runs-on: ${{ matrix.os }}
96-
strategy:
97-
fail-fast: false
98-
matrix:
99-
os: [windows-latest, macOS-latest, ubuntu-latest]
100-
timeout-minutes: 20
101-
steps:
102-
- uses: actions/checkout@v2
103-
- name: Set up Python 3.x
104-
uses: actions/setup-python@v1
105-
with:
106-
python-version: '3.x'
107-
- name: Prepare pip wheel
108-
run: |
109-
which python
110-
python -m pip install --upgrade pip wheel
111-
- name: cache weekly timestamp
112-
id: pip-cache
113-
run: |
114-
echo "::set-output name=datew::$(date '+%Y-%V')"
115-
echo "::set-output name=dir::$(pip cache dir)"
116-
shell: bash
117-
- name: cache for pip
118-
uses: actions/cache@v2
119-
id: cache
120-
with:
121-
path: ${{ steps.pip-cache.outputs.dir }}
122-
key: ${{ matrix.os }}-latest-pip-${{ steps.pip-cache.outputs.datew }}
123-
- if: runner.os == 'windows'
124-
name: Install torch cpu from pytorch.org (Windows only)
125-
run: |
126-
python -m pip install torch==1.4 -f https://download.pytorch.org/whl/cpu/torch_stable.html
127-
- name: Install the dependencies
128-
run: |
129-
# min. requirements for windows instances
130-
python -m pip install torch==1.4
131-
python -c "f=open('requirements-dev.txt', 'r'); txt=f.readlines(); f.close(); print(txt); f=open('requirements-dev.txt', 'w'); f.writelines(txt[1:5]); f.close()"
132-
cat "requirements-dev.txt"
133-
python -m pip install -r requirements-dev.txt
134-
python -m pip list
135-
SKIP_MONAI_BUILD=1 python setup.py develop # no compile of extensions
136-
shell: bash
137-
- name: Run quick tests (CPU ${{ runner.os }})
138-
run: |
139-
python -c 'import torch; print(torch.__version__); print(torch.rand(5,3))'
140-
python -m tests.min_tests
141-
env:
142-
QUICKTEST: True
143-
144-
install:
94+
install: # pip install from github url
14595
runs-on: ubuntu-latest
14696
steps:
14797
- name: Set up Python 3.7
@@ -158,14 +108,19 @@ jobs:
158108
with:
159109
path: ~/.cache/pip
160110
key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }}
161-
- name: Install the default branch
162-
run: |
163-
pip install git+https://github.com/Project-MONAI/MONAI#egg=MONAI
164-
- name: Import
111+
- name: Install the default branch no build
165112
run: |
113+
BUILD_MONAI=0 pip install git+https://github.com/Project-MONAI/MONAI#egg=MONAI
166114
python -c 'import monai; monai.config.print_config()'
167-
- name: Uninstall
115+
cd $(python -c 'import monai; import os; print(os.path.dirname(monai.__file__))')
116+
ls .
117+
pip uninstall -y monai
118+
- name: Install the default branch with build
168119
run: |
120+
BUILD_MONAI=1 pip install git+https://github.com/Project-MONAI/MONAI#egg=MONAI
121+
python -c 'import monai; monai.config.print_config()'
122+
cd $(python -c 'import monai; import os; print(os.path.dirname(monai.__file__))')
123+
ls .
169124
pip uninstall -y monai
170125
171126
docker:

‎MANIFEST.in

-3
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,3 @@ include monai/_version.py
33

44
include README.md
55
include LICENSE
6-
7-
recursive-exclude * __pycache__
8-
recursive-exclude * *.py[co]

‎docs/source/installation.md

+20-8
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@ for the latest features:
4040
```bash
4141
pip install git+https://github.com/Project-MONAI/MONAI#egg=MONAI
4242
```
43+
or, to build with MONAI Cpp/CUDA extensions:
44+
```bash
45+
BUILD_MONAI=1 pip install git+https://github.com/Project-MONAI/MONAI#egg=MONAI
46+
```
4347
this command will download and install the current master branch of [MONAI from
4448
GitHub](https://github.com/Project-MONAI/MONAI).
4549

@@ -55,17 +59,25 @@ You can install it by running:
5559
```bash
5660
cd MONAI/
5761
python setup.py develop
62+
```
63+
or, to build with MONAI Cpp/CUDA extensions:
64+
```bash
65+
cd MONAI/
66+
BUILD_MONAI=1 python setup.py develop
67+
# for MacOS
68+
BUILD_MONAI=1 CC=clang CXX=clang++ python setup.py develop
69+
```
5870

59-
# For MacOS:
60-
# CC=clang CXX=clang++ python setup.py develop
61-
62-
# To install without build
63-
# SKIP_MONAI_BUILD=1 python setup.py develop
64-
65-
# To uninstall the package please run:
71+
To uninstall the package please run:
72+
```bash
73+
cd MONAI/
6674
python setup.py develop --uninstall
75+
76+
# to further clean up the MONAI/ folder (Bash script)
77+
./runtests.sh --clean
6778
```
68-
or simply adding the root directory of the cloned source code (e.g., ``/workspace/Documents/MONAI``) to your ``$PYTHONPATH``
79+
80+
Alternatively, simply adding the root directory of the cloned source code (e.g., ``/workspace/Documents/MONAI``) to your ``$PYTHONPATH``
6981
and the codebase is ready to use (without the additional features of MONAI C++/CUDA extensions).
7082

7183

‎monai/__init__.py

+12-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@
1313
import sys
1414

1515
from ._version import get_versions
16-
from .utils.module import load_submodules
16+
17+
PY_REQUIRED_MAJOR = 3
18+
PY_REQUIRED_MINOR = 6
1719

1820
__version__ = get_versions()["version"]
1921
del get_versions
@@ -22,6 +24,15 @@
2224

2325
__basedir__ = os.path.dirname(__file__)
2426

27+
if not (sys.version_info.major == PY_REQUIRED_MAJOR and sys.version_info.minor >= PY_REQUIRED_MINOR):
28+
raise RuntimeError(
29+
"MONAI requires Python {}.{} or higher. But the current Python is: {}".format(
30+
PY_REQUIRED_MAJOR, PY_REQUIRED_MINOR, sys.version
31+
),
32+
)
33+
34+
from .utils.module import load_submodules # noqa: E402
35+
2536
# handlers_* have some external decorators the users may not have installed
2637
# *.so files and folder "_C" may not exist when the cpp extensions are not compiled
2738
excludes = "(^(monai.handlers))|((\\.so)$)|(^(monai._C))"

‎runtests.sh

+12-1
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,11 @@ function print_usage {
9595
exit 1
9696
}
9797

98+
function check_import {
99+
echo "PYTHON: $(which python)"
100+
${cmdPrefix}python -c "import monai"
101+
}
102+
98103
function print_version {
99104
${cmdPrefix}python -c 'import monai; monai.config.print_config()'
100105
}
@@ -106,6 +111,8 @@ function install_deps {
106111

107112
function compile_cpp {
108113
echo "Compiling and installing MONAI cpp extensions..."
114+
# depends on setup.py behaviour for building
115+
# currently setup.py uses environment variables: BUILD_MONAI and FORCE_CUDA
109116
${cmdPrefix}python setup.py -v develop --uninstall
110117
if [[ "$OSTYPE" == "darwin"* ]];
111118
then # clang for mac os
@@ -138,6 +145,7 @@ function clean_py {
138145

139146
find ${TO_CLEAN} -type f -name "*.py[co]" -delete
140147
find ${TO_CLEAN} -type f -name ".coverage" -delete
148+
find ${TO_CLEAN} -type f -name "*.so" -delete
141149
find ${TO_CLEAN} -type d -name "__pycache__" -delete
142150

143151
find ${TO_CLEAN} -depth -type d -name ".eggs" -exec rm -r "{}" +
@@ -248,7 +256,7 @@ cd "$homedir"
248256

249257
# python path
250258
export PYTHONPATH="$homedir:$PYTHONPATH"
251-
echo "$PYTHONPATH"
259+
echo "PYTHONPATH: $PYTHONPATH"
252260

253261
# by default do nothing
254262
cmdPrefix=""
@@ -260,8 +268,11 @@ then
260268
# commands are echoed instead of ran
261269
cmdPrefix="dryrun "
262270
function dryrun { echo " " "$@"; }
271+
else
272+
check_import
263273
fi
264274

275+
265276
if [ $doCleanup = true ]
266277
then
267278
echo "${separator}${blue}clean${noColor}"

‎setup.cfg

+1-1
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ exclude = *.pyi,.git,.eggs,monai/_version.py,versioneer.py,venv,_version.py
6363
known_first_party = monai
6464
profile = black
6565
line_length = 120
66-
skip = .git, .eggs, venv, versioneer.py, _version.py, conf.py
66+
skip = .git, .eggs, venv, versioneer.py, _version.py, conf.py, monai/__init__.py
6767
skip_glob = *.pyi
6868

6969
[versioneer]

0 commit comments

Comments
 (0)