Skip to content

Commit ad92dd7

Browse files
vmoensvmoens
authored
Create wheels.yml for github actions (#244)
1 parent 009cea6 commit ad92dd7

File tree

19 files changed

+261
-63
lines changed

19 files changed

+261
-63
lines changed

‎.circleci/config.yml‎

Lines changed: 75 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -656,32 +656,102 @@ workflows:
656656

657657
unittest:
658658
jobs:
659+
- unittest_macos_cpu:
660+
cu_version: cpu
661+
name: unittest_macos_cpu_py3.7
662+
python_version: '3.7'
663+
- unittest_linux_cpu:
664+
cu_version: cpu
665+
name: unittest_linux_cpu_py3.7
666+
python_version: '3.7'
667+
- unittest_linux_gpu:
668+
cu_version: cu113
669+
name: unittest_linux_gpu_py3.7
670+
python_version: '3.7'
671+
- unittest_linux_optdeps_gpu:
672+
cu_version: cu113
673+
name: unittest_linux_optdeps_gpu_py3.7
674+
python_version: '3.7'
675+
- unittest_linux_stable_cpu:
676+
cu_version: cpu
677+
name: unittest_linux_stable_cpu_py3.7
678+
python_version: '3.7'
679+
- unittest_linux_stable_gpu:
680+
cu_version: cu113
681+
name: unittest_linux_stable_gpu_py3.7
682+
python_version: '3.7'
683+
659684
- unittest_macos_cpu:
660685
cu_version: cpu
661686
name: unittest_macos_cpu_py3.8
662687
python_version: '3.8'
663-
664688
- unittest_linux_cpu:
665689
cu_version: cpu
666690
name: unittest_linux_cpu_py3.8
667691
python_version: '3.8'
668-
669692
- unittest_linux_gpu:
670693
cu_version: cu113
671694
name: unittest_linux_gpu_py3.8
672695
python_version: '3.8'
673-
674696
- unittest_linux_optdeps_gpu:
675697
cu_version: cu113
676698
name: unittest_linux_optdeps_gpu_py3.8
677699
python_version: '3.8'
678-
679700
- unittest_linux_stable_cpu:
680701
cu_version: cpu
681702
name: unittest_linux_stable_cpu_py3.8
682703
python_version: '3.8'
683-
684704
- unittest_linux_stable_gpu:
685705
cu_version: cu113
686706
name: unittest_linux_stable_gpu_py3.8
687707
python_version: '3.8'
708+
709+
- unittest_macos_cpu:
710+
cu_version: cpu
711+
name: unittest_macos_cpu_py3.9
712+
python_version: '3.9'
713+
- unittest_linux_cpu:
714+
cu_version: cpu
715+
name: unittest_linux_cpu_py3.9
716+
python_version: '3.9'
717+
- unittest_linux_gpu:
718+
cu_version: cu113
719+
name: unittest_linux_gpu_py3.9
720+
python_version: '3.9'
721+
- unittest_linux_optdeps_gpu:
722+
cu_version: cu113
723+
name: unittest_linux_optdeps_gpu_py3.9
724+
python_version: '3.9'
725+
- unittest_linux_stable_cpu:
726+
cu_version: cpu
727+
name: unittest_linux_stable_cpu_py3.9
728+
python_version: '3.9'
729+
- unittest_linux_stable_gpu:
730+
cu_version: cu113
731+
name: unittest_linux_stable_gpu_py3.9
732+
python_version: '3.9'
733+
734+
- unittest_macos_cpu:
735+
cu_version: cpu
736+
name: unittest_macos_cpu_py3.10
737+
python_version: '3.10'
738+
- unittest_linux_cpu:
739+
cu_version: cpu
740+
name: unittest_linux_cpu_py3.10
741+
python_version: '3.10'
742+
- unittest_linux_gpu:
743+
cu_version: cu113
744+
name: unittest_linux_gpu_py3.10
745+
python_version: '3.10'
746+
- unittest_linux_optdeps_gpu:
747+
cu_version: cu113
748+
name: unittest_linux_optdeps_gpu_py3.10
749+
python_version: '3.10'
750+
- unittest_linux_stable_cpu:
751+
cu_version: cpu
752+
name: unittest_linux_stable_cpu_py3.10
753+
python_version: '3.10'
754+
- unittest_linux_stable_gpu:
755+
cu_version: cu113
756+
name: unittest_linux_stable_gpu_py3.10
757+
python_version: '3.10'

‎.circleci/unittest/linux/scripts/environment.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ dependencies:
88
- hypothesis
99
- future
1010
- cloudpickle
11-
- gym_retro
1211
- gym
1312
- pygame
1413
- gym[accept-rom-license]
@@ -24,5 +23,4 @@ dependencies:
2423
- dm_control
2524
- mujoco_py
2625
- hydra-core
27-
- pyrender
2826
- tensorboard

‎.circleci/unittest/linux/scripts/install.sh‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,18 +36,14 @@ printf "Installing PyTorch with %s\n" "${CU_VERSION}"
3636
if [ "${CU_VERSION:-}" == cpu ] ; then
3737
# conda install -y pytorch torchvision cpuonly -c pytorch-nightly
3838
# use pip to install pytorch as conda can frequently pick older release
39-
if [[ $OSTYPE == 'darwin'* ]]; then
40-
pip3 install --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cpu
41-
else
42-
pip3 install torch torchvision -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre
43-
fi
39+
conda install -y pytorch torchvision torchaudio cpuonly -c pytorch-nightly
4440
else
45-
pip3 install --pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cu113
41+
conda install -y pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch-nightly
4642
fi
4743

4844
printf "Installing functorch\n"
49-
pip install ninja # Makes the build go faster
50-
pip install "git+https://github.com/pytorch/functorch.git"
45+
python -m pip install ninja # Makes the build go faster
46+
python -m pip install "git+https://github.com/pytorch/functorch.git"
5147

5248
# smoke test
5349
python -c "import functorch"

‎.circleci/unittest/linux_optdeps/scripts/install.sh‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,18 +36,14 @@ printf "Installing PyTorch with %s\n" "${CU_VERSION}"
3636
if [ "${CU_VERSION:-}" == cpu ] ; then
3737
# conda install -y pytorch torchvision cpuonly -c pytorch-nightly
3838
# use pip to install pytorch as conda can frequently pick older release
39-
if [[ $OSTYPE == 'darwin'* ]]; then
40-
pip3 install --pre torch --extra-index-url https://download.pytorch.org/whl/nightly/cpu
41-
else
42-
pip3 install torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre
43-
fi
39+
conda install -y pytorch cpuonly -c pytorch-nightly
4440
else
45-
pip3 install --pre torch --extra-index-url https://download.pytorch.org/whl/nightly/cu113
41+
conda install -y pytorch cudatoolkit=11.3 -c pytorch-nightly
4642
fi
4743

4844
printf "Installing functorch\n"
49-
pip install ninja # Makes the build go faster
50-
pip install "git+https://github.com/pytorch/functorch.git"
45+
python -m pip install ninja # Makes the build go faster
46+
python -m pip install "git+https://github.com/pytorch/functorch.git"
5147

5248
# smoke test
5349
python -c "import functorch"

‎.circleci/unittest/linux_stable/scripts/environment.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ dependencies:
99
- hypothesis
1010
- future
1111
- cloudpickle
12-
- gym_retro
1312
- gym
1413
- pygame
1514
- gym[accept-rom-license]
@@ -25,5 +24,4 @@ dependencies:
2524
- dm_control
2625
- mujoco_py
2726
- hydra-core
28-
- pyrender
2927
- tensorboard

‎.circleci/unittest/linux_stable/scripts/install.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,13 @@ printf "Installing PyTorch with %s\n" "${CU_VERSION}"
3737
if [ "${CU_VERSION:-}" == cpu ] ; then
3838
# conda install -y pytorch torchvision cpuonly -c pytorch-nightly
3939
# use pip to install pytorch as conda can frequently pick older release
40-
pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
40+
conda install -y pytorch torchvision torchaudio cpuonly -c pytorch
4141
else
42-
pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113
42+
conda install -y pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch
4343
fi
4444

4545
printf "Installing functorch\n"
46-
pip install git+https://github.com/pytorch/functorch.git@release/0.2
46+
python -m pip install git+https://github.com/pytorch/functorch.git@release/0.2
4747

4848
# smoke test
4949
python -c "import functorch"

‎.github/workflows/wheels.yml‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: Wheels
2+
on:
3+
pull_request:
4+
types: [opened, synchronize, reopened]
5+
push:
6+
branches:
7+
- release/0.0.1
8+
9+
jobs:
10+
11+
build-wheel-linux:
12+
runs-on: ubuntu-18.04
13+
strategy:
14+
matrix:
15+
python_version: [["3.7", "cp37-cp37m"], ["3.8", "cp38-cp38"], ["3.9", "cp39-cp39"], ["3.10", "cp310-cp310"]]
16+
cuda_support: [["", "--extra-index-url https://download.pytorch.org/whl/cpu", "\"['cpu', '11.3', '11.6']\"", "cpu"], ["+cu102", "", "\"['10.2']\"", "cuda102"]]
17+
container: pytorch/manylinux-${{ matrix.cuda_support[3] }}
18+
steps:
19+
- name: Checkout torchrl
20+
uses: actions/checkout@v2
21+
- name: Install PyTorch 1.12 RC
22+
run: |
23+
export PATH="/opt/python/${{ matrix.python_version[1] }}/bin:$PATH"
24+
python3 -mpip install torch==1.12 ${{ matrix.cuda_support[1] }}
25+
python3 -mpip install "git+https://github.com/pytorch/functorch.git@release/0.2"
26+
- name: Build wheel
27+
run: |
28+
export PATH="/opt/python/${{ matrix.python_version[1] }}/bin:$PATH"
29+
python3 -mpip install wheel
30+
VERSION_TAG=${{ matrix.cuda_support[0] }} PYTORCH_CUDA_RESTRICTIONS=${{ matrix.cuda_support[2] }} python3 setup.py bdist_wheel
31+
# NB: wheels have the linux_x86_64 tag so we rename to manylinux1
32+
# find . -name 'dist/*whl' -exec bash -c ' mv $0 ${0/linux/manylinux1}' {} \;
33+
# pytorch/pytorch binaries are also manylinux_2_17 compliant but they
34+
# pretend that they're manylinux1 compliant so we do the same.
35+
- name: Show auditwheel output; confirm 2-17
36+
run: |
37+
python3 -mpip install auditwheel
38+
auditwheel show dist/*
39+
- name: Upload wheel for the test-wheel job
40+
uses: actions/upload-artifact@v2
41+
with:
42+
name: torchrl-linux-${{ matrix.python_version[0] }}.whl
43+
path: dist/torchrl-0.0.1-*.whl
44+
- name: Upload wheel for download
45+
uses: actions/upload-artifact@v2
46+
with:
47+
name: torchrl-batch.whl
48+
path: dist/*.whl
49+
50+
build-wheel-mac:
51+
runs-on: macos-latest
52+
strategy:
53+
matrix:
54+
python_version: [["3.7", "3.7"], ["3.8", "3.8"], ["3.9", "3.9"], ["3.10", "3.10.3"]]
55+
steps:
56+
- name: Setup Python
57+
uses: actions/setup-python@v2
58+
with:
59+
python-version: ${{ matrix.python_version[1] }}
60+
architecture: x64
61+
- name: Checkout torchrl
62+
uses: actions/checkout@v2
63+
- name: Install PyTorch 1.12 RC
64+
run: |
65+
python3 -mpip install torch==1.12 --extra-index-url https://download.pytorch.org/whl/cpu
66+
python3 -mpip install "git+https://github.com/pytorch/functorch.git@release/0.2"
67+
- name: Build wheel
68+
run: |
69+
export CC=clang CXX=clang++
70+
python3 -mpip install wheel
71+
python3 setup.py bdist_wheel
72+
- name: Upload wheel for the test-wheel job
73+
uses: actions/upload-artifact@v2
74+
with:
75+
name: torchrl-mac-${{ matrix.python_version[0] }}.whl
76+
path: dist/*.whl
77+
- name: Upload wheel for download
78+
uses: actions/upload-artifact@v2
79+
with:
80+
name: torchrl-batch.whl
81+
path: dist/*.whl
82+
83+
test-wheel:
84+
needs: [build-wheel-linux, build-wheel-mac]
85+
strategy:
86+
matrix:
87+
os: [["linux", "ubuntu-18.04"], ["mac", "macos-latest"]]
88+
python_version: [ "3.7", "3.8", "3.9", "3.10" ]
89+
runs-on: ${{ matrix.os[1] }}
90+
steps:
91+
- name: Setup Python
92+
uses: actions/setup-python@v2
93+
with:
94+
python-version: ${{ matrix.python_version }}
95+
architecture: x64
96+
- name: Checkout torchrl
97+
uses: actions/checkout@v2
98+
- name: Install PyTorch 1.12 RC
99+
run: |
100+
python3 -mpip install torch==1.12 torchvision --extra-index-url https://download.pytorch.org/whl/cpu
101+
python3 -mpip install "git+https://github.com/pytorch/functorch.git@release/0.2"
102+
- name: Upgrade pip
103+
run: |
104+
python3 -mpip install --upgrade pip
105+
- name: Install test dependencies
106+
run: |
107+
python3 -mpip install numpy pytest pytest-cov codecov unittest-xml-reporting pillow>=4.1.1 scipy av networkx expecttest pyyaml
108+
- name: Download built wheels
109+
uses: actions/download-artifact@v2
110+
with:
111+
name: torchrl-${{ matrix.os[0] }}-${{ matrix.python_version }}.whl
112+
path: /tmp/wheels
113+
- name: Install built wheels
114+
run: |
115+
python3 -mpip install /tmp/wheels/*
116+
- name: Log version string
117+
run: |
118+
# Avoid ambiguity of "import torchrl" by deleting the source files.
119+
rm -rf torchrl/
120+
python -c "import torchrl; print(torchrl.__version__)"
121+
- name: Run tests
122+
run: |
123+
set -e
124+
export IN_CI=1
125+
mkdir test-reports
126+
python -m torch.utils.collect_env
127+
python -c "import torchrl; print(torchrl.__version__)"
128+
EXIT_STATUS=0
129+
pytest test/smoke_test.py -v --durations 20
130+
exit $EXIT_STATUS

‎test/test_tensordict.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@
77
import os.path
88
import re
99

10-
import numpy as np
1110
import pytest
1211
import torch
1312
from _utils_internal import get_available_devices
1413
from torch import multiprocessing as mp
14+
from torchrl import prod
1515
from torchrl.data import SavedTensorDict, TensorDict, MemmapTensor
1616
from torchrl.data.tensordict.tensordict import (
1717
assert_allclose_td,
@@ -810,7 +810,7 @@ def test_view(self, td_name):
810810
td = getattr(self, td_name)
811811
td_view = td.view(-1)
812812
tensor = td.get("a")
813-
tensor = tensor.view(-1, tensor.numel() // np.prod(td.batch_size))
813+
tensor = tensor.view(-1, tensor.numel() // prod(td.batch_size))
814814
tensor = torch.ones_like(tensor)
815815
if td_name == "sub_td":
816816
td_view.set_("a", tensor)

‎test/test_transforms.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
# This source code is licensed under the MIT license found in the
44
# LICENSE file in the root directory of this source tree.
55
import argparse
6-
import math
76
from multiprocessing import Lock
87

98
import pytest
@@ -12,6 +11,7 @@
1211
from mocking_classes import ContinuousActionVecMockEnv
1312
from torch import Tensor
1413
from torch import multiprocessing as mp
14+
from torchrl import prod
1515
from torchrl.data import NdBoundedTensorSpec, CompositeSpec
1616
from torchrl.data import TensorDict
1717
from torchrl.envs import EnvCreator, SerialEnv
@@ -448,7 +448,7 @@ def test_flatten(self, keys, size, nchannels, batch, device):
448448
)
449449
td.set("dont touch", dont_touch.clone())
450450
flatten(td)
451-
expected_size = math.prod(size + [nchannels])
451+
expected_size = prod(size + [nchannels])
452452
for key in keys:
453453
assert td.get(key).shape[-3] == expected_size
454454
assert (td.get("dont touch") == dont_touch).all()

‎torchrl/__init__.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@
55

66
import abc
77
import collections
8+
import math
89
import time
10+
import typing
11+
from typing import Optional, Type, Tuple
912
from warnings import warn
1013

1114
import numpy as np
@@ -112,3 +115,10 @@ def __init__(self, fun=lambda x: x):
112115
def __missing__(self, key):
113116
value = self.fun(key)
114117
return value
118+
119+
120+
def prod(sequence):
121+
if hasattr(math, "prod"):
122+
return math.prod(sequence)
123+
else:
124+
return int(np.prod(sequence))

0 commit comments

Comments
 (0)