Skip to content

Commit

Permalink
[CI] Fix releases on GH actions (#505)
Browse files Browse the repository at this point in the history
### Summary:

Fix the release upload process for Mac builds.

### Test Plan:

CI + test release that worked as expected
  • Loading branch information
VivekPanyam authored Aug 17, 2021
1 parent 75cdce6 commit 6ffc925
Show file tree
Hide file tree
Showing 3 changed files with 40 additions and 19 deletions.
1 change: 1 addition & 0 deletions .github/workflows/mac_ci.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions build/ci_matrix.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
run: ./build/ci/gh_actions_build.sh
env:
NEUROPOD_TEST_FRAMEWORKS: ${{{{ matrix.test_frameworks }}}}
GH_UPLOAD_TOKEN: ${{{{ secrets.GITHUB_TOKEN }}}}
strategy:
matrix:
include:
Expand Down
57 changes: 38 additions & 19 deletions build/upload_release.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,31 @@
CUDA_VERSION = os.getenv("NEUROPOD_CUDA_VERSION") or "10.0"
IS_MAC = platform.system() == "Darwin"

GIT_TAG = os.getenv("TRAVIS_TAG", os.getenv("BUILDKITE_TAG"))
GIT_TAG = os.getenv("BUILDKITE_TAG")
if GIT_TAG is None:
# Check GH actions
github_ref = os.getenv("GITHUB_REF", default="")
if github_ref.startswith("refs/tags/"):
GIT_TAG = github_ref.replace("refs/tags/","", 1)

PYTHON_VERSION = "{}{}".format(sys.version_info.major, sys.version_info.minor)
GH_UPLOAD_TOKEN = os.getenv("GH_UPLOAD_TOKEN")

# Get the frameworks we should upload
NEUROPOD_TEST_FRAMEWORKS = os.getenv("NEUROPOD_TEST_FRAMEWORKS")
if NEUROPOD_TEST_FRAMEWORKS is not None:
NEUROPOD_TEST_FRAMEWORKS = set(NEUROPOD_TEST_FRAMEWORKS.split(","))

def should_upload(framework):
"""
Whether we should upload an artifact for `framework`
"""
if NEUROPOD_TEST_FRAMEWORKS is None:
return True

return framework in NEUROPOD_TEST_FRAMEWORKS


def upload():
release_id = get_release_id(GIT_TAG)
platform = "libneuropod-{gpustring}-{os}-{tag}".format(
Expand All @@ -53,24 +74,26 @@ def upload():
)
)

# For each OS: For each backend version: For each CPU/GPU:
upload_package("source/bazel-bin/neuropod/backends/torchscript/neuropod_torchscript_backend.tar.gz", release_id, "{}-torchscript-{}-backend.tar.gz".format(platform, REQUESTED_TORCH_VERSION))
if should_upload("torchscript"):
# For each OS: For each backend version: For each CPU/GPU:
upload_package("source/bazel-bin/neuropod/backends/torchscript/neuropod_torchscript_backend.tar.gz", release_id, "{}-torchscript-{}-backend.tar.gz".format(platform, REQUESTED_TORCH_VERSION))

# Only upload these once
if REQUESTED_TORCH_VERSION != "1.6.0" and REQUESTED_TORCH_VERSION != "1.7.0":
if should_upload("tensorflow"):
# For each OS: For each backend version: For each CPU/GPU:
upload_package("source/bazel-bin/neuropod/backends/tensorflow/neuropod_tensorflow_backend.tar.gz", release_id, "{}-tensorflow-{}-backend.tar.gz".format(platform, REQUESTED_TF_VERSION))

# The python package is the same across CPU/GPU and different versions of backends so we'll only upload once for mac and once for linux
# The python package is the same across CPU/GPU and different versions of backends so we'll only upload once for mac and once for linux
# TODO(vip): Do this better
if REQUESTED_TORCH_VERSION != "1.6.0" and REQUESTED_TORCH_VERSION != "1.7.0" and not IS_GPU:
# For each OS: For each python version
if not IS_GPU:
# Upload the pythonbridge backend
upload_package("source/bazel-bin/neuropod/backends/python_bridge/neuropod_pythonbridge_backend.tar.gz", release_id, "{}-python-{}-backend.tar.gz".format(platform, PYTHON_VERSION))
# Upload the pythonbridge backend
upload_package("source/bazel-bin/neuropod/backends/python_bridge/neuropod_pythonbridge_backend.tar.gz", release_id, "{}-python-{}-backend.tar.gz".format(platform, PYTHON_VERSION))

# Upload the wheels
for gpath in ["source/python/dist/neuropod-*.whl"]:
whl_path = glob.glob(gpath)[0]
fname = os.path.basename(whl_path)
upload_package(whl_path, release_id, fname, content_type="application/zip")
# Upload the wheels
for gpath in ["source/python/dist/neuropod-*.whl"]:
whl_path = glob.glob(gpath)[0]
fname = os.path.basename(whl_path)
upload_package(whl_path, release_id, fname, content_type="application/zip")

def get_release_id(tag_name):
# https://api.github.com/repos/uber/neuropod/releases/tags/{tag_name}
Expand Down Expand Up @@ -111,10 +134,6 @@ def upload_package(local_path, release_id, asset_filename, content_type="applica
if not GIT_TAG or not GH_UPLOAD_TOKEN:
# Don't upload if we don't have a tag or token
pass
elif os.getenv("TRAVIS_TAG") is not None and not IS_MAC:
# Don't push releases from linux on Travis
# (buildkite pushes all the linux releases)
pass
else:
print("Uploading packages...")
print("Uploading packages for tag {}...".format(GIT_TAG))
upload()

0 comments on commit 6ffc925

Please sign in to comment.