Adding google-nucleus to the REQUIRED_PACKAGES list (#416)

* Adding google-nucleus to the REQUIRED_PACKAGES list

In PR #346 we added a temporary solution to install Nucleus using its
wheel file. Nucleus now has pip install so we can remove that hack.

* Review comments

Author: samanvp

cloudbuild.yaml

@@ -14,7 +14,7 @@
 
 # Builds docker image for gcp-variant-transforms:
 # Run using:
-# $ gcloud container builds submit --config cloudbuild.yaml --timeout 1h .
+# $ gcloud builds submit --config cloudbuild.yaml --timeout 1h .
 substitutions:
   _CUSTOM_TAG_NAME: 'latest'
 steps:

deploy_and_run_tests.sh

@@ -207,8 +207,9 @@ if [[ -z "${skip_build}" ]]; then
   # TODO(bashir2): This will pick and include all directories in the image,
   # including local build and library dirs that do not need to be included.
   # Update this to include only the required files/directories.
-  gcloud container builds submit --config "${build_file}" \
+  gcloud builds submit --config "${build_file}" \
       --project "${project}" \
+      --timeout '30m' \
       --substitutions _CUSTOM_TAG_NAME="${image_tag}" .
 fi
 

gcp_variant_transforms/beam_io/vcfio_test.py

@@ -101,11 +101,14 @@ def _get_sample_variant_1(is_for_nucleus=False):
         vcfio.VariantCall(name='Sample2', genotype=[1, 0], info={'GQ': 20}))
   else:
     # 0.1 -> 0.25 float precision loss due to binary floating point conversion.
-    vcf_line = ('20	1234	rs123;rs2	C	A,T	50	'
+    # rs123;rs2 -> rs123 it seems nuclues does not parse IDs correctly.
+    # quality=50 -> 50.0 nucleus converts quality values to float.
+    # TODO(samanvp): convert all quality values to float.
+    vcf_line = ('20	1234	rs123	C	A,T	50	'
                 'PASS	AF=0.5,0.25;NS=1	GT:GQ	0/0:48	1/0:20\n')
     variant = vcfio.Variant(
         reference_name='20', start=1233, end=1234, reference_bases='C',
-        alternate_bases=['A', 'T'], names=['rs123', 'rs2'], quality=50,
+        alternate_bases=['A', 'T'], names=['rs123'], quality=50.0,
         filters=['PASS'], info={'AF': [0.5, 0.25], 'NS': 1})
     variant.calls.append(
         vcfio.VariantCall(name='Sample1', genotype=[0, 0], info={'GQ': 48}))

setup.py

@@ -14,9 +14,7 @@
 
 """Beam pipelines for processing variants based on VCF files."""
 
-import os
 import setuptools
-from setuptools.command.build_py import build_py
 
 REQUIRED_PACKAGES = [
     'cython>=0.28.1',
@@ -28,6 +26,9 @@
     'google-api-python-client>=1.6',
     'intervaltree>=2.1.0,<2.2.0',
     'pyvcf<0.7.0',
+    'google-nucleus==0.2.0',
+    # Nucleus needs uptodate protocol buffer compiler (protoc).
+    'protobuf>=3.6.1',
     'mmh3<2.6',
     # Need to explicitly install v<=1.2.0. apache-beam requires
     # google-cloud-pubsub 0.26.0, which relies on google-cloud-core<0.26dev,
@@ -45,20 +46,6 @@
     'nose>=1.0',
 ]
 
-
-class BuildPyCommand(build_py):
-  """Custom build command for installing libraries outside of PyPi."""
-
-  _NUCLEUS_WHEEL_PATH = (
-      'https://storage.googleapis.com/gcp-variant-transforms-setupfiles/'
-      'nucleus/Nucleus-0.1.0-py2-none-any.whl')
-
-  def run(self):
-    # Temporary workaround for installing Nucleus until it's available via PyPi.
-    os.system('pip install {}'.format(BuildPyCommand._NUCLEUS_WHEEL_PATH))
-    build_py.run(self)
-
-
 setuptools.setup(
     name='gcp_variant_transforms',
     version='0.5.1',
@@ -90,7 +77,4 @@ def run(self):
     package_data={
         'gcp_variant_transforms': ['gcp_variant_transforms/testing/testdata/*']
     },
-    cmdclass={
-        'build_py': BuildPyCommand,
-    },
 )