feat: annotation reading

README.md

@@ -3,7 +3,7 @@
 # CloudVolume: IO for Neuroglancer Datasets
 
 ```python
-from cloudvolume import CloudVolume
+from cloudvolume import CloudVolume, from_cloudpath, Bbox
 
 vol = CloudVolume('gs://mylab/mouse/image', parallel=True, progress=True)
 image = vol[:,:,:] # Download a whole image stack into a numpy array from the cloud
@@ -12,6 +12,9 @@ vol[:,:,:] = image # Upload an entire image stack from a numpy array to the clou
 label = 1
 mesh = vol.mesh.get(label)
 skel = vol.skeleton.get(label)
+
+asrc = from_cloudpath("gs://mylab/mouse/annotation/synapses")
+annotations = asrc.get(label)
 ```
 
 CloudVolume is a serverless Python client for random access reading and writing of [Neuroglancer](https://github.com/google/neuroglancer/) volumes in "[Precomputed](https://github.com/google/neuroglancer/tree/master/src/datasource/precomputed#readme)" format, a set of representations for arbitrarily large volumetric images, meshes, and skeletons. CloudVolume is typically paired with [Igneous](https://github.com/seung-lab/igneous), a Kubernetes compatible system for generating image hierarchies, meshes, skeletons, and other dependency free jobs that can be applied to petavoxel scale images.
@@ -344,6 +347,22 @@ skel = skel.average_smoothing(3) # rolling average, n=3
 skel1 == skel2 # check if contents of internal arrays match
 Skeleton.equivalent(skel1, skel2) # ...even if there are differences like differently numbered edges
 
+# Annotations
+import cloudvolume
+
+asrc = cloudvolume.from_cloudpath("gs://mybucket/retina/annotations", cache=True, progress=True, mip=3)
+
+print(asrc.summary()) # get basic info about annotation set
+
+annotations = asrc.get([1,2,3,]) # tries to interpret input to mean get_by_id or get_by_bbox
+annotations = asrc.get_by_id([1,2,3,])
+annotations = asrc.get(bbox, mip=3)
+annotations = asrc.get_all()
+annotations = asrc.get_by_bbox(bbox, mip=3)
+annotations = asrc[bbox] # can use slice notation
+annotations = asrc.get_by_relationship("synapses", 1231)
+ids = asrc.ids()
+
 # Parallel Operation
 vol = CloudVolume('gs://mybucket/retina/image', parallel=True) # Use all cores
 vol.parallel = 4 # e.g. any number > 1, use this many cores
@@ -709,7 +728,7 @@ Python 2.7 is no longer supported by CloudVolume. Updated versions of `pip` will
 
 Thank you to everyone that has contributed past or current to CloudVolume or the ecosystem it serves. We love you!  
 
-Jeremy Maitin-Shepard created [Neuroglancer](https://github.com/google/neuroglancer) and defined the Precomputed format. Yann Leprince provided a [pure Python codec](https://github.com/HumanBrainProject/neuroglancer-scripts) for the compressed_segmentation format. Jeremy Maitin-Shepard and Stephen Plaza created C++ code defining the compression and decompression (respectively) protocol for [compressed_segmentation](https://github.com/janelia-flyem/compressedseg). Peter Lindstrom et al. created [the fpzip algorithm](https://computation.llnl.gov/projects/floating-point-compression), and contributed a C++ implementation and advice. Nico Kemnitz adapted our data to fpzip using the "Kempression" protocol (we named it, not him). Dan Bumbarger contributed code and information helpful for getting CloudVolume working on Windows. Fredrik Kihlander's [pure python implementation](https://github.com/wc-duck/pymmh3) of murmurhash3 and [Austin Appleby](https://github.com/aappleby/smhasher) developed murmurhash3 which is necessary for the sharded format. Ben Falk advocated for and did the bulk of the work on brotli compression. Some of the ideas in CloudVolume are based on work by Jingpeng Wu in [BigArrays.jl](https://github.com/seung-lab/BigArrays.jl).  Sven Dorkenwald, Manuel Castro, and Akhilesh Halageri contributed advice and code towards implementing the graphene interface. Oluwaseun Ogedengbe contributed documentation for the sharded format. Eric Perlman wrote the reader for Neuroglancer Multi-LOD meshes. Ignacio Tartavull and William Silversmith wrote the initial version of CloudVolume.
+Jeremy Maitin-Shepard created [Neuroglancer](https://github.com/google/neuroglancer) and defined the Precomputed format. Yann Leprince provided a [pure Python codec](https://github.com/HumanBrainProject/neuroglancer-scripts) for the compressed_segmentation format. Jeremy Maitin-Shepard and Stephen Plaza created C++ code defining the compression and decompression (respectively) protocol for [compressed_segmentation](https://github.com/janelia-flyem/compressedseg). Peter Lindstrom et al. created [the fpzip algorithm](https://computation.llnl.gov/projects/floating-point-compression), and contributed a C++ implementation and advice. Nico Kemnitz adapted our data to fpzip using the "Kempression" protocol (we named it, not him). Dan Bumbarger contributed code and information helpful for getting CloudVolume working on Windows. Fredrik Kihlander's [pure python implementation](https://github.com/wc-duck/pymmh3) of murmurhash3 and [Austin Appleby](https://github.com/aappleby/smhasher) developed murmurhash3 which is necessary for the sharded format. Ben Falk advocated for and did the bulk of the work on brotli compression. Some of the ideas in CloudVolume are based on work by Jingpeng Wu in [BigArrays.jl](https://github.com/seung-lab/BigArrays.jl).  Sven Dorkenwald, Manuel Castro, and Akhilesh Halageri contributed advice and code towards implementing the graphene interface. Oluwaseun Ogedengbe contributed documentation for the sharded format. Eric Perlman wrote the reader for Neuroglancer Multi-LOD meshes. Forrest Collman and Jeremy Maitin-Shepard both wrote the versions of the annotations service that was then adapted to CloudVolume by William Silversmith. Ignacio Tartavull and William Silversmith wrote the initial version of CloudVolume.
 
 ## Citation
 Please cite the Igneous paper if you used this package in your research:

cloudvolume/__init__.py

@@ -46,7 +46,7 @@
   skel = vol.skeletons.get(label)
 """
 
-from .cloudvolume import CloudVolume, register_plugin
+from .cloudvolume import CloudVolume, from_cloudpath, register_plugin
 
 from .connectionpools import ConnectionPool
 from .lib import Bbox, Vec
@@ -68,15 +68,20 @@
 __version__ = '12.8.0'
 
 # Register plugins
-from .datasource.precomputed import register as register_precomputed
+from .datasource.precomputed import (
+  register as register_precomputed, 
+  register_annotation as register_precomputed_annotation,
+)
 from .datasource.graphene import register as register_graphene
 from .datasource.n5 import register as register_n5
 from .datasource.zarr import register as register_zarr
 from .datasource.zarr2 import register as register_zarr2
 from .datasource.zarr3 import register as register_zarr3
 
 register_precomputed()
+register_precomputed_annotation()
 register_graphene()
+
 register_n5()
 register_zarr()
 register_zarr2()

cloudvolume/cloudvolume.py

@@ -1,11 +1,12 @@
 import sys
 import time
-from typing import Optional, Union, Tuple
+from typing import Optional, Union, Tuple, Any
 
 import multiprocessing as mp
 import numpy as np
 from tqdm import tqdm
 
+from cloudfiles import CloudFiles
 from cloudfiles.paths import normalize
 
 from .exceptions import UnsupportedFormatError, DimensionError, InfoUnavailableError
@@ -21,9 +22,13 @@
 except AttributeError:
   INTERACTIVE = bool(sys.flags.interactive)
 
-REGISTERED_PLUGINS = {}
+REGISTERED_IMAGE_PLUGINS = {}
 def register_plugin(key, creation_function):
-  REGISTERED_PLUGINS[key.lower()] = creation_function
+  REGISTERED_IMAGE_PLUGINS[key.lower()] = creation_function
+
+REGISTERED_ANNOTATION_PLUGINS = {}
+def register_annotation_plugin(key, creation_function):
+  REGISTERED_ANNOTATION_PLUGINS[key.lower()] = creation_function
 
 def compute_num_threads(num_threads:ParallelType) -> int:
   if isinstance(num_threads, bool):
@@ -268,9 +273,9 @@ def __new__(cls,
 
     def init(cloudpath):
       path = strict_extract(cloudpath)
-      if path.format in REGISTERED_PLUGINS:
+      if path.format in REGISTERED_IMAGE_PLUGINS:
         kwargs["cloudpath"] = normalize(cloudpath)
-        return REGISTERED_PLUGINS[path.format](**kwargs)
+        return REGISTERED_IMAGE_PLUGINS[path.format](**kwargs)
       else:
         raise UnsupportedFormatError(
           "Unknown format {}".format(path.format)
@@ -287,7 +292,6 @@ def init(cloudpath):
       else:
         raise err
 
-
   @classmethod
   def create_new_info(cls, *args, **kwargs):
     from .frontends import CloudVolumePrecomputed
@@ -413,3 +417,19 @@ def from_numpy(cls,
     # save the numpy array
     vol[:,:,:] = arr
     return vol
+
+def from_cloudpath(cloudpath:str, *args, **kwargs) -> Any:
+  """Create the appropriate object for the given cloudpath."""
+  path = strict_extract(cloudpath)
+
+  if path.format != "precomputed":
+    return CloudVolume(cloudpath, *args, **kwargs)
+
+  info = CloudFiles(cloudpath).get_json("info")
+  kwargs["info"] = info
+
+  if info["@type"] == "neuroglancer_annotations_v1":
+    return REGISTERED_ANNOTATION_PLUGINS['precomputed'](cloudpath, *args, **kwargs)
+
+  return CloudVolume(cloudpath, *args, **kwargs)
+

cloudvolume/datasource/graphene/mesh/sharded.py

@@ -27,7 +27,7 @@ def __init__(self, mesh_meta, cache, config, readonly):
     self.readers = {}
     for level, sharding in self.meta.info['sharding'].items(): # { level: std sharding, ... }
       spec = ShardingSpecification.from_dict(sharding)
-      self.readers[int(level)] = GrapheneShardReader(self.meta, self.cache, spec)
+      self.readers[int(level)] = GrapheneShardReader(self.meta.cloudpath, self.cache, spec)
 
   def initial_path(self, level):
     return self.meta.join(self.meta.mesh_path, self.meta.sharded_mesh_dir, str(level))

cloudvolume/datasource/precomputed/__init__.py

@@ -1,13 +1,15 @@
 from typing import Optional, Union
 
+from .annotation import PrecomputedAnnotationSource
 from .image import PrecomputedImageSource
 from .metadata import PrecomputedMetadata
 from .mesh import PrecomputedMeshSource
 from .skeleton import PrecomputedSkeletonSource
 
 from .. import get_cache_path
 from ...cloudvolume import (
-  register_plugin, SharedConfiguration,
+  register_plugin, register_annotation_plugin,
+  SharedConfiguration,
   CompressType, ParallelType, CacheType,
   SecretsType
 )
@@ -116,6 +118,31 @@ def create_precomputed(
 
     return cv
 
+def create_precomputed_annotation(
+  cloudpath:str, 
+  cache:CacheType = False,
+  info:Optional[dict] = None,
+  mip:int = -1,
+  progress:bool = False,
+  secrets:SecretsType = None,
+  use_https:bool = False,
+) -> PrecomputedAnnotationSource:
+  """
+  Note: for annotations, mips are coarsest to finest, so -1
+  means pick the finest (i.e. the scientifically useful one).
+  """
+  return PrecomputedAnnotationSource(
+    cloudpath,
+    cache=cache,
+    info=info,
+    mip=mip,
+    progress=progress,
+    secrets=secrets,
+    use_https=use_https,
+  )
+
+def register_annotation():
+  register_annotation_plugin('precomputed', create_precomputed_annotation)
 
 def register():
   register_plugin('precomputed', create_precomputed)