Re-enable tests

plasticityai · Nov 9, 2018 · 960877f · 960877f
1 parent 858ab00
commit 960877f
Show file tree

Hide file tree

Showing 4 changed files with 309 additions and 301 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -39,92 +39,90 @@ Lint:
 Test Python 2:
   stage: Test
   script:
-    - echo pass
-    # - apt-get install python-pip -y
-    # - pip2 install torch\>=0.4.1 # This is not in requirements.txt so must be installed like this
-    # - pip2 install -U pip setuptools
-    # - python2 setup.py install
-    # - pip install gensim
-    # - wget --quiet http://magnitude.plasticity.ai/word2vec/light/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.light.magnitude
-    # - wget --quiet http://magnitude.plasticity.ai/word2vec/medium/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.medium.magnitude
-    # - wget --quiet http://magnitude.plasticity.ai/word2vec/heavy/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.heavy.magnitude
-    # - python2 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude
-    # - python2 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude
-    # - python2 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude
-    # - python2 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude
-    # - python2 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude
-    # - python2 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude
-    # - python2 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude -s
-    # - python2 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude -s
-    # - python2 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude -s
-    # - python2 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude -s
-    # - python2 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude -s
-    # - python2 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude -s
-    # - python2 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude -s -a
-    # - python2 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude -s -a
-    # - python2 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude -s -a
-    # - python2 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude -s -a
-    # - python2 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude -s -a
-    # - python2 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude -s -a
-    # - python2 -m pymagnitude.converter -i tests/models/word2vec.bin -o w2v.magnitude
-    # - python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude
-    # - python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -s
-    # - python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -s -a
-    # - python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude
-    # - python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude -s
-    # - python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude -a
-    # - ls
-    # - python2 -m tests.tests -i GoogleNews-vectors-negative300.light.magnitude -s GoogleNews-vectors-negative300.medium.magnitude -a GoogleNews-vectors-negative300.heavy.magnitude -- -v
+    - apt-get install python-pip -y
+    - pip2 install torch\>=0.4.1 # This is not in requirements.txt so must be installed like this
+    - pip2 install -U pip setuptools
+    - python2 setup.py install
+    - pip install gensim
+    - wget --quiet http://magnitude.plasticity.ai/word2vec/light/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.light.magnitude
+    - wget --quiet http://magnitude.plasticity.ai/word2vec/medium/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.medium.magnitude
+    - wget --quiet http://magnitude.plasticity.ai/word2vec/heavy/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.heavy.magnitude
+    - python2 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude
+    - python2 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude
+    - python2 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude
+    - python2 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude
+    - python2 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude
+    - python2 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude
+    - python2 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude -s
+    - python2 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude -s
+    - python2 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude -s
+    - python2 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude -s
+    - python2 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude -s
+    - python2 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude -s
+    - python2 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude -s -a
+    - python2 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude -s -a
+    - python2 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude -s -a
+    - python2 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude -s -a
+    - python2 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude -s -a
+    - python2 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude -s -a
+    - python2 -m pymagnitude.converter -i tests/models/word2vec.bin -o w2v.magnitude
+    - python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude
+    - python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -s
+    - python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -s -a
+    - python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude
+    - python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude -s
+    - python2 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude -a
+    - ls
+    - python2 -m tests.tests -i GoogleNews-vectors-negative300.light.magnitude -s GoogleNews-vectors-negative300.medium.magnitude -a GoogleNews-vectors-negative300.heavy.magnitude -- -v
 
 Test Python 3:
   stage: Test
   script:
-    - echo pass
-    # - apt-get install build-essential -y
-    # - apt-get install libffi-dev libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev  -y
-    # - apt-get install libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev -y
-    # - wget https://www.python.org/ftp/python/3.7.1/Python-3.7.1.tar.xz
-    # - tar xf Python-3.7.1.tar.xz
-    # - cd Python-3.7.1 && ./configure --enable-shared && make -j 8 && make altinstall && cd ../
-    # - echo $(pwd)/Python-3.7.1
-    # - ls $(pwd)/Python-3.7.1
-    # - export LD_LIBRARY_PATH=$(pwd)/Python-3.7.1:$LD_LIBRARY_PATH
-    # - echo $LD_LIBRARY_PATH
-    # - curl https://bootstrap.pypa.io/get-pip.py | python3.7
-    # - python3.7 -m pip install torch\>=0.4.1 # This is not in requirements.txt so must be installed like this
-    # - python3.7 -m pip install -U wheel setuptools # TEMP: removing upgrading of pip here, because the newest one is broken for Python 3
-    # - python3.7 setup.py install
-    # - python3.7 -m pip install gensim
-    # - wget --quiet http://magnitude.plasticity.ai/word2vec/light/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.light.magnitude
-    # - wget --quiet http://magnitude.plasticity.ai/word2vec/medium/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.medium.magnitude
-    # - wget --quiet http://magnitude.plasticity.ai/word2vec/heavy/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.heavy.magnitude
-    # - python3.7 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude
-    # - python3.7 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude
-    # - python3.7 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude
-    # - python3.7 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude
-    # - python3.7 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude
-    # - python3.7 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude
-    # - python3.7 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude -s
-    # - python3.7 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude -s
-    # - python3.7 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude -s
-    # - python3.7 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude -s
-    # - python3.7 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude -s
-    # - python3.7 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude -s
-    # - python3.7 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude -s -a
-    # - python3.7 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude -s -a
-    # - python3.7 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude -s -a
-    # - python3.7 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude -s -a
-    # - python3.7 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude -s -a
-    # - python3.7 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude -s -a
-    # - python3.7 -m pymagnitude.converter -i tests/models/word2vec.bin -o w2v.magnitude
-    # - python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude
-    # - python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -s
-    # - python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -s -a
-    # - python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude
-    # - python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude -s
-    # - python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude -a
-    # - ls
-    # - python3.7 -m tests.tests -i GoogleNews-vectors-negative300.light.magnitude -s GoogleNews-vectors-negative300.medium.magnitude -a GoogleNews-vectors-negative300.heavy.magnitude -- -v
+    - apt-get install build-essential -y
+    - apt-get install libffi-dev libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev  -y
+    - apt-get install libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev -y
+    - wget https://www.python.org/ftp/python/3.7.1/Python-3.7.1.tar.xz
+    - tar xf Python-3.7.1.tar.xz
+    - cd Python-3.7.1 && ./configure --enable-shared && make -j 8 && make altinstall && cd ../
+    - echo $(pwd)/Python-3.7.1
+    - ls $(pwd)/Python-3.7.1
+    - export LD_LIBRARY_PATH=$(pwd)/Python-3.7.1:$LD_LIBRARY_PATH
+    - echo $LD_LIBRARY_PATH
+    - curl https://bootstrap.pypa.io/get-pip.py | python3.7
+    - python3.7 -m pip install torch\>=0.4.1 # This is not in requirements.txt so must be installed like this
+    - python3.7 -m pip install -U wheel setuptools # TEMP: removing upgrading of pip here, because the newest one is broken for Python 3
+    - python3.7 setup.py install
+    - python3.7 -m pip install gensim
+    - wget --quiet http://magnitude.plasticity.ai/word2vec/light/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.light.magnitude
+    - wget --quiet http://magnitude.plasticity.ai/word2vec/medium/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.medium.magnitude
+    - wget --quiet http://magnitude.plasticity.ai/word2vec/heavy/GoogleNews-vectors-negative300.magnitude -O GoogleNews-vectors-negative300.heavy.magnitude
+    - python3.7 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude
+    - python3.7 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude
+    - python3.7 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude
+    - python3.7 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude
+    - python3.7 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude
+    - python3.7 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude
+    - python3.7 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude -s
+    - python3.7 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude -s
+    - python3.7 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude -s
+    - python3.7 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude -s
+    - python3.7 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude -s
+    - python3.7 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude -s
+    - python3.7 -m pymagnitude.converter -i tests/models/word2vec.txt -o out.magnitude -s -a
+    - python3.7 -m pymagnitude.converter -i tests/models/word2vec.bin -o out.magnitude -s -a
+    - python3.7 -m pymagnitude.converter -i tests/models/glove.txt -o out.magnitude -s -a
+    - python3.7 -m pymagnitude.converter -i tests/models/fasttext.vec -o out.magnitude -s -a
+    - python3.7 -m pymagnitude.converter -i tests/models/zero.txt -o out.magnitude -s -a
+    - python3.7 -m pymagnitude.converter -i tests/models/large.txt -o out.magnitude -s -a
+    - python3.7 -m pymagnitude.converter -i tests/models/word2vec.bin -o w2v.magnitude
+    - python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude
+    - python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -s
+    - python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -s -a
+    - python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude
+    - python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude -s
+    - python3.7 -m pymagnitude.converter -i tests/models/elmo_weights.hdf5 -o elmo.magnitude -v w2v.magnitude -a
+    - ls
+    - python3.7 -m tests.tests -i GoogleNews-vectors-negative300.light.magnitude -s GoogleNews-vectors-negative300.medium.magnitude -a GoogleNews-vectors-negative300.heavy.magnitude -- -v
 
 Deploy to PyPI:
   stage: Deploy to PyPI

diff --git a/pymagnitude/__init__.py b/pymagnitude/__init__.py
@@ -2403,31 +2403,29 @@ def write_data(self, start_offset, data, amount, offset):
             """Writes data fetched to the network cache and
             returns only the amount requested back."""
             # Writes the entire data fetched to the cache
+            if self.vfsfile.should_cache:
+                # Uses itself as a cache object
+                self._start_offset = start_offset
+                self.set_data(data)
+                if self.vfsfile.trace_log:
+                    print("[HTTPVFS] Cache Size: %d bytes" % (self.cache_size,))
 
-            # Uses itself as a cache object
-            self._start_offset = start_offset
-            self.set_data(data)
-            if self.vfsfile.trace_log:
-                print("[HTTPVFS] Cache Size: %d bytes" % (self.cache_size,))
+                # Purge old caches
+                current_time = time.time()
+                if ((current_time -
+                        self.vfsfile.last_cache_purge) >
+                        self.vfsfile.ttl_purge_interval):
+                    if self.vfsfile.trace_log:
+                        print("[HTTPVFS] Purging expired caches...")
+                    self.vfsfile.last_cache_purge = current_time
+                    self.delete_caches()
 
-            # Purge old caches
-            current_time = time.time()
-            if ((current_time -
-                    self.vfsfile.last_cache_purge) >
-                    self.vfsfile.ttl_purge_interval):
-                if self.vfsfile.trace_log:
-                    print("[HTTPVFS] Purging expired caches...")
-                self.vfsfile.last_cache_purge = current_time
-                self.delete_caches()
-
-            # Adds itself to the cache array, so the next read
-            # succeed
-            self.add_to_caches()
-            if amount > 0:
-                return data[offset -
-                            start_offset: (offset - start_offset) + amount]
-            else:
-                return "".encode('utf-8')
+                # Adds itself to the cache array, so the next read
+                # succeed
+                self.add_to_caches()
+
+            return data[offset -
+                        start_offset: (offset - start_offset) + amount]
 
         def _prefetch_in_background(
                 self,
@@ -2526,6 +2524,12 @@ def read_data(self, amount, offset, _prefetch_in_background=None):
             into the cache using _prefetch_in_background(amount, offset)
             if it detects a non-sequential access pattern in the
             cache misses."""
+
+            # Don't do anything if caching is disabled
+            if not self.vfsfile.should_cache:
+                return None
+
+            # Find the closest cache match
             current_time = time.time()
             (
                 running_hit_amount,
@@ -2829,6 +2833,7 @@ def __init__(self, inheritfromvfsname, name, flags, vfs, options=None):
 
             # Cache + Network configuration
             defaults = {
+                'should_cache': True,
                 'network_retry_delay': 10,
                 'max_network_retries': 10,
                 'sequential_cache_default_read': 4096 * 2,
@@ -2838,7 +2843,7 @@ def __init__(self, inheritfromvfsname, name, flags, vfs, options=None):
                 'prefetch_thread_limit': 3,
                 'sequential_cache_prefetch': True,
                 'random_access_cache_prefetch': True,
-                'random_access_cache_range': 500 * (1024 ** 2),
+                'random_access_cache_range': 100 * (1024 ** 2),
                 'random_access_hit_tracker_ttl': 60,
                 'cache_ttl': 60,
                 'ttl_purge_interval': 5,
@@ -2851,6 +2856,11 @@ def __init__(self, inheritfromvfsname, name, flags, vfs, options=None):
             for k, v in defaults.items():
                 setattr(self, k, v)
             self.max_network_retries = max(self.max_network_retries, 4)
+            if not self.should_cache:
+                self.sequential_cache_prefetch = False
+                self.random_access_cache_prefetch = False
+                self.sequential_cache_default_read = 0
+                self.cache_amount = 0
 
             # Cache initialization
             self.caches = []
@@ -3050,7 +3060,7 @@ def xRead(self, amount, offset):  # noqa: N802
                     data = cache.read_data(
                         amount, offset, self._prefetch_in_background)
                     if data is None:
-                        if self.trace_log:
+                        if self.trace_log and self.should_cache:
                             print(
                                 "[HTTPVFS] Cache miss for request @ %d + %d" %
                                 (offset, amount))
@@ -3111,7 +3121,7 @@ def xRead(self, amount, offset):  # noqa: N802
 
                         # Prefetch the next sequential chunk of data in the
                         # background
-                        if self.sequential_cache_prefetch:
+                        if self.sequential_cache_prefetch and self.should_cache:
                             if self.running_hit_direction >= 0:
                                 cache.prefetch_in_background(
                                     self._prefetch_in_background,