Update Python package

.gitignore

@@ -1,5 +1,6 @@
 # Random editor stuff
 *~
+*.swp
 
 # Python byte-compiled / optimized / DLL files
 *.pyc
@@ -9,9 +10,12 @@ src/Python/dist/*
 src/Python/tests/test_chunk*
 src/Python/wcon.egg*
 .pypirc
+openworm/
 
 #Matlab
 *.asv
 
 # Scala
 src/scala/target/*
+src/scala/**/target/*
+

src/Python/Pipfile

@@ -0,0 +1,17 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+pandas = "*"
+six = "*"
+numpy = "*"
+psutil = "*"
+jsonschema = "*"
+scipy = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.9"

src/Python/pyproject.toml

@@ -0,0 +1,21 @@
+[project]
+name = "wcon"
+version = "1.1.0"
+authors = [
+  { name="Kerr, R; Brown, A; Currie, M; OpenWorm", email="[email protected]" },
+]
+description = "Worm tracker Commons Object Notation"
+readme = "README.md"
+requires-python = ">=3.9"
+classifiers = [
+    'Development Status :: 4 - Beta',
+    'Intended Audience :: Science/Research',
+    'Topic :: Scientific/Engineering :: Bio-Informatics',
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+license = "MIT"
+
+[project.urls]
+Homepage = "https://github.com/openworm/tracker-commons"
+Issues = "https://github.com/openworm/tracker-commons/issues"

src/Python/requirements.txt

@@ -0,0 +1,6 @@
+six
+jsonschema
+numpy
+pandas
+psutil
+scipy

src/Python/tests/diagnostic_test.py

@@ -19,7 +19,8 @@
 import numpy as np
 import time
 
-sys.path.append('..')
+dir_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(dir_path, '..'))
 from wcon import WCONWorms, MeasurementUnit
 from wcon.measurement_unit import MeasurementUnitAtom
 

src/Python/tests/tests.py

@@ -16,20 +16,20 @@
 import shutil
 from scipy.constants import pi
 
-sys.path.append('..')
+dir_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(dir_path, '..'))
 from wcon import WCONWorms, MeasurementUnit
 from wcon.measurement_unit import MeasurementUnitAtom
 
-
 def setUpModule():
     # If the wcon module is installed via pip, wcon_schema.json is included
     # in the proper place.  In the git repo it is not, however, so to test
-    # we must copy it over temporarily, then remove it once tests are done.
-    shutil.copyfile('../../../wcon_schema.json', '../wcon/wcon_schema.json')
+    # we must copy it over temporarily, then remove it once tests are done.    
+    shutil.copyfile(os.path.join(dir_path, '..', '..', '..', 'wcon_schema.json'), os.path.join(dir_path, '..', 'wcon', 'wcon_schema.json'))
 
 
 def tearDownModule():
-    os.remove('../wcon/wcon_schema.json')
+    os.remove(os.path.join(dir_path, '..', 'wcon', 'wcon_schema.json'))
 
 
 def flatten(list_of_lists):
@@ -41,7 +41,7 @@ def flatten(list_of_lists):
     for element in list_of_lists:
         # If it's iterable but not a string or bytes, then recurse, otherwise
         # we are at a "leaf" node of our traversal
-        if(isinstance(element, collections.Iterable) and
+        if(isinstance(element, collections.abc.Iterable) and
            not isinstance(element, (str, bytes))):
             for sub_element in flatten(element):
                 yield sub_element
@@ -192,7 +192,9 @@ def _validate_from_schema(self, wcon_string):
         except AttributeError:
             # Only load _wcon_schema if this method gets called.  Once
             # it's loaded, though, persist it in memory and don't lose it
-            with open("../../../wcon_schema.json", "r") as wcon_schema_file:
+            wcon_schema_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                                '..', '..', '..', 'wcon_schema.json'))
+            with open(wcon_schema_path, "r") as wcon_schema_file:
                 self._wcon_schema = json.loads(wcon_schema_file.read())
 
             # Now that the schema has been loaded, we can try again
@@ -204,7 +206,8 @@ def test_schema(self):
         self._validate_from_schema(basic_wcon)
 
     def test_equality_operator(self):
-        JSON_path = '../../../tests/minimax.wcon'
+        JSON_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                                '..', '..', '..', 'tests', 'minimax.wcon'))
         w2 = WCONWorms.load_from_file(JSON_path)
         w2.units['y'] = MeasurementUnit.create('m')
         w2data = w2.data.copy()

src/Python/wcon/measurement_unit.py

@@ -21,8 +21,16 @@
 import six
 import ast
 import operator as op
-from scipy.constants import F2C, K2C, C2F, C2K, pi
+from scipy.constants import convert_temperature, pi
 
+def F2C(val):
+    return convert_temperature(val, 'F', 'C')
+def C2F(val):
+    return convert_temperature(val, 'C', 'F')
+def K2C(val):
+    return convert_temperature(val, 'K', 'C')
+def C2K(val):
+    return convert_temperature(val, 'C', 'K')
 
 def C2C(x):
     """
@@ -629,8 +637,8 @@ def _create_from_node(cls, node):
             The expression to be transformed into a MeasurementUnit
 
         """
-        if isinstance(node, ast.Num):  # <number>
-            n = node.n
+        if isinstance(node, ast.Constant):  # <number>
+            n = node.value
             assert(n != 0)  # A unit cannot have zero in the expression
 
             u = cls()

src/Python/wcon/wcon_data.py

@@ -116,8 +116,8 @@ def df_upsert(src, dest):
                         src.columns.isin(dest.columns)]
 
             # Sort our slices so they will be lined up for comparison
-            dest_sliced.sort_index(inplace=True)
-            src_sliced.sort_index(inplace=True)
+            dest_sliced.sort_index(axis=1, inplace=True)
+            src_sliced.sort_index(axis=1, inplace=True)
 
             # Obtain a mask of the conflicts in the current segment
             # as compared with all previously loaded data.  That is:
@@ -190,10 +190,10 @@ def convert_origin(df):
 
             if offset in cur_worm.columns.get_level_values(0):
                 # Consider offset as 0 if not available in a certain frame
-                ox_column = cur_worm.loc[:, (offset)].fillna(0)
+                ox_column = cur_worm.loc[:, (offset)].fillna(0).astype('float64')
 
                 # Shift our 'x' values by offset
-                all_x_columns = cur_worm.loc[:, (coord)]
+                all_x_columns = cur_worm.loc[:, (coord)].fillna(0).astype('float64')
                 ox_affine_change = (np.array(ox_column) *
                                     np.ones(all_x_columns.shape))
                 all_x_columns += ox_affine_change
@@ -218,13 +218,16 @@ def convert_origin(df):
 
                 # Now reset our 'ox' values to zero.
                 if offset in cur_worm.columns.get_level_values(0):
-                    df.loc[:, (worm_id, offset)] = np.zeros(ox_column.shape)
+                    df.loc[:, (worm_id, offset)] = np.zeros(ox_column.shape, dtype=pd.Int64Dtype)
+            else:
+                all_x_columns = cur_worm.loc[:, (coord)].fillna(0).astype('float64')
+                df.loc[:, (worm_id, coord)] = all_x_columns.values
 
     # Drop the offset columns entirely from the dataframe.
     # This is so DataFrames with and without offsets
     # will show as comparing identically.
     for offset_key in offset_keys:
-        df.drop(offset_key, axis=1, level='key', inplace=True)
+        df.drop(offset_key, axis=1, level='key', inplace=True, errors='ignore')
 
     # Because of a known issue in Pandas
     # (https://github.com/pydata/pandas/issues/2770), the dropped columns
@@ -389,7 +392,7 @@ def _obtain_time_series_data_frame(time_series_data):
             for i in range(len(cur_timeframes)):
                 data_segment[k][i] = (
                     data_segment[k][i] +
-                    [np.NaN] * (max_aspect_size - len(data_segment[k][i])))
+                    [np.nan] * (max_aspect_size - len(data_segment[k][i])))
 
         num_timeframes = len(cur_timeframes)
 
@@ -402,7 +405,7 @@ def _obtain_time_series_data_frame(time_series_data):
         cur_df = pd.DataFrame(cur_data, columns=cur_columns)
 
         cur_df.index = cur_timeframes
-        cur_df.index.names = 't'
+        cur_df.index.name = 't'
 
         # We want the index (time) to be in order.
         cur_df.sort_index(axis=0, inplace=True)
@@ -466,7 +469,7 @@ def _obtain_time_series_data_frame(time_series_data):
         with warnings.catch_warnings():
             warnings.filterwarnings(action="ignore", category=FutureWarning)
             df_odict[worm_id] = \
-                df_odict[worm_id].convert_objects(convert_numeric=True)
+                df_odict[worm_id].convert_dtypes(convert_floating=True)
 
         # If 'head' or 'ventral' is NaN, we must specify '?' since
         # otherwise, when saving this object, to specify "no value" we would

src/Python/wcon/wcon_parser.py

@@ -396,10 +396,10 @@ def to_canon(self):
                 try:
                     # Apply across all worm ids and all aspects
                     mu_slice = \
-                        w._data[worm_id].loc[:, idx[:, data_key, :]].copy()
+                        w._data[worm_id].loc[:, idx[:, data_key, :]].copy().astype('float64')
 
                     w._data[worm_id].loc[:, idx[:, data_key, :]] = \
-                        mu_slice.applymap(mu.to_canon)
+                        mu_slice.map(mu.to_canon)
                 except KeyError:
                     # Just ignore cases where there are "units" entries but no
                     # corresponding data
@@ -822,7 +822,7 @@ def pd_equals(df1, df2):
         return False
 
     try:
-        pd.util.testing.assert_frame_equal(df1, df2)
+        pd.testing.assert_frame_equal(df1, df2)
     except AssertionError:
         return False
 

src/scala/project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.10.11

tests/minimax.wcon

@@ -24,7 +24,7 @@
         "x":"mm", 
         "y":"m",
         "ox":"mm",
-	"oy":"mm",
+	    "oy":"mm",
         "speed":"mm/s",
         "curvature":"1/mm",
         "width":"mm",

wcon_schema.json

@@ -1,5 +1,5 @@
 {
-    "$schema": "http://json-schema.org/schema",
+    "$schema": "http://json-schema.org/draft-07/schema",
     "title": "Worm tracker Commons Object Notation (WCON)",
     "description": "A text-based data interchange format for *C. elegans* trackers. It is a constrained subset of JSON. It is designed to be both human and machine readable and to facilitate data exchange and inter-operability for worm tracking data that is independent of platform and the language used for implementation.",
     "type": "object",