Merge pull request #556 from USACE/dss-py

switch to dsspy from tiffdss

Author: Enovotny

async_packager/Dockerfile

@@ -1,16 +1,24 @@
 # Packager
-FROM ghcr.io/osgeo/gdal:ubuntu-full-3.7.0 AS base
+ARG GDAL_TAG=ubuntu-full-3.9.1
+
+
+FROM ghcr.io/osgeo/gdal:${GDAL_TAG} AS builder
 
 # force stdout and stderr to be unbuffered setting to non-empty
 ENV PYTHONUNBUFFERED=1
 
+#GDAL
 ENV GDAL_DATA=/usr/share/gdal
 ENV GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR
 
-ENV PACKAGE=async_packager
+# packager
+ENV USERNAME=packager
+ENV PACKAGER=/opt/packager
+ENV PACKAGER_VENV=${PACKAGER}/venv
+
+ENV PATH=${PACKAGER_VENV}/bin:$PATH
 
-ENV TIFFDSS_VERSION=v0.2.2
-ENV TIFFDSS=tiffdss_ubuntu-latest
+#ENV PACKAGE=async_packager
 
 # solving possible issue of ATP behind a proxy
 RUN cat <<EOF > /etc/apt/apt.conf.d/99proxy
@@ -19,39 +27,30 @@ Acquire::http::No-Cache true;
 Acquire::BrokenProxy true;
 EOF
 
-# update and install other dependencies
-RUN apt-get -y update && \
-    apt-get -y upgrade && \
-    apt-get install -y python3-pip curl && \
-    rm -rf /var/lib/apt/lists/* && \
-    pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir --upgrade setuptools && \
-    pip install --no-cache-dir --upgrade wheel && \
-    pip install --no-cache-dir --upgrade pillow
-    # pip install --no-cache-dir --upgrade numpy
-
 
-RUN mkdir -p /app/${PACKAGE}
+# update and install other dependencies
+RUN apt-get -y update \
+    # && apt-get -y upgrade \
+    && apt-get install -y python3.12-venv \
+    && apt-get remove python3-pil -y \
+    && apt clean
 
-WORKDIR /app/${PACKAGE}
+# install package in venv
+COPY . ${PACKAGER}/pkg
 
-COPY . .
+RUN python3 -m venv --system-site-packages ${PACKAGER_VENV} \
+    && . activate \
+    && pip install ${PACKAGER}/pkg/ \
+    && pip install hecdss
 
-# pip install the package and upgrade outdated packages
-RUN pip3 install /app/${PACKAGE}/
+RUN groupadd -r ${USERNAME} && \
+    useradd -g ${USERNAME} -ms /bin/bash ${USERNAME}
 
-# get libtiffdss.so
-RUN curl -L https://github.com/HydrologicEngineeringCenter/tiffdss/releases/download/${TIFFDSS_VERSION}/${TIFFDSS}_${TIFFDSS_VERSION}.tar.gz \
-    --output ${TIFFDSS}_${TIFFDSS_VERSION}.tar.gz && \
-    tar -zxvf ${TIFFDSS}_${TIFFDSS_VERSION}.tar.gz && \
-    mv libtiffdss.so /usr/lib/ && \
-    mv tiffdss /usr/local/bin/ && \
-    rm -f ${TIFFDSS}_${TIFFDSS_VERSION}.tar.gz tiffdss *.o
+WORKDIR ${PACKAGER}
 
-RUN useradd appuser && \
-    mkdir -p -m 775 docs && \
-    chown appuser:appuser docs
+# packager and run it
+COPY ./packager.py ./
 
-USER appuser
+USER ${USERNAME}:${USERNAME}
 
-CMD ["./packager.py"]
+ENTRYPOINT ["python3", "packager.py"]

async_packager/setup.cfg

@@ -30,6 +30,7 @@ install_requires =
     gdal-utils
     psycopg2-binary
     codetiming
+    hecdss
 include_package_data = True
 
 [options.packages.find]

…/src/cumulus_packager/heclib/__init__.py …src/cumulus_packager/dssutil/__init__.pyasync_packager/src/cumulus_packager/heclib/__init__.py renamed to async_packager/src/cumulus_packager/dssutil/__init__.py async_packager/src/cumulus_packager/heclib/__init__.py renamed to async_packager/src/cumulus_packager/dssutil/__init__.py

@@ -1,28 +1,11 @@
-"""Module 'heclib'
+"""Module 'dssutil' for handling DSS grids
 
 Enumerations, constants and functions
 
 """
 
-from ctypes import (
-    CDLL,
-    POINTER,
-    LibraryLoader,
-    Structure,
-    c_char_p,
-    c_float,
-    c_int,
-    c_void_p,
-    pointer,
-)
 from enum import Enum, auto
 
-import numpy
-
-# libtiffdss.so is compiled and put in /usr/lib during image creation
-tiffdss = LibraryLoader(CDLL).LoadLibrary("libtiffdss.so")
-"""ctypes.CDLL: tiff to dss shared object"""
-
 FLOAT_MAX = 3.40282347e38
 UNDEFINED = -FLOAT_MAX
 
@@ -159,88 +142,4 @@ class TimeZone(Enum):
     HST = auto()
 
 
-time_zone = {i.name: i.value for i in TimeZone}
-
-
-class zStructSpatialGrid(Structure):
-    _fields_ = [
-        ("structType", c_int),
-        ("pathname", c_char_p),
-        ("_structVersion", c_int),
-        ("_type", c_int),
-        ("_version", c_int),
-        ("_dataUnits", c_char_p),
-        ("_dataType", c_int),
-        ("_dataSource", c_char_p),
-        ("_lowerLeftCellX", c_int),
-        ("_lowerLeftCellY", c_int),
-        ("_numberOfCellsX", c_int),
-        ("_numberOfCellsY", c_int),
-        ("_cellSize", c_float),
-        ("_compressionMethod", c_int),
-        ("_sizeofCompressedElements", c_int),
-        ("_compressionParameters", c_void_p),
-        ("_srsName", c_char_p),
-        ("_srsDefinitionType", c_int),
-        ("_srsDefinition", c_char_p),
-        ("_xCoordOfGridCellZero", c_float),
-        ("_yCoordOfGridCellZero", c_float),
-        ("_nullValue", c_float),
-        ("_timeZoneID", c_char_p),
-        ("_timeZoneRawOffset", c_int),
-        ("_isInterval", c_int),
-        ("_isTimeStamped", c_int),
-        ("_numberOfRanges", c_int),
-        ("_storageDataType", c_int),
-        ("_maxDataValue", c_void_p),
-        ("_minDataValue", c_void_p),
-        ("_meanDataValue", c_void_p),
-        ("_rangeLimitTable", c_void_p),
-        ("_numberEqualOrExceedingRangeLimit", c_int),
-        ("_data", c_void_p),
-    ]
-
-    def __init__(self, *args, **kw):
-        self._nullValue = UNDEFINED
-        super().__init__(*args, **kw)
-
-
-def zwrite_record(
-    dssfilename: str,
-    gridStructStore: zStructSpatialGrid,
-    data_flat: numpy,
-):
-    """Write the data array to DSS record using the 'writeRecord' C function
-
-    Parameters
-    ----------
-    dssfilename : str
-        DSS file name and path
-    gridStructStore : zStructSpatialGrid
-        ctypes structure
-    data_flat : numpy
-        1D numpy array
-    gridStats : GridStats
-        ctypes structure
-
-    Returns
-    -------
-    int
-        Response from the C function
-    """
-    ND_POINTER_1 = numpy.ctypeslib.ndpointer(dtype=numpy.float32, ndim=1, flags="C")
-
-    tiffdss.writeRecord_External.argtypes = (
-        c_char_p,
-        POINTER(zStructSpatialGrid),
-        ND_POINTER_1,
-    )
-    tiffdss.writeRecord_External.restype = c_int
-
-    res = tiffdss.writeRecord_External(
-        c_char_p(dssfilename.encode()),
-        pointer(gridStructStore),
-        data_flat,
-    )
-
-    return res
+time_zone = {i.name: i.value for i in TimeZone}