diff --git a/.github/workflows/cwbi-prod-build-push-api.yml b/.github/workflows/cwbi-prod-build-push-api.yml new file mode 100644 index 00000000..8ba2f07d --- /dev/null +++ b/.github/workflows/cwbi-prod-build-push-api.yml @@ -0,0 +1,54 @@ +# This is a basic workflow to help you get started with Actions +name: Build API Image, Push to Prod + +# Controls when the action will run. Invokes the workflow on push events but only for the main branch +on: + push: + branches: [cwbi-prod] + paths: + - .github/workflows/cwbi-prod-build-push-api.yml + - 'api/**' + workflow_dispatch: + +env: + AWS_REGION: aws-us-gov #Change to reflect your Region + +# Permission can be added at job level or workflow level +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout +jobs: + Build-Push-API-to-Prod: + runs-on: ubuntu-latest + steps: + - name: Git clone the repository + uses: actions/checkout@v4 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-gov-west-1 + role-to-assume: arn:aws-us-gov:iam::648157167324:role/github-actions-ecr-cumulus + output-credentials: true + # Hello from AWS: WhoAmI + - name: Sts GetCallerIdentity + run: | + aws sts get-caller-identity + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + mask-password: 'true' + - name: Build Image; Push to ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: cumulus-api + IMAGE_TAG: latest + run: | + docker build api \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:prod + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker push $ECR_REGISTRY/$ECR_REPOSITORY:prod + - name: ECR Logout + if: always() + run: docker logout ${{ steps.login-ecr.outputs.registry }} diff --git a/.github/workflows/cwbi-prod-build-push-geoproc.yml b/.github/workflows/cwbi-prod-build-push-geoproc.yml new file mode 100644 index 00000000..88313350 --- /dev/null +++ b/.github/workflows/cwbi-prod-build-push-geoproc.yml @@ -0,0 +1,54 @@ +# This is a basic workflow to help you get started with Actions +name: Build Geoprocess Image, Push to Prod + +# Controls when the action will run. Invokes the workflow on push events but only for the main branch +on: + push: + branches: [cwbi-prod] + paths: + - .github/workflows/cwbi-prod-build-push-geoproc.yml + - 'async_geoprocess/**' + workflow_dispatch: + +env: + AWS_REGION: aws-us-gov #Change to reflect your Region + +# Permission can be added at job level or workflow level +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout +jobs: + Build-Push-GeoProc-to-Prod: + runs-on: ubuntu-latest + steps: + - name: Git clone the repository + uses: actions/checkout@v4 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-gov-west-1 + role-to-assume: arn:aws-us-gov:iam::648157167324:role/github-actions-ecr-cumulus + output-credentials: true + # Hello from AWS: WhoAmI + - name: Sts GetCallerIdentity + run: | + aws sts get-caller-identity + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + mask-password: 'true' + - name: Build Image; Push to ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: cumulus-geoprocess + IMAGE_TAG: latest + run: | + docker build --build-arg GEOPROC_PACKAGE=main:geoproc async_geoprocess \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:prod + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker push $ECR_REGISTRY/$ECR_REPOSITORY:prod + - name: ECR Logout + if: always() + run: docker logout ${{ steps.login-ecr.outputs.registry }} diff --git a/.github/workflows/cwbi-prod-build-push-listener.yml b/.github/workflows/cwbi-prod-build-push-listener.yml new file mode 100644 index 00000000..1f9f7964 --- /dev/null +++ b/.github/workflows/cwbi-prod-build-push-listener.yml @@ -0,0 +1,54 @@ +#This is a basic workflow to help you get started with Actions +name: Build Listener Image, Push to Prod + +# Controls when the action will run. Invokes the workflow on push events but only for the main branch +on: + push: + branches: [cwbi-prod] + paths: + - .github/workflows/cwbi-prod-build-push-listener.yml + - 'async_listener/**' + workflow_dispatch: + +env: + AWS_REGION: aws-us-gov #Change to reflect your Region + +# Permission can be added at job level or workflow level +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout +jobs: + Build-Push-Listener-to-Prod: + runs-on: ubuntu-latest + steps: + - name: Git clone the repository + uses: actions/checkout@v4 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-gov-west-1 + role-to-assume: arn:aws-us-gov:iam::648157167324:role/github-actions-ecr-cumulus + output-credentials: true + # Hello from AWS: WhoAmI + - name: Sts GetCallerIdentity + run: | + aws sts get-caller-identity + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + mask-password: 'true' + - name: Build Image; Push to ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: cumulus-listener + IMAGE_TAG: latest + run: | + docker build async_listener \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:prod + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker push $ECR_REGISTRY/$ECR_REPOSITORY:prod + - name: ECR Logout + if: always() + run: docker logout ${{ steps.login-ecr.outputs.registry }} diff --git a/.github/workflows/cwbi-prod-build-push-migration.yml b/.github/workflows/cwbi-prod-build-push-migration.yml new file mode 100644 index 00000000..248b200a --- /dev/null +++ b/.github/workflows/cwbi-prod-build-push-migration.yml @@ -0,0 +1,54 @@ +# This is a basic workflow to help you get started with Actions +name: Build Migration Image, Push to Prod + +# Controls when the action will run. Invokes the workflow on push events but only for the main branch +on: + push: + branches: [cwbi-prod] + paths: + - .github/workflows/cwbi-prod-build-push-migration.yml + - 'sql/**' + workflow_dispatch: + +env: + AWS_REGION: aws-us-gov #Change to reflect your Region + +# Permission can be added at job level or workflow level +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout +jobs: + Build-Push-Migration-to-Prod: + runs-on: ubuntu-latest + steps: + - name: Git clone the repository + uses: actions/checkout@v4 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-gov-west-1 + role-to-assume: arn:aws-us-gov:iam::648157167324:role/github-actions-ecr-cumulus + output-credentials: true + # Hello from AWS: WhoAmI + - name: Sts GetCallerIdentity + run: | + aws sts get-caller-identity + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + mask-password: 'true' + - name: Build Image; Push to ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: cumulus-migration + IMAGE_TAG: latest + run: | + docker build sql \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:prod + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker push $ECR_REGISTRY/$ECR_REPOSITORY:prod + - name: ECR Logout + if: always() + run: docker logout ${{ steps.login-ecr.outputs.registry }} diff --git a/.github/workflows/cwbi-prod-build-push-packager.yml b/.github/workflows/cwbi-prod-build-push-packager.yml new file mode 100644 index 00000000..1d1d34d6 --- /dev/null +++ b/.github/workflows/cwbi-prod-build-push-packager.yml @@ -0,0 +1,54 @@ +#This is a basic workflow to help you get started with Actions +name: Build Packager Image, Push to Prod + +# Controls when the action will run. Invokes the workflow on push events but only for the main branch +on: + push: + branches: [cwbi-prod] + paths: + - .github/workflows/cwbi-prod-build-push-packager.yml + - 'async_packager/**' + workflow_dispatch: + +env: + AWS_REGION: aws-us-gov #Change to reflect your Region + +# Permission can be added at job level or workflow level +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout +jobs: + Build-Push-Packager-to-Prod: + runs-on: ubuntu-latest + steps: + - name: Git clone the repository + uses: actions/checkout@v4 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-gov-west-1 + role-to-assume: arn:aws-us-gov:iam::648157167324:role/github-actions-ecr-cumulus + output-credentials: true + # Hello from AWS: WhoAmI + - name: Sts GetCallerIdentity + run: | + aws sts get-caller-identity + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + mask-password: 'true' + - name: Build Image; Push to ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: cumulus-packager + IMAGE_TAG: latest + run: | + docker build async_packager \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:prod + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker push $ECR_REGISTRY/$ECR_REPOSITORY:prod + - name: ECR Logout + if: always() + run: docker logout ${{ steps.login-ecr.outputs.registry }} diff --git a/.github/workflows/cwbi-prod-build-push-pg_featureserv.yml b/.github/workflows/cwbi-prod-build-push-pg_featureserv.yml new file mode 100644 index 00000000..58f7fe8b --- /dev/null +++ b/.github/workflows/cwbi-prod-build-push-pg_featureserv.yml @@ -0,0 +1,54 @@ +#This is a basic workflow to help you get started with Actions +name: Build pg_featureserv Image, Push to Prod + +# Controls when the action will run. Invokes the workflow on push events but only for the main branch +on: + push: + branches: [cwbi-prod] + paths: + - .github/workflows/cwbi-prod-build-push-pg_featureserv.yml + - 'pg_featureserv/**' + workflow_dispatch: + +env: + AWS_REGION: aws-us-gov #Change to reflect your Region + +# Permission can be added at job level or workflow level +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout +jobs: + Build-Push-pg_featureserv-to-Prod: + runs-on: ubuntu-latest + steps: + - name: Git clone the repository + uses: actions/checkout@v4 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-gov-west-1 + role-to-assume: arn:aws-us-gov:iam::648157167324:role/github-actions-ecr-cumulus + output-credentials: true + # Hello from AWS: WhoAmI + - name: Sts GetCallerIdentity + run: | + aws sts get-caller-identity + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + mask-password: 'true' + - name: Build Image; Push to ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: cumulus-pg_featureserv + IMAGE_TAG: latest + run: | + docker build pg_featureserv \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:prod + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker push $ECR_REGISTRY/$ECR_REPOSITORY:prod + - name: ECR Logout + if: always() + run: docker logout ${{ steps.login-ecr.outputs.registry }} diff --git a/async_packager/Dockerfile b/async_packager/Dockerfile index 4a59fc3b..8566ac3a 100644 --- a/async_packager/Dockerfile +++ b/async_packager/Dockerfile @@ -1,16 +1,24 @@ # Packager -FROM ghcr.io/osgeo/gdal:ubuntu-full-3.7.0 AS base +ARG GDAL_TAG=ubuntu-full-3.9.1 + + +FROM ghcr.io/osgeo/gdal:${GDAL_TAG} AS builder # force stdout and stderr to be unbuffered setting to non-empty ENV PYTHONUNBUFFERED=1 +#GDAL ENV GDAL_DATA=/usr/share/gdal ENV GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR -ENV PACKAGE=async_packager +# packager +ENV USERNAME=packager +ENV PACKAGER=/opt/packager +ENV PACKAGER_VENV=${PACKAGER}/venv + +ENV PATH=${PACKAGER_VENV}/bin:$PATH -ENV TIFFDSS_VERSION=v0.2.2 -ENV TIFFDSS=tiffdss_ubuntu-latest +#ENV PACKAGE=async_packager # solving possible issue of ATP behind a proxy RUN cat < /etc/apt/apt.conf.d/99proxy @@ -19,39 +27,30 @@ Acquire::http::No-Cache true; Acquire::BrokenProxy true; EOF -# update and install other dependencies -RUN apt-get -y update && \ - apt-get -y upgrade && \ - apt-get install -y python3-pip curl && \ - rm -rf /var/lib/apt/lists/* && \ - pip install --no-cache-dir --upgrade pip && \ - pip install --no-cache-dir --upgrade setuptools && \ - pip install --no-cache-dir --upgrade wheel && \ - pip install --no-cache-dir --upgrade pillow - # pip install --no-cache-dir --upgrade numpy - -RUN mkdir -p /app/${PACKAGE} +# update and install other dependencies +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get install -y python3.12-venv \ + && apt-get remove python3-pil -y \ + && apt clean -WORKDIR /app/${PACKAGE} +# install package in venv +COPY . ${PACKAGER}/pkg -COPY . . +RUN python3 -m venv --system-site-packages ${PACKAGER_VENV} \ + && . activate \ + && pip install ${PACKAGER}/pkg/ \ + && pip install hecdss -# pip install the package and upgrade outdated packages -RUN pip3 install /app/${PACKAGE}/ +RUN groupadd -r ${USERNAME} && \ + useradd -g ${USERNAME} -ms /bin/bash ${USERNAME} -# get libtiffdss.so -RUN curl -L https://github.com/HydrologicEngineeringCenter/tiffdss/releases/download/${TIFFDSS_VERSION}/${TIFFDSS}_${TIFFDSS_VERSION}.tar.gz \ - --output ${TIFFDSS}_${TIFFDSS_VERSION}.tar.gz && \ - tar -zxvf ${TIFFDSS}_${TIFFDSS_VERSION}.tar.gz && \ - mv libtiffdss.so /usr/lib/ && \ - mv tiffdss /usr/local/bin/ && \ - rm -f ${TIFFDSS}_${TIFFDSS_VERSION}.tar.gz tiffdss *.o +WORKDIR ${PACKAGER} -RUN useradd appuser && \ - mkdir -p -m 775 docs && \ - chown appuser:appuser docs +# packager and run it +COPY ./packager.py ./ -USER appuser +USER ${USERNAME}:${USERNAME} -CMD ["./packager.py"] +ENTRYPOINT ["python3", "packager.py"] diff --git a/async_packager/setup.cfg b/async_packager/setup.cfg index dec078bd..2756347a 100644 --- a/async_packager/setup.cfg +++ b/async_packager/setup.cfg @@ -30,6 +30,7 @@ install_requires = gdal-utils psycopg2-binary codetiming + hecdss include_package_data = True [options.packages.find] diff --git a/async_packager/src/cumulus_packager/heclib/__init__.py b/async_packager/src/cumulus_packager/dssutil/__init__.py similarity index 56% rename from async_packager/src/cumulus_packager/heclib/__init__.py rename to async_packager/src/cumulus_packager/dssutil/__init__.py index 30db715e..a23ef98d 100644 --- a/async_packager/src/cumulus_packager/heclib/__init__.py +++ b/async_packager/src/cumulus_packager/dssutil/__init__.py @@ -1,28 +1,11 @@ -"""Module 'heclib' +"""Module 'dssutil' for handling DSS grids Enumerations, constants and functions """ -from ctypes import ( - CDLL, - POINTER, - LibraryLoader, - Structure, - c_char_p, - c_float, - c_int, - c_void_p, - pointer, -) from enum import Enum, auto -import numpy - -# libtiffdss.so is compiled and put in /usr/lib during image creation -tiffdss = LibraryLoader(CDLL).LoadLibrary("libtiffdss.so") -"""ctypes.CDLL: tiff to dss shared object""" - FLOAT_MAX = 3.40282347e38 UNDEFINED = -FLOAT_MAX @@ -159,88 +142,4 @@ class TimeZone(Enum): HST = auto() -time_zone = {i.name: i.value for i in TimeZone} - - -class zStructSpatialGrid(Structure): - _fields_ = [ - ("structType", c_int), - ("pathname", c_char_p), - ("_structVersion", c_int), - ("_type", c_int), - ("_version", c_int), - ("_dataUnits", c_char_p), - ("_dataType", c_int), - ("_dataSource", c_char_p), - ("_lowerLeftCellX", c_int), - ("_lowerLeftCellY", c_int), - ("_numberOfCellsX", c_int), - ("_numberOfCellsY", c_int), - ("_cellSize", c_float), - ("_compressionMethod", c_int), - ("_sizeofCompressedElements", c_int), - ("_compressionParameters", c_void_p), - ("_srsName", c_char_p), - ("_srsDefinitionType", c_int), - ("_srsDefinition", c_char_p), - ("_xCoordOfGridCellZero", c_float), - ("_yCoordOfGridCellZero", c_float), - ("_nullValue", c_float), - ("_timeZoneID", c_char_p), - ("_timeZoneRawOffset", c_int), - ("_isInterval", c_int), - ("_isTimeStamped", c_int), - ("_numberOfRanges", c_int), - ("_storageDataType", c_int), - ("_maxDataValue", c_void_p), - ("_minDataValue", c_void_p), - ("_meanDataValue", c_void_p), - ("_rangeLimitTable", c_void_p), - ("_numberEqualOrExceedingRangeLimit", c_int), - ("_data", c_void_p), - ] - - def __init__(self, *args, **kw): - self._nullValue = UNDEFINED - super().__init__(*args, **kw) - - -def zwrite_record( - dssfilename: str, - gridStructStore: zStructSpatialGrid, - data_flat: numpy, -): - """Write the data array to DSS record using the 'writeRecord' C function - - Parameters - ---------- - dssfilename : str - DSS file name and path - gridStructStore : zStructSpatialGrid - ctypes structure - data_flat : numpy - 1D numpy array - gridStats : GridStats - ctypes structure - - Returns - ------- - int - Response from the C function - """ - ND_POINTER_1 = numpy.ctypeslib.ndpointer(dtype=numpy.float32, ndim=1, flags="C") - - tiffdss.writeRecord_External.argtypes = ( - c_char_p, - POINTER(zStructSpatialGrid), - ND_POINTER_1, - ) - tiffdss.writeRecord_External.restype = c_int - - res = tiffdss.writeRecord_External( - c_char_p(dssfilename.encode()), - pointer(gridStructStore), - data_flat, - ) - - return res +time_zone = {i.name: i.value for i in TimeZone} \ No newline at end of file diff --git a/async_packager/src/cumulus_packager/writers/dss7.py b/async_packager/src/cumulus_packager/writers/dss7.py index 13f66564..89e29846 100644 --- a/async_packager/src/cumulus_packager/writers/dss7.py +++ b/async_packager/src/cumulus_packager/writers/dss7.py @@ -1,22 +1,22 @@ -"""DSS7 package writer - -""" +"""DSS7 package writer""" import json -import os import sys from collections import namedtuple -from ctypes import c_char_p, c_float, c_int from pathlib import Path import numpy import pyplugs from codetiming import Timer -from cumulus_packager import heclib, logger +from cumulus_packager import dssutil, logger from cumulus_packager.configurations import PACKAGER_UPDATE_INTERVAL from cumulus_packager.packager.handler import PACKAGE_STATUS, update_status from osgeo import gdal, osr +from hecdss import HecDss +from hecdss.gridded_data import GriddedData +from importlib.metadata import version, PackageNotFoundError + gdal.UseExceptions() @@ -52,8 +52,12 @@ def writer( FQPN to dss file """ + try: + pkg_version = version("hecdss") + except Exception: + pkg_version = "unknown" logger.info( - f"Write Records to DSS using TiffDss {os.getenv('TIFFDSS_VERSION')}", + f"Write Records to DSS using hecdss {pkg_version}", ) # convert the strings back to json objects; needed for pyplugs @@ -75,132 +79,133 @@ def writer( grid_type = 430 else: grid_type_name = "SHG" - grid_type = heclib.dss_grid_type[grid_type_name] + grid_type = dssutil.dss_grid_type[grid_type_name] logger.info( f"grid type name {grid_type_name}", ) - zcompression = heclib.compression_method["ZLIB_COMPRESSION"] - srs_definition = heclib.spatial_reference_definition[grid_type_name] + srs_definition = dssutil.spatial_reference_definition[grid_type_name] tz_name = "GMT" - tz_offset = heclib.time_zone[tz_name] + tz_offset = dssutil.time_zone[tz_name] is_interval = 1 dssfilename = Path(dst).joinpath(id).with_suffix(".dss").as_posix() + with HecDss(dssfilename) as dss: + for idx, tif in enumerate(src): + TifCfg = namedtuple("TifCfg", tif)(**tif) + dsspathname = f"/{grid_type_name}/{_extent_name}/{TifCfg.dss_cpart}/{TifCfg.dss_dpart}/{TifCfg.dss_epart}/{TifCfg.dss_fpart}/" - for idx, tif in enumerate(src): - TifCfg = namedtuple("TifCfg", tif)(**tif) - dsspathname = f"/{grid_type_name}/{_extent_name}/{TifCfg.dss_cpart}/{TifCfg.dss_dpart}/{TifCfg.dss_epart}/{TifCfg.dss_fpart}/" - - try: - data_type = heclib.data_type[TifCfg.dss_datatype] - ds = gdal.Open(f"/vsis3_streaming/{TifCfg.bucket}/{TifCfg.key}") - - # GDAL Warp the Tiff to what we need for DSS - filename_ = Path(TifCfg.key).name - mem_raster = f"/vsimem/{filename_}" - warp_ds = gdal.Warp( - mem_raster, - ds, - format="GTiff", - outputBounds=_bbox, - xRes=cellsize, - yRes=cellsize, - targetAlignedPixels=True, - dstSRS=destination_srs.ExportToWkt(), - resampleAlg="bilinear", - copyMetadata=False, - ) - - # Read data into 1D array - raster = warp_ds.GetRasterBand(1) - nodata = raster.GetNoDataValue() - - data = raster.ReadAsArray(resample_alg=gdal.gdalconst.GRIORA_Bilinear) - # Flip the dataset up/down because tif and dss have different origins - data = numpy.flipud(data) - data_flat = data.flatten() - - # GeoTransforma and lower X Y - xsize = warp_ds.RasterXSize - ysize = warp_ds.RasterYSize - adfGeoTransform = warp_ds.GetGeoTransform() - llx = int(adfGeoTransform[0] / adfGeoTransform[1]) - lly = int( - (adfGeoTransform[5] * ysize + adfGeoTransform[3]) / adfGeoTransform[1] - ) - - spatialGridStruct = heclib.zStructSpatialGrid() - spatialGridStruct.pathname = c_char_p(dsspathname.encode()) - spatialGridStruct._structVersion = c_int(-100) - spatialGridStruct._type = c_int(grid_type) - spatialGridStruct._version = c_int(1) - spatialGridStruct._dataUnits = c_char_p(str.encode(TifCfg.dss_unit)) - spatialGridStruct._dataType = c_int(data_type) - spatialGridStruct._dataSource = c_char_p("INTERNAL".encode()) - spatialGridStruct._lowerLeftCellX = c_int(llx) - spatialGridStruct._lowerLeftCellY = c_int(lly) - spatialGridStruct._numberOfCellsX = c_int(xsize) - spatialGridStruct._numberOfCellsY = c_int(ysize) - spatialGridStruct._cellSize = c_float(cellsize) - spatialGridStruct._compressionMethod = c_int(zcompression) - spatialGridStruct._srsName = c_char_p(grid_type_name.encode()) - spatialGridStruct._srsDefinitionType = c_int(1) - spatialGridStruct._srsDefinition = c_char_p(srs_definition.encode()) - spatialGridStruct._xCoordOfGridCellZero = c_float(0) - spatialGridStruct._yCoordOfGridCellZero = c_float(0) - if nodata is not None: - spatialGridStruct._nullValue = c_float(nodata) - spatialGridStruct._timeZoneID = c_char_p(tz_name.encode()) - spatialGridStruct._timeZoneRawOffset = c_int(tz_offset) - spatialGridStruct._isInterval = c_int(is_interval) - spatialGridStruct._isTimeStamped = c_int(1) - - # Call heclib.zwrite_record() in different process space to release memory after each iteration - t = Timer(name="accumuluated", logger=None) - t.start() - result = heclib.zwrite_record( - dssfilename, spatialGridStruct, data_flat.astype(numpy.float32) - ) - elapsed_time = t.stop() - logger.debug(f'Processed "{TifCfg.key}" in {elapsed_time:.4f} seconds') - if result != 0: - logger.info(f'TiffDss write record failed for "{TifCfg.key}": {result}') - - _progress = int(((idx + 1) / gridcount) * 100) - # Update progress at predefined interval - if idx % PACKAGER_UPDATE_INTERVAL == 0 or idx == gridcount - 1: - update_status( - id=id, status_id=PACKAGE_STATUS["INITIATED"], progress=_progress - ) - if _progress % PACKAGER_UPDATE_INTERVAL == 0: - logger.info(f'Download ID "{id}" progress: {_progress}%') - - except (RuntimeError, Exception): - exc_type, exc_value, exc_traceback = sys.exc_info() - traceback_details = { - "filename": Path(exc_traceback.tb_frame.f_code.co_filename).name, - "line number": exc_traceback.tb_lineno, - "method": exc_traceback.tb_frame.f_code.co_name, - "type": exc_type.__name__, - "message": exc_value, - } - logger.error(traceback_details) - - continue - - finally: - data = None - raster = None - warp_ds = None - # Try to unlink and remove the memory raster object for each source file processed try: - gdal.Unlink(mem_raster) - mem_raster = None - except Exception as ex: - logger.debug("vismem unlink exception: %s", ex) - ds = None - spatialGridStruct = None + data_type = dssutil.data_type[TifCfg.dss_datatype] + ds = gdal.Open(f"/vsis3_streaming/{TifCfg.bucket}/{TifCfg.key}") + + # GDAL Warp the Tiff to what we need for DSS + filename_ = Path(TifCfg.key).name + mem_raster = f"/vsimem/{filename_}" + warp_ds = gdal.Warp( + mem_raster, + ds, + format="GTiff", + outputBounds=_bbox, + xRes=cellsize, + yRes=cellsize, + targetAlignedPixels=True, + dstSRS=destination_srs.ExportToWkt(), + resampleAlg="bilinear", + copyMetadata=False, + ) + + # Read data into 1D array + raster = warp_ds.GetRasterBand(1) + nodata = raster.GetNoDataValue() + + data = raster.ReadAsArray(resample_alg=gdal.gdalconst.GRIORA_Bilinear) + # Flip the dataset up/down because tif and dss have different origins + data = numpy.flipud(data) + DSS_UNDEFINED_VALUE = -3.4028234663852886e+38 + data[data == nodata] = numpy.nan # Replace nodata with NaN for processing + # GeoTransforma and lower X Y + xsize = warp_ds.RasterXSize + ysize = warp_ds.RasterYSize + adfGeoTransform = warp_ds.GetGeoTransform() + llx = int(adfGeoTransform[0] / adfGeoTransform[1]) + lly = int( + (adfGeoTransform[5] * ysize + adfGeoTransform[3]) + / adfGeoTransform[1] + ) + + gd = GriddedData.create( + path=dsspathname, + type=grid_type, + dataType=data_type, + lowerLeftCellX=llx, + lowerLeftCellY=lly, + numberOfCellsX=xsize, + numberOfCellsY=ysize, + srsName=grid_type_name, + srsDefinitionType=1, + srsDefinition=srs_definition, + dataUnits=TifCfg.dss_unit, + dataSource="INTERNAL", + timeZoneID=tz_name, + timeZoneRawOffset=tz_offset, + isInterval=is_interval, + isTimeStamped=1, + cellSize=cellsize, + xCoordOfGridCellZero=0.0, + yCoordOfGridCellZero=0.0, + nullValue=DSS_UNDEFINED_VALUE, + data=data, + ) + + # Call HecDss.put() in different process space to release memory after each iteration + t = Timer(name="accumuluated", logger=None) + t.start() + result = dss.put(gd) + elapsed_time = t.stop() + logger.debug( + f'DSS put Processed "{TifCfg.key}" in {elapsed_time:.4f} seconds' + ) + if result != 0: + logger.info( + f'HEC-DSS-PY write record failed for "{TifCfg.key}": {result}' + ) + + _progress = int(((idx + 1) / gridcount) * 100) + # Update progress at predefined interval + if idx % PACKAGER_UPDATE_INTERVAL == 0 or idx == gridcount - 1: + update_status( + id=id, status_id=PACKAGE_STATUS["INITIATED"], progress=_progress + ) + if _progress % PACKAGER_UPDATE_INTERVAL == 0: + logger.info(f'Download ID "{id}" progress: {_progress}%') + + except (RuntimeError, Exception): + exc_type, exc_value, exc_traceback = sys.exc_info() + traceback_details = { + "filename": Path(exc_traceback.tb_frame.f_code.co_filename).name, + "line number": exc_traceback.tb_lineno, + "method": exc_traceback.tb_frame.f_code.co_name, + "type": exc_type.__name__, + "message": exc_value, + } + logger.error(traceback_details) + + continue + + finally: + data = None + raster = None + warp_ds = None + # Try to unlink and remove the memory raster object for each source file processed + try: + gdal.Unlink(mem_raster) + mem_raster = None + except Exception as ex: + logger.debug("vismem unlink exception: %s", ex) + + ds = None + spatialGridStruct = None # If no progress was made for any items in the payload (ex: all tifs could not be projected properly), # don't return a dssfilename diff --git a/docker-compose.minio.yml b/docker-compose.minio.yml index e8067d0a..009935bf 100644 --- a/docker-compose.minio.yml +++ b/docker-compose.minio.yml @@ -4,7 +4,7 @@ networks: services: minio: - image: minio/minio + image: minio/minio:RELEASE.2025-04-22T22-12-26Z environment: - MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE - MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY @@ -16,7 +16,7 @@ services: # inspired by https://github.com/minio/minio/issues/4769 # and https://gist.github.com/haxoza/22afe7cc4a9da7e8bdc09aad393a99cc minio_init: - image: minio/mc + image: minio/mc:RELEASE.2025-04-16T18-13-26Z depends_on: - minio entrypoint: > diff --git a/docker-compose.yml b/docker-compose.yml index 51c0457f..ed983e21 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,7 +81,6 @@ services: context: async_packager args: - GDAL_TAG:ubuntu-full-3.9.1 - - TIFFDSS_VERSION=v0.2.2 # entrypoint: sleep infinity # override the Dockerfile entrypoint environment: - AWS_SQS_ENDPOINT=elasticmq:9324 diff --git a/sql/Dockerfile b/sql/Dockerfile index b3e07e86..d672dac8 100644 --- a/sql/Dockerfile +++ b/sql/Dockerfile @@ -1,5 +1,5 @@ # FROM flyway/flyway -FROM flyway/flyway:latest-alpine +FROM ghcr.io/cwbi-apps/flyway:latest COPY . /flyway/sql/