diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml index 88dff7cd3..6bf901701 100644 --- a/workflows/raster/standardising.yaml +++ b/workflows/raster/standardising.yaml @@ -341,7 +341,72 @@ spec: expression: 'false' inputs: parameters: + - name: version_argo_tasks + value: '{{workflow.parameters.version_argo_tasks}}' + - name: version_basemaps_cli + value: '{{workflow.parameters.version_basemaps_cli}}' + - name: version_topo_imagery + value: '{{workflow.parameters.version_topo_imagery}}' + - name: user_group + value: '{{workflow.parameters.user_group}}' + - name: ticket + value: '{{workflow.parameters.ticket}}' + - name: region + value: '{{workflow.parameters.region}}' - name: source + value: '{{workflow.parameters.source}}' + - name: include + value: '{{workflow.parameters.include}}' + - name: scale + value: '{{workflow.parameters.scale}}' + - name: validate + value: '{{workflow.parameters.validate}}' + - name: retile + value: '{{workflow.parameters.retile}}' + - name: source_epsg + value: '{{workflow.parameters.source_epsg}}' + - name: target_epsg + value: '{{workflow.parameters.target_epsg}}' + - name: group + value: '{{workflow.parameters.group}}' + - name: compression + value: '{{workflow.parameters.compression}}' + - name: create_capture_area + value: '{{workflow.parameters.create_capture_area}}' + - name: cutline + value: '{{workflow.parameters.cutline}}' + - name: odr_url + value: '{{workflow.parameters.odr_url}}' + - name: category + value: '{{workflow.parameters.category}}' + - name: gsd + value: '{{workflow.parameters.gsd}}' + - name: producer + value: '{{workflow.parameters.producer}}' + - name: producer_list + value: '{{workflow.parameters.producer_list}}' + - name: licensor + value: '{{workflow.parameters.licensor}}' + - name: licensor_list + value: '{{workflow.parameters.licensor_list}}' + - name: start_datetime + value: '{{workflow.parameters.start_datetime}}' + - name: end_datetime + value: '{{workflow.parameters.end_datetime}}' + - name: geographic_description + value: '{{workflow.parameters.geographic_description}}' + - name: lifecycle + value: '{{workflow.parameters.lifecycle}}' + - name: event + value: '{{workflow.parameters.event}}' + - name: historic_survey_number + value: '{{workflow.parameters.historic_survey_number}}' + - name: publish_to_odr + value: '{{workflow.parameters.publish_to_odr}}' + - name: target_bucket_name + value: '{{workflow.parameters.target_bucket_name}}' + - name: copy_option + value: '{{workflow.parameters.copy_option}}' dag: tasks: - name: stac-setup @@ -351,21 +416,21 @@ spec: arguments: parameters: - name: start_datetime - value: '{{=sprig.trim(workflow.parameters.start_datetime)}}' + value: '{{=sprig.trim(input.parameters.start_datetime)}}' - name: end_datetime - value: '{{=sprig.trim(workflow.parameters.end_datetime)}}' + value: '{{=sprig.trim(input.parameters.end_datetime)}}' - name: gsd - value: '{{=sprig.trim(workflow.parameters.gsd)}}' + value: '{{=sprig.trim(input.parameters.gsd)}}' - name: region - value: '{{=sprig.trim(workflow.parameters.region)}}' + value: '{{=sprig.trim(input.parameters.region)}}' - name: geographic_description - value: '{{=sprig.trim(workflow.parameters.geographic_description)}}' + value: '{{=sprig.trim(input.parameters.geographic_description)}}' - name: geospatial_category - value: '{{=sprig.trim(workflow.parameters.category)}}' + value: '{{=sprig.trim(input.parameters.category)}}' - name: odr_url - value: '{{=sprig.trim(workflow.parameters.odr_url)}}' + value: '{{=sprig.trim(input.parameters.odr_url)}}' - name: version - value: '{{=sprig.trim(workflow.parameters.version_argo_tasks)}}' + value: '{{=sprig.trim(input.parameters.version_argo_tasks)}}' - name: tile-index-validate templateRef: @@ -374,21 +439,21 @@ spec: arguments: parameters: - name: scale - value: '{{workflow.parameters.scale}}' + value: '{{input.parameters.scale}}' - name: include - value: '{{workflow.parameters.include}}' + value: '{{input.parameters.include}}' - name: source value: '{{=sprig.trim(inputs.parameters.source)}}' - name: source_epsg - value: '{{=sprig.trim(workflow.parameters.source_epsg)}}' + value: '{{=sprig.trim(input.parameters.source_epsg)}}' - name: validate - value: '{{= workflow.parameters.validate}}' + value: '{{= input.parameters.validate}}' - name: retile - value: '{{= workflow.parameters.retile}}' + value: '{{= input.parameters.retile}}' - name: preset - value: '{{= workflow.parameters.compression}}' + value: '{{= input.parameters.compression}}' - name: version - value: '{{= workflow.parameters.version_argo_tasks}}' + value: '{{= input.parameters.version_argo_tasks}}' - name: group templateRef: @@ -400,9 +465,9 @@ spec: from: '{{ tasks.tile-index-validate.outputs.artifacts.files }}' parameters: - name: size - value: '{{workflow.parameters.group}}' + value: '{{input.parameters.group}}' - name: version - value: '{{= workflow.parameters.version_argo_tasks}}' + value: '{{= input.parameters.version_argo_tasks}}' depends: 'tile-index-validate' - name: standardise-validate @@ -457,7 +522,7 @@ spec: template: main - name: create-overview - when: "'{{workflow.parameters.target_epsg}}' =~ '2193|3857' && '{{workflow.parameters.compression}}' != 'dem_lerc'" + when: "'{{input.parameters.target_epsg}}' =~ '2193|3857' && '{{input.parameters.compression}}' != 'dem_lerc'" arguments: parameters: - name: location @@ -466,7 +531,7 @@ spec: depends: 'standardise-validate' - name: create-config - when: "'{{workflow.parameters.target_epsg}}' =~ '2193|3857'" + when: "'{{input.parameters.target_epsg}}' =~ '2193|3857'" arguments: parameters: - name: location @@ -482,17 +547,17 @@ spec: templateRef: name: publish-odr template: main - when: "'{{workflow.parameters.publish_to_odr}}' == 'true'" + when: "'{{input.parameters.publish_to_odr}}' == 'true'" arguments: parameters: - name: source value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/flat/' - name: target_bucket_name - value: '{{workflow.parameters.target_bucket_name}}' + value: '{{input.parameters.target_bucket_name}}' - name: copy_option - value: '{{workflow.parameters.copy_option}}' + value: '{{input.parameters.copy_option}}' - name: ticket - value: '{{=sprig.trim(workflow.parameters.ticket)}}' + value: '{{=sprig.trim(input.parameters.ticket)}}' depends: 'stac-validate && create-config' outputs: @@ -515,7 +580,7 @@ spec: - name: group_data path: /tmp/input/ container: - image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:{{=sprig.trim(workflow.parameters.version_topo_imagery)}}' + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:{{=sprig.trim(input.parameters.version_topo_imagery)}}' resources: requests: memory: 7.8Gi @@ -532,25 +597,25 @@ spec: - '--target' - '{{inputs.parameters.target}}' - '--preset' - - '{{workflow.parameters.compression}}' + - '{{input.parameters.compression}}' - '--start-datetime' - - '{{=sprig.trim(workflow.parameters.start_datetime)}}' + - '{{=sprig.trim(input.parameters.start_datetime)}}' - '--end-datetime' - - '{{=sprig.trim(workflow.parameters.end_datetime)}}' + - '{{=sprig.trim(input.parameters.end_datetime)}}' - '--collection-id' - '{{inputs.parameters.collection_id}}' - '--create-footprints' - - '{{workflow.parameters.create_capture_area}}' + - '{{input.parameters.create_capture_area}}' - '--cutline' - - '{{=sprig.trim(workflow.parameters.cutline)}}' + - '{{=sprig.trim(input.parameters.cutline)}}' - '--source-epsg' - - '{{=sprig.trim(workflow.parameters.source_epsg)}}' + - '{{=sprig.trim(input.parameters.source_epsg)}}' - '--target-epsg' - - '{{=sprig.trim(workflow.parameters.target_epsg)}}' + - '{{=sprig.trim(input.parameters.target_epsg)}}' - '--gsd' - - '{{=sprig.trim(workflow.parameters.gsd)}}' + - '{{=sprig.trim(input.parameters.gsd)}}' - '--odr-url' - - '{{=sprig.trim(workflow.parameters.odr_url)}}' + - '{{=sprig.trim(input.parameters.odr_url)}}' - '--current-datetime' - '{{inputs.parameters.current_datetime}}' @@ -571,7 +636,7 @@ spec: archive: none: {} container: - image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:{{=sprig.trim(workflow.parameters.version_topo_imagery)}}' + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:{{=sprig.trim(input.parameters.version_topo_imagery)}}' resources: requests: memory: 7.8Gi @@ -586,30 +651,30 @@ spec: - '--linz-slug' - '{{inputs.parameters.linz_slug}}' - '--odr-url' - - '{{=sprig.trim(workflow.parameters.odr_url)}}' + - '{{=sprig.trim(input.parameters.odr_url)}}' - '--category' - - '{{=sprig.trim(workflow.parameters.category)}}' + - '{{=sprig.trim(input.parameters.category)}}' - '--region' - - '{{=sprig.trim(workflow.parameters.region)}}' + - '{{=sprig.trim(input.parameters.region)}}' - '--gsd' - - '{{=sprig.trim(workflow.parameters.gsd)}}' + - '{{=sprig.trim(input.parameters.gsd)}}' - '--geographic-description' - - '{{=sprig.trim(workflow.parameters.geographic_description)}}' + - '{{=sprig.trim(input.parameters.geographic_description)}}' - '--event' - - '{{=sprig.trim(workflow.parameters.event)}}' + - '{{=sprig.trim(input.parameters.event)}}' - '--historic-survey-number' - - '{{=sprig.trim(workflow.parameters.historic_survey_number)}}' + - '{{=sprig.trim(input.parameters.historic_survey_number)}}' - '--lifecycle' - - '{{=sprig.trim(workflow.parameters.lifecycle)}}' + - '{{=sprig.trim(input.parameters.lifecycle)}}' - '--add-title-suffix' - '--producer' - - '{{workflow.parameters.producer}}' + - '{{input.parameters.producer}}' - '--producer-list' - - '{{=sprig.trim(workflow.parameters.producer_list)}}' + - '{{=sprig.trim(input.parameters.producer_list)}}' - '--licensor' - - '{{workflow.parameters.licensor}}' + - '{{input.parameters.licensor}}' - '--licensor-list' - - '{{=sprig.trim(workflow.parameters.licensor_list)}}' + - '{{=sprig.trim(input.parameters.licensor_list)}}' - '--concurrency' - '25' - '--current-datetime' @@ -648,7 +713,7 @@ spec: - name: bucket - name: key container: - image: 'ghcr.io/linz/basemaps/cli:{{=sprig.trim(workflow.parameters.version_basemaps_cli)}}' + image: 'ghcr.io/linz/basemaps/cli:{{=sprig.trim(input.parameters.version_basemaps_cli)}}' command: [node, /app/node_modules/@basemaps/cogify/dist/index.cjs] env: - name: AWS_ROLE_CONFIG_PATH diff --git a/workflows/test/test-imagery-standardising.yaml b/workflows/test/test-imagery-standardising.yaml new file mode 100644 index 000000000..92a207dcf --- /dev/null +++ b/workflows/test/test-imagery-standardising.yaml @@ -0,0 +1,408 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json + +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: test-imagery-standardising- + annotations: + workflows.argoproj.io/title: '' # defaults to `metadata.name` if not specified + workflows.argoproj.io/description: '' + labels: + linz.govt.nz/category: test + linz.govt.nz/data-type: raster +spec: + parallelism: 50 + nodeSelector: + karpenter.sh/capacity-type: 'spot' + entrypoint: main + synchronization: + semaphore: + configMapKeyRef: + name: semaphores + key: standardising + workflowMetadata: + labelsFrom: + linz.govt.nz/user-group: + expression: workflow.parameters.user_group + linz.govt.nz/ticket: + expression: workflow.parameters.ticket + linz.govt.nz/region: + expression: workflow.parameters.region + podMetadata: + labels: + linz.govt.nz/user-group: '{{workflow.parameters.user_group}}' + linz.govt.nz/category: test + linz.govt.nz/data-type: raster + linz.govt.nz/ticket: '{{workflow.parameters.ticket}}' + linz.govt.nz/region: '{{workflow.parameters.region}}' + arguments: + parameters: + - name: version_argo_tasks + description: 'Specify a version of the argo-tasks image to use, e.g. "v4.1" or "latest"' + value: 'v4' + - name: version_basemaps_cli + description: 'Specify a version of the basemaps-cli image to use, e.g. "v7.1" or "latest"' + value: 'v7' + - name: version_topo_imagery + description: 'Specify a version of the topo-imagery image to use, e.g. "v4.8" or "latest"' + value: 'latest' + - name: user_group + description: Group of users running the workflow + value: 'none' + enum: + - 'land' + - 'sea' + - 'none' + - name: ticket + description: 'Ticket ID, e.g. "LI-1570"' + value: '' + - name: region + description: 'Region of the dataset from the list below' + value: 'new-zealand' + enum: + - 'antarctica' + - 'auckland' + - 'bay-of-plenty' + - 'canterbury' + - 'gisborne' + - 'global' + - 'hawkes-bay' + - 'manawatu-whanganui' + - 'marlborough' + - 'nelson' + - 'new-zealand' + - 'northland' + - 'otago' + - 'pacific-islands' + - 'southland' + - 'taranaki' + - 'tasman' + - 'waikato' + - 'wellington' + - 'west-coast' + - name: source + description: 'S3 location of the source dataset(s). Separate multiple sources with a semicolon (;) for merging. Target imagery will be layered (bottom to top) in order specified (left to right)' + value: '' + - name: include + description: 'Regular expression pattern match for paths/files to include e.g ".tiff?$"' + value: '.tiff?$' + - name: scale + description: 'Scale of the standardised output imagery' + value: '500' + enum: + - '500' + - '1000' + - '2000' + - '5000' + - '10000' + - '50000' + - 'None' + - name: validate + description: 'Validate the tiles match LINZ map sheet tile index and there are no duplicate tiles' + value: 'false' + - name: retile + description: 'Retile the dataset to the output scale specified in the "scale" parameter' + value: 'false' + - name: source_epsg + description: 'EPSG of the source files' + value: '2193' + - name: target_epsg + description: 'EPSG of the standardised output files' + value: '2193' + - name: group + description: 'How many output tiles to process in each standardise-validate task "pod". Change if you have resource or performance issues when standardising a dataset.' + value: '50' + - name: compression + description: 'Compression type to use when standardising TIFFs, e.g. "webp" for imagery or "dem_lerc" for elevation data' + value: 'webp' + enum: + - 'webp' + - 'lzw' + - 'dem_lerc' + - name: create_capture_area + description: 'Create a capture area GeoJSON file for the standardised dataset' + value: 'true' + - name: cutline + description: '(Optional) location of a cutline file to cut the imagery to .fgb or .geojson' + value: '' + - name: odr_url + description: '(Optional) If an existing dataset add the S3 path to the dataset here to load existing metadata e.g. "s3://nz-imagery/taranaki/new-plymouth_2017_0.1m/rgb/2193/"' + value: '' + - name: category + description: 'Geospatial category of the dataset' + value: 'aerial-photos' + enum: + - 'aerial-photos' + - 'urban-aerial-photos' + - 'rural-aerial-photos' + - 'scanned-aerial-photos' + - 'dem' + - 'dsm' + - 'satellite-imagery' + - name: gsd + description: 'Dataset GSD in metres, e.g., "0.3" for 30 centimetres' + value: '0.3' + - name: producer + description: 'The producer of the source dataset, e.g. aerial or bathymetric survey company' + value: 'Unknown' + enum: + [ + 'Unknown', + 'AAM NZ', + 'Aerial Surveys', + 'Beca', + 'Chang Guang Satellite Technology', + 'Christchurch Helicopters', + 'Dimap', + 'European Space Agency', + 'GeoSmart', + 'Landpro', + 'Maxar', + 'NZ Aerial Mapping', + 'Ocean Infinity', + 'Recon', + 'RPS', + 'SKYCAN', + 'SKYVUW', + 'Terralink International', + 'UAV Mapping NZ', + 'University of Canterbury', + 'Woolpert', + ] + - name: producer_list + description: '(Optional) List of imagery producers, separated by semicolon (;). Has no effect unless a semicolon delimited list is entered.' + value: '' + - name: licensor + description: 'The licensor of the dataset, e.g. local or regional council, government agency, satellite provider' + value: 'Unknown' + enum: + [ + 'Unknown', + 'Ashburton District Council', + 'Auckland Council', + 'BOPLASS', + 'Bay of Plenty Regional Council', + 'Buller District Council', + 'Canterbury Aerial Imagery Consortium (CAI)', + 'Carterton District Council', + "Central Hawke's Bay District Council", + 'Central Otago District Council', + 'Chang Guang Satellite Technology', + 'Chatham Islands Council', + 'Christchurch City Council', + 'Clutha District Council', + 'CoLAB', + 'Department of Conservation', + 'Dunedin City Council', + 'Environment Canterbury', + 'Environment Southland', + 'Far North District Council', + 'Gisborne District Council', + 'Gore District Council', + 'Greater Wellington Regional Council', + 'Grey District Council', + 'Hamilton City Council', + 'Hastings District Council', + 'Hauraki District Council', + "Hawke's Bay Local Authority Shared Services (HB LASS)", + "Hawke's Bay Regional Council", + 'Horizons Regional Council', + 'Horowhenua District Council', + 'Hurunui District Council', + 'Hutt City Council', + 'Invercargill City Council', + 'Kaikōura District Council', + 'Kaipara District Council', + 'Kawerau District Council', + 'Kāpiti Coast District Council', + 'Mackenzie District Council', + 'Manawatū District Council', + 'Manawatū-Whanganui LASS', + 'Marlborough District Council', + 'Masterton District Council', + 'Matamata-Piako District Council', + 'Maxar Technologies', + 'Ministry of Business, Innovation and Employment', + 'Ministry of Primary Industries', + 'NZ Aerial Mapping', + 'Napier City Council', + 'National Emergency Management Agency', + 'National Institute of Water and Atmospheric Research', + 'Nelson City Council', + 'New Plymouth District Council', + 'Northland Regional Council', + 'Ōpōtiki District Council', + 'Ōtorohanga District Council', + 'Otago Regional Council', + 'Palmerston North City Council', + 'Planet', + 'Porirua City Council', + 'Queenstown-Lakes District Council', + 'Rangitīkei District Council', + 'Regional Software Holdings Limited', + 'Rotorua District Council', + 'Ruapehu District Council', + 'Selwyn District Council', + 'Sinergise', + 'South Taranaki District Council', + 'South Waikato District Council', + 'South Wairarapa District Council', + 'Southland District Council', + 'Stratford District Council', + 'Taranaki Regional Council', + 'Tararua District Council', + 'Tasman District Council', + 'Taupō District Council', + 'Tauranga City Council', + 'Terralink International', + 'Thames-Coromandel District Council', + 'Timaru District Council', + 'Toitū Te Whenua Land Information New Zealand', + 'Upper Hutt City Council', + 'Waikato District Council', + 'Waikato Regional Aerial Photography Service (WRAPS)', + 'Waikato Regional Council', + 'Waimakariri District Council', + 'Waimate District Council', + 'Waipā District Council', + 'Wairoa District Council', + 'Waitaki District Council', + 'Waitomo District Council', + 'Waka Kotahi', + 'Wellington City Council', + 'West Coast Regional Council', + 'Western Bay of Plenty District Council', + 'Westland District Council', + 'Whakatāne District Council', + 'Whanganui District Council', + 'Whangārei District Council', + ] + - name: licensor_list + description: '(Optional) List of imagery licensors, separated by semicolon (;). Has no effect unless a semicolon delimited list is entered.' + value: '' + - name: start_datetime + description: 'Dataset capture start date in numeric format YYYY-MM-DD, e.g. "2024-01-14"' + value: '2025-01-01' + - name: end_datetime + description: 'Dataset capture end date in numeric format YYYY-MM-DD, e.g. "2024-02-23"' + value: '2025-01-01' + - name: geographic_description + description: '(Optional) Additional dataset description, to be used in the title in place of the Region, e.g. "Hamilton"' + value: '' + - name: lifecycle + description: 'The release lifecycle status of the dataset, e.g. "completed or "ongoing"' + value: 'under development' + enum: + - 'under development' + - 'preview' + - 'ongoing' + - 'completed' + - 'deprecated' + - name: event + description: '(Optional) Event name if dataset has been captured in association with an event, e.g. "Top of the South Floods"' + value: '' + - name: historic_survey_number + description: '(Optional) Survey Number associated with historical datasets, e.g. "SN8844"' + value: '' + - name: publish_to_odr + description: 'Create a Pull Request for publishing to imagery or elevation ODR bucket' + value: 'false' + - name: target_bucket_name + description: 'The ODR bucket name to publish to' + value: '' + enum: + - '' + - 'nz-imagery' + - 'nz-elevation' + - '' + - name: copy_option + description: 'Do not overwrite existing files with "no-clobber", "force" overwriting files in the target location, or "force-no-clobber" overwriting only changed files, skipping unchanged files' + value: '--no-clobber' + enum: + - '--no-clobber' + - '--force' + - '--force-no-clobber' + templateDefaults: + container: + imagePullPolicy: Always + image: '' + templates: + - name: main + retryStrategy: + expression: 'false' + steps: + - - name: standardise-validate + templateRef: + name: imagery-standardising + template: main + arguments: + parameters: + - name: version_argo_tasks + value: '{{workflow.parameters.version_argo_tasks}}' + - name: version_basemaps_cli + value: '{{workflow.parameters.version_basemaps_cli}}' + - name: version_topo_imagery + value: '{{workflow.parameters.version_topo_imagery}}' + - name: user_group + value: '{{workflow.parameters.user_group}}' + - name: ticket + value: '{{workflow.parameters.ticket}}' + - name: region + value: '{{workflow.parameters.region}}' + - name: source + value: '{{workflow.parameters.source}}' + - name: include + value: '{{workflow.parameters.include}}' + - name: scale + value: '{{workflow.parameters.scale}}' + - name: validate + value: '{{workflow.parameters.validate}}' + - name: retile + value: '{{workflow.parameters.retile}}' + - name: source_epsg + value: '{{workflow.parameters.source_epsg}}' + - name: target_epsg + value: '{{workflow.parameters.target_epsg}}' + - name: group + value: '{{workflow.parameters.group}}' + - name: compression + value: '{{workflow.parameters.compression}}' + - name: create_capture_area + value: '{{workflow.parameters.create_capture_area}}' + - name: cutline + value: '{{workflow.parameters.cutline}}' + - name: odr_url + value: '{{workflow.parameters.odr_url}}' + - name: category + value: '{{workflow.parameters.category}}' + - name: gsd + value: '{{workflow.parameters.gsd}}' + - name: producer + value: '{{workflow.parameters.producer}}' + - name: producer_list + value: '{{workflow.parameters.producer_list}}' + - name: licensor + value: '{{workflow.parameters.licensor}}' + - name: licensor_list + value: '{{workflow.parameters.licensor_list}}' + - name: start_datetime + value: '{{workflow.parameters.start_datetime}}' + - name: end_datetime + value: '{{workflow.parameters.end_datetime}}' + - name: geographic_description + value: '{{workflow.parameters.geographic_description}}' + - name: lifecycle + value: '{{workflow.parameters.lifecycle}}' + - name: event + value: '{{workflow.parameters.event}}' + - name: historic_survey_number + value: '{{workflow.parameters.historic_survey_number}}' + - name: publish_to_odr + value: '{{workflow.parameters.publish_to_odr}}' + - name: target_bucket_name + value: '{{workflow.parameters.target_bucket_name}}' + - name: copy_option + value: '{{workflow.parameters.copy_option}}' + volumes: + - name: ephemeral + emptyDir: {}