diff --git a/.eslintrc.cjs b/.eslintrc.cjs index bce6af37b..a0447db0a 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -5,4 +5,22 @@ const cfg = { const testOverrides = cfg.overrides.find((ovr) => ovr.files.find((f) => f.includes('.test.ts'))); testOverrides.rules['@typescript-eslint/no-floating-promises'] = 'off'; +cfg.overrides.push({ + /** Overrides for YAML */ + files: ['*.yaml', '*.yml'], + extends: ['plugin:yml/standard'], + rules: { + // Prettier uses single quotes so be consistent + 'yml/quotes': ['error', { prefer: 'single' }], + // Prettier will reformat this back into a single line + 'yml/block-sequence-hyphen-indicator-newline': ['off'], + // Prettier will sometimes convert these to multiline, so be safe and always use multi + 'yml/block-sequence': ['error', { singleline: 'always' }], + + // Force string values to be quoted + 'yml/plain-scalar': ['error', 'never', { overrides: { mappingKey: 'always' } }], + }, + parser: 'yaml-eslint-parser', +}); + module.exports = cfg; diff --git a/cdk8s.yaml b/cdk8s.yaml index a593cdf76..4b3051aef 100644 --- a/cdk8s.yaml +++ b/cdk8s.yaml @@ -1,5 +1,5 @@ -app: npx tsx infra/cdk8s.ts -language: typescript +app: 'npx tsx infra/cdk8s.ts' +language: 'typescript' imports: - - https://raw.githubusercontent.com/aws/karpenter/main/pkg/apis/crds/karpenter.sh_provisioners.yaml - - https://raw.githubusercontent.com/aws/karpenter/main/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml + - 'https://raw.githubusercontent.com/aws/karpenter/main/pkg/apis/crds/karpenter.sh_provisioners.yaml' + - 'https://raw.githubusercontent.com/aws/karpenter/main/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml' diff --git a/docs/training/examples/wf_hello_world.yaml b/docs/training/examples/wf_hello_world.yaml index f15d45d92..997799ca6 100644 --- a/docs/training/examples/wf_hello_world.yaml +++ b/docs/training/examples/wf_hello_world.yaml @@ -1,21 +1,23 @@ --- -apiVersion: argoproj.io/v1alpha1 -kind: Workflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'Workflow' metadata: - generateName: test-hello-world- - namespace: argo + generateName: 'test-hello-world-' + namespace: 'argo' spec: nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' dag: tasks: - - name: say-hello-task - template: say-hello-template - - name: say-hello-template + - name: 'say-hello-task' + template: 'say-hello-template' + - name: 'say-hello-template' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:v3' - command: [echo] - args: ['hello world'] + command: + - 'echo' + args: + - 'hello world' diff --git a/docs/training/examples/wf_hello_world_args.yaml b/docs/training/examples/wf_hello_world_args.yaml index 038e81880..4ea7124f9 100644 --- a/docs/training/examples/wf_hello_world_args.yaml +++ b/docs/training/examples/wf_hello_world_args.yaml @@ -1,32 +1,34 @@ --- -apiVersion: argoproj.io/v1alpha1 -kind: Workflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'Workflow' metadata: - generateName: test-hello-world-args- - namespace: argo + generateName: 'test-hello-world-args-' + namespace: 'argo' spec: nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main + entrypoint: 'main' arguments: parameters: - - name: message + - name: 'message' value: 'hello world' templates: - - name: main + - name: 'main' dag: tasks: - - name: say-hello-task - template: say-hello-template + - name: 'say-hello-task' + template: 'say-hello-template' arguments: parameters: - - name: message + - name: 'message' value: '{{workflow.parameters.message}}' - - name: say-hello-template + - name: 'say-hello-template' inputs: parameters: - - name: message + - name: 'message' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:v3' - command: [echo] - args: ['{{inputs.parameters.message}}'] + command: + - 'echo' + args: + - '{{inputs.parameters.message}}' diff --git a/docs/training/examples/wf_hello_world_args_tasks.yaml b/docs/training/examples/wf_hello_world_args_tasks.yaml index dace2c433..0b8c669a2 100644 --- a/docs/training/examples/wf_hello_world_args_tasks.yaml +++ b/docs/training/examples/wf_hello_world_args_tasks.yaml @@ -1,40 +1,42 @@ --- -apiVersion: argoproj.io/v1alpha1 -kind: Workflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'Workflow' metadata: - generateName: test-hello-world-args-tasks- - namespace: argo + generateName: 'test-hello-world-args-tasks-' + namespace: 'argo' spec: nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main + entrypoint: 'main' arguments: parameters: - - name: message1 + - name: 'message1' value: 'hello world 1' - - name: message2 + - name: 'message2' value: 'hello world 2' templates: - - name: main + - name: 'main' dag: tasks: - - name: say-hello-task1 - template: say-hello-template + - name: 'say-hello-task1' + template: 'say-hello-template' arguments: parameters: - - name: message + - name: 'message' value: '{{workflow.parameters.message1}}' - - name: say-hello-task2 - template: say-hello-template + - name: 'say-hello-task2' + template: 'say-hello-template' arguments: parameters: - - name: message + - name: 'message' value: '{{workflow.parameters.message2}}' - - name: say-hello-template + - name: 'say-hello-template' inputs: parameters: - - name: message + - name: 'message' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:v3' - command: [echo] - args: ['{{inputs.parameters.message}}'] + command: + - 'echo' + args: + - '{{inputs.parameters.message}}' diff --git a/docs/training/examples/wf_hello_world_dag.yaml b/docs/training/examples/wf_hello_world_dag.yaml index 60e335ecc..827a0bfc0 100644 --- a/docs/training/examples/wf_hello_world_dag.yaml +++ b/docs/training/examples/wf_hello_world_dag.yaml @@ -1,55 +1,57 @@ --- -apiVersion: argoproj.io/v1alpha1 -kind: Workflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'Workflow' metadata: - generateName: test-hello-world-dag- - namespace: argo + generateName: 'test-hello-world-dag-' + namespace: 'argo' spec: nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main + entrypoint: 'main' arguments: parameters: - - name: message1 + - name: 'message1' value: 'hello world 1' - - name: message2 + - name: 'message2' value: 'hello world 2' templates: - - name: main + - name: 'main' dag: tasks: - - name: say-hello-task1 - template: say-hello-template + - name: 'say-hello-task1' + template: 'say-hello-template' arguments: parameters: - - name: message + - name: 'message' value: '{{workflow.parameters.message1}}' - - name: say-hello-task2 - template: say-hello-template + - name: 'say-hello-task2' + template: 'say-hello-template' arguments: parameters: - - name: message + - name: 'message' value: '{{workflow.parameters.message2}}' depends: 'say-hello-task1' - - name: say-hello-task3 - template: say-hello-template + - name: 'say-hello-task3' + template: 'say-hello-template' arguments: parameters: - - name: message + - name: 'message' value: 'say hello task 3' depends: 'say-hello-task1' - - name: say-hello-task4 - template: say-hello-template + - name: 'say-hello-task4' + template: 'say-hello-template' arguments: parameters: - - name: message + - name: 'message' value: 'say hello task 4' depends: 'say-hello-task2 && say-hello-task3' - - name: say-hello-template + - name: 'say-hello-template' inputs: parameters: - - name: message + - name: 'message' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:v3' - command: [echo] - args: ['{{inputs.parameters.message}}'] + command: + - 'echo' + args: + - '{{inputs.parameters.message}}' diff --git a/docs/training/examples/wf_output_parallel.yaml b/docs/training/examples/wf_output_parallel.yaml index 4698689cb..3e8ae48cf 100644 --- a/docs/training/examples/wf_output_parallel.yaml +++ b/docs/training/examples/wf_output_parallel.yaml @@ -1,77 +1,76 @@ --- -apiVersion: argoproj.io/v1alpha1 -kind: Workflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'Workflow' metadata: - generateName: test-output-parallel- + generateName: 'test-output-parallel-' spec: parallelism: 20 nodeSelector: karpenter.sh/capacity-type: 'spot' - serviceAccountName: workflow-runner-sa - entrypoint: main + serviceAccountName: 'workflow-runner-sa' + entrypoint: 'main' arguments: parameters: - - name: uri + - name: 'uri' value: 's3://linz-imagery-staging/test/stac-validate/' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' templates: - - name: main + - name: 'main' dag: tasks: - - name: aws-list - template: aws-list + - name: 'aws-list' + template: 'aws-list' arguments: parameters: - - name: uri + - name: 'uri' value: '{{workflow.parameters.uri}}' - - name: include + - name: 'include' value: 'json$' - - name: stac-print-path - template: stac-print-path + - name: 'stac-print-path' + template: 'stac-print-path' arguments: parameters: - - name: file + - name: 'file' value: '{{item}}' depends: 'aws-list' withParam: '{{tasks.aws-list.outputs.parameters.files}}' - - name: aws-list + - name: 'aws-list' inputs: parameters: - - name: uri - - name: include + - name: 'uri' + - name: 'include' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:v2' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - [ - 'list', - '--verbose', - '--include', - '{{inputs.parameters.include}}', - '--group', - '4', - '--output', - '/tmp/file_list.json', - '{{inputs.parameters.uri}}', - ] + - 'list' + - '--verbose' + - '--include' + - '{{inputs.parameters.include}}' + - '--group' + - '4' + - '--output' + - '/tmp/file_list.json' + - '{{inputs.parameters.uri}}' + outputs: parameters: - - name: files + - name: 'files' valueFrom: - path: /tmp/file_list.json - - name: stac-print-path + path: '/tmp/file_list.json' + - name: 'stac-print-path' inputs: parameters: - - name: file + - name: 'file' script: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:v2' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' command: - 'bash' source: | diff --git a/package-lock.json b/package-lock.json index 9fdbbee74..cecdf54cb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,8 @@ "cdk8s-cli": "^2.198.228", "cdk8s-plus-30": "^2.2.5", "constructs": "^10.3.0", + "eslint": "^8.57.1", + "eslint-plugin-yml": "^1.17.0", "tsx": "^4.6.2", "yaml": "^2.5.1" }, @@ -1249,6 +1251,7 @@ "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", "dev": true, + "license": "MIT", "dependencies": { "ajv": "^6.12.4", "debug": "^4.3.2", @@ -1268,22 +1271,24 @@ } }, "node_modules/@eslint/js": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", - "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", + "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", "dev": true, + "license": "MIT", "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, "node_modules/@humanwhocodes/config-array": { - "version": "0.11.14", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", - "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", + "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", "deprecated": "Use @eslint/config-array instead", "dev": true, + "license": "Apache-2.0", "dependencies": { - "@humanwhocodes/object-schema": "^2.0.2", + "@humanwhocodes/object-schema": "^2.0.3", "debug": "^4.3.1", "minimatch": "^3.0.5" }, @@ -1309,7 +1314,8 @@ "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", "deprecated": "Use @eslint/object-schema instead", - "dev": true + "dev": true, + "license": "BSD-3-Clause" }, "node_modules/@jsii/check-node": { "version": "1.103.1", @@ -2007,6 +2013,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.18.0.tgz", "integrity": "sha512-94EQTWZ40mzBc42ATNIBimBEDltSJ9RQHCC8vc/PDbxi4k8dVwUAv4o98dk50M1zB+JGFxp43FP7f8+FP8R6Sw==", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.10.0", "@typescript-eslint/scope-manager": "7.18.0", @@ -2040,6 +2047,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.18.0.tgz", "integrity": "sha512-4Z+L8I2OqhZV8qA132M4wNL30ypZGYOQVBfMgxDH/K5UX0PNqTu1c6za9ST5r9+tavvHiTWmBnKzpCJ/GlVFtg==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { "@typescript-eslint/scope-manager": "7.18.0", "@typescript-eslint/types": "7.18.0", @@ -2085,6 +2093,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.18.0.tgz", "integrity": "sha512-XL0FJXuCLaDuX2sYqZUUSOJ2sG5/i1AAze+axqmLnSkNEVMVYLF+cbwlB2w8D1tinFuSikHmFta+P+HOofrLeA==", "dev": true, + "license": "MIT", "dependencies": { "@typescript-eslint/typescript-estree": "7.18.0", "@typescript-eslint/utils": "7.18.0", @@ -2177,6 +2186,7 @@ "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.18.0.tgz", "integrity": "sha512-kK0/rNa2j74XuHVcoCZxdFBMF+aq/vH83CXAOHieC+2Gis4mF8jJXT5eAfyD3K0sAxtPuwxaIOIOvhwzVDt/kw==", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", "@typescript-eslint/scope-manager": "7.18.0", @@ -2212,10 +2222,11 @@ } }, "node_modules/@ungap/structured-clone": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", - "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", - "dev": true + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", + "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", + "dev": true, + "license": "ISC" }, "node_modules/@xmldom/xmldom": { "version": "0.9.3", @@ -2227,10 +2238,11 @@ } }, "node_modules/acorn": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", - "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", + "version": "8.14.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz", + "integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==", "dev": true, + "license": "MIT", "bin": { "acorn": "bin/acorn" }, @@ -2243,6 +2255,7 @@ "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", "dev": true, + "license": "MIT", "peerDependencies": { "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } @@ -2252,6 +2265,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "dev": true, + "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -2291,7 +2305,8 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true + "dev": true, + "license": "Python-2.0" }, "node_modules/array-buffer-byte-length": { "version": "1.0.1", @@ -2913,6 +2928,7 @@ "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } @@ -3443,10 +3459,11 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, + "license": "MIT", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -3620,6 +3637,7 @@ "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", "dev": true, + "license": "Apache-2.0", "dependencies": { "esutils": "^2.0.2" }, @@ -3886,16 +3904,18 @@ } }, "node_modules/eslint": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", - "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", + "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", + "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, + "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", "@eslint/eslintrc": "^2.1.4", - "@eslint/js": "8.57.0", - "@humanwhocodes/config-array": "^0.11.14", + "@eslint/js": "8.57.1", + "@humanwhocodes/config-array": "^0.13.0", "@humanwhocodes/module-importer": "^1.0.1", "@nodelib/fs.walk": "^1.2.8", "@ungap/structured-clone": "^1.2.0", @@ -3940,6 +3960,22 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint-compat-utils": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/eslint-compat-utils/-/eslint-compat-utils-0.6.4.tgz", + "integrity": "sha512-/u+GQt8NMfXO8w17QendT4gvO5acfxQsAKirAt0LVxDnr2N8YLCVbregaNc/Yhp7NM128DwCaRvr8PLDfeNkQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.4" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "eslint": ">=6.0.0" + } + }, "node_modules/eslint-config-prettier": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/eslint-config-prettier/-/eslint-config-prettier-9.1.0.tgz", @@ -4019,6 +4055,7 @@ "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.2.tgz", "integrity": "sha512-QzliNJq4GinDBcD8gPB5v0wh6g8q3SUi6EFF0x8N/BL9PoVs0atuGc47ozMRyOWAKdwaZ5OnbOEa3WR+dSGKuQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" }, @@ -4066,11 +4103,35 @@ "eslint": ">=5.0.0" } }, + "node_modules/eslint-plugin-yml": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-yml/-/eslint-plugin-yml-1.17.0.tgz", + "integrity": "sha512-Q3LXFRnNpGYAK/PM0BY1Xs0IY1xTLfM0kC986nNQkx1l8tOGz+YS50N6wXkAJkrBpeUN9OxEMB7QJ+9MTDAqIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.3.2", + "escape-string-regexp": "4.0.0", + "eslint-compat-utils": "^0.6.0", + "natural-compare": "^1.4.0", + "yaml-eslint-parser": "^1.2.1" + }, + "engines": { + "node": "^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ota-meshi" + }, + "peerDependencies": { + "eslint": ">=6.0.0" + } + }, "node_modules/eslint-scope": { "version": "7.2.2", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { "esrecurse": "^4.3.0", "estraverse": "^5.2.0" @@ -4099,6 +4160,7 @@ "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { "acorn": "^8.9.0", "acorn-jsx": "^5.3.2", @@ -4128,6 +4190,7 @@ "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { "estraverse": "^5.2.0" }, @@ -4197,7 +4260,8 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/fast-levenshtein": { "version": "2.0.6", @@ -4247,6 +4311,7 @@ "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", "dev": true, + "license": "MIT", "dependencies": { "flat-cache": "^3.0.4" }, @@ -4283,17 +4348,18 @@ } }, "node_modules/flat-cache": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.1.1.tgz", - "integrity": "sha512-/qM2b3LUIaIgviBQovTLvijfyOQXPtSRnRK26ksj2J7rzPIecePUIpJsZ4T02Qg+xiAEKIs5K8dsHEd+VaKa/Q==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", + "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", "dev": true, + "license": "MIT", "dependencies": { "flatted": "^3.2.9", "keyv": "^4.5.3", "rimraf": "^3.0.2" }, "engines": { - "node": ">=12.0.0" + "node": "^10.12.0 || >=12.0.0" } }, "node_modules/flatted": { @@ -4475,6 +4541,7 @@ "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", "dev": true, + "license": "MIT", "dependencies": { "type-fest": "^0.20.2" }, @@ -4645,10 +4712,11 @@ } }, "node_modules/import-fresh": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", - "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", "dev": true, + "license": "MIT", "dependencies": { "parent-module": "^1.0.0", "resolve-from": "^4.0.0" @@ -4931,6 +4999,7 @@ "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -5073,7 +5142,8 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true + "dev": true, + "license": "ISC" }, "node_modules/iterator.prototype": { "version": "1.1.2", @@ -5099,6 +5169,7 @@ "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "dev": true, + "license": "MIT", "dependencies": { "argparse": "^2.0.1" }, @@ -5455,7 +5526,8 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/json-schema": { "version": "0.4.0", @@ -5467,7 +5539,8 @@ "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -5515,6 +5588,7 @@ "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", "dev": true, + "license": "MIT", "dependencies": { "json-buffer": "3.0.1" } @@ -5852,6 +5926,7 @@ "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", "dev": true, + "license": "MIT", "dependencies": { "callsites": "^3.0.0" }, @@ -5882,6 +5957,7 @@ "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -5970,10 +6046,11 @@ } }, "node_modules/punycode": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", - "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } @@ -6118,6 +6195,7 @@ "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true, + "license": "MIT", "engines": { "node": ">=4" } @@ -6151,7 +6229,9 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", "dev": true, + "license": "ISC", "dependencies": { "glob": "^7.1.3" }, @@ -6293,6 +6373,7 @@ "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, + "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" }, @@ -6305,6 +6386,7 @@ "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" } @@ -6550,6 +6632,7 @@ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true, + "license": "MIT", "engines": { "node": ">=8" }, @@ -6645,7 +6728,8 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/to-regex-range": { "version": "5.0.1", @@ -6727,6 +6811,7 @@ "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", "dev": true, + "license": "(MIT OR CC0-1.0)", "engines": { "node": ">=10" }, @@ -6855,6 +6940,7 @@ "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "dev": true, + "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" } @@ -6877,6 +6963,7 @@ "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, + "license": "ISC", "dependencies": { "isexe": "^2.0.0" }, @@ -7028,6 +7115,23 @@ "node": ">= 14" } }, + "node_modules/yaml-eslint-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/yaml-eslint-parser/-/yaml-eslint-parser-1.3.0.tgz", + "integrity": "sha512-E/+VitOorXSLiAqtTd7Yqax0/pAS3xaYMP+AUUJGOK1OZG3rhcj9fcJOM5HJ2VrP1FrStVCWr1muTfQCdj4tAA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.0.0", + "yaml": "^2.0.0" + }, + "engines": { + "node": "^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ota-meshi" + } + }, "node_modules/yargs": { "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", diff --git a/package.json b/package.json index c48686b24..59b91ecfa 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,8 @@ "cdk8s-cli": "^2.198.228", "cdk8s-plus-30": "^2.2.5", "constructs": "^10.3.0", + "eslint": "^8.57.1", + "eslint-plugin-yml": "^1.17.0", "tsx": "^4.6.2", "yaml": "^2.5.1" } diff --git a/templates/argo-tasks/copy.yml b/templates/argo-tasks/copy.yml index 283c74c70..988d32527 100644 --- a/templates/argo-tasks/copy.yml +++ b/templates/argo-tasks/copy.yml @@ -1,47 +1,50 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: # Template for copying a manifest of files between two locations # See https://github.com/linz/argo-tasks#copy - name: tpl-copy + name: 'tpl-copy' spec: templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' inputs: parameters: - - name: file - description: Path to the manifest file detailing source and target + - name: 'file' + description: 'Path to the manifest file detailing source and target' - - name: version_argo_tasks - description: version of argo-tasks to use + - name: 'version_argo_tasks' + description: 'version of argo-tasks to use' default: 'v4' - - name: copy_option - description: --no-clobber Skip overwriting existing files. --force Overwrite all files. --force-no-clobber Overwrite only changed files, skip unchanged files. + - name: 'copy_option' + description: '--no-clobber Skip overwriting existing files. --force Overwrite all files. --force-no-clobber Overwrite only changed files, skip unchanged files.' default: '--no-clobber' enum: - '--no-clobber' - '--force' - '--force-no-clobber' - - name: aws_role_config_path - description: s3 URL or comma-separated list of s3 URLs allowing the workflow to write to a target(s) + - name: 'aws_role_config_path' + description: 's3 URL or comma-separated list of s3 URLs allowing the workflow to write to a target(s)' default: 's3://linz-bucket-config/config-write.imagery.json,s3://linz-bucket-config/config-write.elevation.json,s3://linz-bucket-config/config-write.topographic.json' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}}' resources: requests: - memory: 7.8Gi - cpu: 2000m + memory: '7.8Gi' + cpu: '2000m' env: - - name: AWS_ROLE_CONFIG_PATH + - name: 'AWS_ROLE_CONFIG_PATH' value: '{{inputs.parameters.aws_role_config_path}},s3://linz-bucket-config/config.json' - args: ['copy', '{{inputs.parameters.copy_option}}', '{{inputs.parameters.file}}'] + args: + - 'copy' + - '{{inputs.parameters.copy_option}}' + - '{{inputs.parameters.file}}' diff --git a/templates/argo-tasks/create-manifest.yml b/templates/argo-tasks/create-manifest.yml index e5117fb51..712ae3da5 100644 --- a/templates/argo-tasks/create-manifest.yml +++ b/templates/argo-tasks/create-manifest.yml @@ -1,96 +1,96 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: # Template for creating a manifest to be copied and their target path # See https://github.com/linz/argo-tasks#create-manifest - name: tpl-create-manifest + name: 'tpl-create-manifest' spec: templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' inputs: parameters: - - name: source - description: location of data to publish + - name: 'source' + description: 'location of data to publish' - - name: target - description: location to publish data to + - name: 'target' + description: 'location to publish data to' - - name: version_argo_tasks - description: version of argo-tasks to use + - name: 'version_argo_tasks' + description: 'version of argo-tasks to use' default: 'v4' - - name: include - description: A regular expression to match object path(s) or name(s) from within the source path to include in the copy + - name: 'include' + description: 'A regular expression to match object path(s) or name(s) from within the source path to include in the copy' default: '\.tiff?$|\.json$|\.tfw$' - - name: exclude - description: A regular expression to match object path(s) or name(s) from within the source path to exclude in the copy + - name: 'exclude' + description: 'A regular expression to match object path(s) or name(s) from within the source path to exclude in the copy' default: '' - - name: copy_option - description: --no-clobber Skip overwriting existing files. --force Overwrite all files. --force-no-clobber Overwrite only changed files, skip unchanged files. + - name: 'copy_option' + description: '--no-clobber Skip overwriting existing files. --force Overwrite all files. --force-no-clobber Overwrite only changed files, skip unchanged files.' default: '--no-clobber' enum: - '--no-clobber' - '--force' - '--force-no-clobber' - - name: flatten - description: Flatten the files in the target location + - name: 'flatten' + description: 'Flatten the files in the target location' default: 'false' enum: - 'true' - 'false' - - name: group - description: The maximum number of files for each pod to copy (will use the value of `group` or `group_size` that is reached first) + - name: 'group' + description: 'The maximum number of files for each pod to copy (will use the value of `group` or `group_size` that is reached first)' default: '1000' - - name: group_size - description: The maximum group size of files for each pod to copy (will use the value of `group` or `group_size` that is reached first) + - name: 'group_size' + description: 'The maximum group size of files for each pod to copy (will use the value of `group` or `group_size` that is reached first)' default: '100Gi' - - name: transform - description: String to be transformed from source to target to renamed filenames, e.g. `f.replace("text to replace", "new_text_to_use")`. Leave as `f` for no transformation. + - name: 'transform' + description: 'String to be transformed from source to target to renamed filenames, e.g. `f.replace("text to replace", "new_text_to_use")`. Leave as `f` for no transformation.' default: 'f' outputs: parameters: - - name: files + - name: 'files' valueFrom: - path: /tmp/file_list.json + path: '/tmp/file_list.json' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(inputs.parameters.version_argo_tasks)}}' - command: [node, /app/index.js] + command: + - 'node' + - '/app/index.js' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - [ - 'create-manifest', - '--verbose', - '--include', - '{{=sprig.trim(inputs.parameters.include)}}', - '--exclude', - '{{inputs.parameters.exclude}}', - '--flatten={{=sprig.trim(inputs.parameters.flatten)}}', - '--group', - '{{=sprig.trim(inputs.parameters.group)}}', - '--group-size', - '{{=sprig.trim(inputs.parameters.group_size)}}', - '--output', - '/tmp/file_list.json', - '--target', - '{{=sprig.trim(inputs.parameters.target)}}', - '{{=sprig.trim(inputs.parameters.source)}}', - '--transform', - '{{=sprig.trim(inputs.parameters.transform)}}', - ] + - 'create-manifest' + - '--verbose' + - '--include' + - '{{=sprig.trim(inputs.parameters.include)}}' + - '--exclude' + - '{{inputs.parameters.exclude}}' + - '--flatten={{=sprig.trim(inputs.parameters.flatten)}}' + - '--group' + - '{{=sprig.trim(inputs.parameters.group)}}' + - '--group-size' + - '{{=sprig.trim(inputs.parameters.group_size)}}' + - '--output' + - '/tmp/file_list.json' + - '--target' + - '{{=sprig.trim(inputs.parameters.target)}}' + - '{{=sprig.trim(inputs.parameters.source)}}' + - '--transform' + - '{{=sprig.trim(inputs.parameters.transform)}}' diff --git a/templates/argo-tasks/generate-path.yml b/templates/argo-tasks/generate-path.yml index 24f6399e7..1d3ad3c39 100644 --- a/templates/argo-tasks/generate-path.yml +++ b/templates/argo-tasks/generate-path.yml @@ -1,34 +1,34 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: # Template from linz/argo-tasks # see https://github.com/linz/argo-tasks#generate-path - name: tpl-at-generate-path + name: 'tpl-at-generate-path' spec: templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' inputs: parameters: - - name: source - description: s3 path of source data - - name: target_bucket_name - description: target bucket name e.g. 'nz-imagery' - - name: version - description: argo-task Container version to use + - name: 'source' + description: 's3 path of source data' + - name: 'target_bucket_name' + description: "target bucket name e.g. 'nz-imagery'" + - name: 'version' + description: 'argo-task Container version to use' default: 'v4' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{= inputs.parameters.version }}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - 'generate-path' - '--target-bucket-name' @@ -37,7 +37,7 @@ spec: outputs: parameters: - - name: target - description: ODR target path + - name: 'target' + description: 'ODR target path' valueFrom: path: '/tmp/generate-path/target' diff --git a/templates/argo-tasks/group.yml b/templates/argo-tasks/group.yml index 2a430f24d..066eec28a 100644 --- a/templates/argo-tasks/group.yml +++ b/templates/argo-tasks/group.yml @@ -1,19 +1,19 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: # Template from linz/argo-tasks # see https://github.com/linz/argo-tasks#group - name: tpl-at-group + name: 'tpl-at-group' spec: templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' inputs: artifacts: # @@ -23,15 +23,15 @@ spec: # ```json # ["a.json", "b.json", "c.json"] # ``` - - name: input - path: /tmp/group/input.json + - name: 'input' + path: '/tmp/group/input.json' parameters: - - name: size - description: Group into this number of records per group + - name: 'size' + description: 'Group into this number of records per group' - - name: version - description: argo-task Container version to use + - name: 'version' + description: 'argo-task Container version to use' default: 'v4' outputs: @@ -45,9 +45,9 @@ spec: # ``` # # Workflows should use the "000" string to access the data from the output artifact folder - - name: output + - name: 'output' valueFrom: - path: /tmp/group/output.json + path: '/tmp/group/output.json' artifacts: # Grouped output of the input as one file per output groupId @@ -56,14 +56,14 @@ spec: # - /output/001.json # - ... # - - name: output - path: /tmp/group/output/ + - name: 'output' + path: '/tmp/group/output/' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{= inputs.parameters.version }}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - 'group' - '--verbose' diff --git a/templates/argo-tasks/push-to-github.yml b/templates/argo-tasks/push-to-github.yml index 05094e3fa..33baba7ae 100644 --- a/templates/argo-tasks/push-to-github.yml +++ b/templates/argo-tasks/push-to-github.yml @@ -1,61 +1,59 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: # Template for Formatting and Pushing STAC Collections to Github # See https://github.com/linz/argo-tasks#stac-github-import - name: tpl-push-to-github + name: 'tpl-push-to-github' spec: templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' inputs: parameters: - - name: source - description: location where the source collection.json file is + - name: 'source' + description: 'location where the source collection.json file is' - - name: target - description: location where the destination collection.json file will be + - name: 'target' + description: 'location where the destination collection.json file will be' - - name: version_argo_tasks - description: version of argo-tasks to use + - name: 'version_argo_tasks' + description: 'version of argo-tasks to use' default: 'v4' - - name: repository - description: Repository Name + - name: 'repository' + description: 'Repository Name' - - name: ticket - description: Ticket ID e.g. 'AIP-55' + - name: 'ticket' + description: "Ticket ID e.g. 'AIP-55'" - - name: copy_option - description: --no-clobber Skip overwriting existing files. --force Overwrite all files. --force-no-clobber Overwrite only changed files, skip unchanged files. + - name: 'copy_option' + description: '--no-clobber Skip overwriting existing files. --force Overwrite all files. --force-no-clobber Overwrite only changed files, skip unchanged files.' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(inputs.parameters.version_argo_tasks)}}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json - - name: GITHUB_API_TOKEN + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' + - name: 'GITHUB_API_TOKEN' valueFrom: secretKeyRef: - name: github-linz-li-bot-pat - key: pat + name: 'github-linz-li-bot-pat' + key: 'pat' args: - [ - 'stac', - 'github-import', - '--source', - '{{=sprig.trim(inputs.parameters.source)}}', - '--target', - '{{inputs.parameters.target}}', - '--repo-name', - 'linz/{{=sprig.regexFind(inputs.parameters.repository, inputs.parameters.target)}}', - '--ticket', - '{{inputs.parameters.ticket}}', - '--copy-option', - '{{inputs.parameters.copy_option}}', - ] + - 'stac' + - 'github-import' + - '--source' + - '{{=sprig.trim(inputs.parameters.source)}}' + - '--target' + - '{{inputs.parameters.target}}' + - '--repo-name' + - 'linz/{{=sprig.regexFind(inputs.parameters.repository, inputs.parameters.target)}}' + - '--ticket' + - '{{inputs.parameters.ticket}}' + - '--copy-option' + - '{{inputs.parameters.copy_option}}' diff --git a/templates/argo-tasks/stac-setup.yml b/templates/argo-tasks/stac-setup.yml index dcd5bc0d1..602e70e9e 100644 --- a/templates/argo-tasks/stac-setup.yml +++ b/templates/argo-tasks/stac-setup.yml @@ -1,48 +1,48 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: # Template from linz/argo-tasks # see https://github.com/linz/argo-tasks#stac-setup - name: tpl-at-stac-setup + name: 'tpl-at-stac-setup' spec: templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' inputs: parameters: - - name: start_datetime + - name: 'start_datetime' description: '(Optional) Dataset capture start date in numeric format YYYY-MM-DD' default: '' - - name: end_datetime + - name: 'end_datetime' description: '(Optional) Dataset capture end date in numeric format YYYY-MM-DD' default: '' - - name: gsd + - name: 'gsd' description: 'Dataset GSD in metres' - - name: region + - name: 'region' description: 'Region of the dataset' - - name: geographic_description + - name: 'geographic_description' description: '(Optional) Additional dataset description' default: '' - - name: geospatial_category + - name: 'geospatial_category' description: 'Geospatial category of the dataset' - - name: odr_url + - name: 'odr_url' description: '(Optional) S3 path to existing dataset collection.json' default: '' - - name: version + - name: 'version' description: 'argo-task Container version to use' default: 'v4' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=inputs.parameters.version}}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - 'stac' - 'setup' @@ -55,11 +55,11 @@ spec: - '--gsd={{inputs.parameters.gsd}}' outputs: parameters: - - name: collection_id - description: ID for collection and item STAC + - name: 'collection_id' + description: 'ID for collection and item STAC' valueFrom: path: '/tmp/stac-setup/collection-id' - - name: linz_slug - description: linz:slug field for collection STAC + - name: 'linz_slug' + description: 'linz:slug field for collection STAC' valueFrom: path: '/tmp/stac-setup/linz-slug' diff --git a/templates/argo-tasks/stac-validate.yml b/templates/argo-tasks/stac-validate.yml index 78cb33f43..dee82ceab 100644 --- a/templates/argo-tasks/stac-validate.yml +++ b/templates/argo-tasks/stac-validate.yml @@ -1,54 +1,54 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: # Template from linz/argo-tasks # see https://github.com/linz/argo-tasks?tab=readme-ov-file#stac-validate - name: tpl-at-stac-validate + name: 'tpl-at-stac-validate' spec: templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' inputs: parameters: - - name: uri - description: STAC file uri to validate + - name: 'uri' + description: 'STAC file uri to validate' default: '' - - name: recursive - description: Follow and validate STAC links + - name: 'recursive' + description: 'Follow and validate STAC links' default: 'true' - - name: concurrency - description: Number of requests to run concurrently + - name: 'concurrency' + description: 'Number of requests to run concurrently' default: '50' - - name: checksum_assets - description: Validate the file:checksum of each asset if it exists + - name: 'checksum_assets' + description: 'Validate the file:checksum of each asset if it exists' default: 'false' - - name: checksum_links - description: Validate the file:checksum of each link if it exists + - name: 'checksum_links' + description: 'Validate the file:checksum of each link if it exists' default: 'false' - - name: version - description: container version to use + - name: 'version' + description: 'container version to use' default: 'v4' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(inputs.parameters.version)}}' resources: requests: - cpu: 15000m - memory: 7.8Gi + cpu: '15000m' + memory: '7.8Gi' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - 'stac' - 'validate' diff --git a/templates/argo-tasks/tile-index-validate.yml b/templates/argo-tasks/tile-index-validate.yml index 2898b1a62..6d7f5a482 100644 --- a/templates/argo-tasks/tile-index-validate.yml +++ b/templates/argo-tasks/tile-index-validate.yml @@ -1,75 +1,75 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: # Template from linz/argo-tasks # see https://github.com/linz/argo-tasks#tileindex-validate - name: tpl-at-tile-index-validate + name: 'tpl-at-tile-index-validate' spec: templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' inputs: parameters: - - name: scale - description: Tile Index scale to validate against or target scale for retiling + - name: 'scale' + description: 'Tile Index scale to validate against or target scale for retiling' - - name: source_epsg - description: Force source EPSG, "" will attempt to load EPSG from the source tiffs + - name: 'source_epsg' + description: 'Force source EPSG, "" will attempt to load EPSG from the source tiffs' default: '' - - name: source - description: Location to list + - name: 'source' + description: 'Location to list' default: '' - - name: include - description: Limit the input list to this regexp + - name: 'include' + description: 'Limit the input list to this regexp' default: '' - - name: includeDerived - description: Sets includeDerived=true/false in file-list.json + - name: 'includeDerived' + description: 'Sets includeDerived=true/false in file-list.json' default: 'false' - - name: validate - description: Validate that all input tiffs perfectly align to tile grid + - name: 'validate' + description: 'Validate that all input tiffs perfectly align to tile grid' default: 'true' - - name: retile - description: Output tile configuration for retiling + - name: 'retile' + description: 'Output tile configuration for retiling' default: 'false' - - name: preset - description: Compression type of dataset e.g. webp, lzw, dem_lerc + - name: 'preset' + description: 'Compression type of dataset e.g. webp, lzw, dem_lerc' default: 'webp' - - name: version - description: container version to use + - name: 'version' + description: 'container version to use' default: 'v4' outputs: artifacts: # List of tiff files that need to be processed - - name: files - path: /tmp/tile-index-validate/file-list.json + - name: 'files' + path: '/tmp/tile-index-validate/file-list.json' optional: true archive: none: {} # GeoJSON containing all the bounding box of all input tiffs - - name: input - path: /tmp/tile-index-validate/input.geojson + - name: 'input' + path: '/tmp/tile-index-validate/input.geojson' optional: true archive: none: {} # GeoJSON containing the bounding boxes of all the output tiffs - - name: output - path: /tmp/tile-index-validate/output.geojson + - name: 'output' + path: '/tmp/tile-index-validate/output.geojson' optional: true archive: none: {} @@ -77,8 +77,8 @@ spec: container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=inputs.parameters.version}}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - 'tileindex-validate' - '--verbose' @@ -92,4 +92,4 @@ spec: - '{{= sprig.trim(inputs.parameters.source) }}' resources: requests: - memory: 3.9Gi + memory: '3.9Gi' diff --git a/templates/common/exit.handler.yml b/templates/common/exit.handler.yml index 2733d5740..3693b0406 100644 --- a/templates/common/exit.handler.yml +++ b/templates/common/exit.handler.yml @@ -1,30 +1,31 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: # Template to log the workflow status and its parameters. - name: tpl-exit-handler + name: 'tpl-exit-handler' spec: templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' inputs: parameters: - - name: workflow_status - description: Status of the workflow - - name: workflow_parameters - description: Parameters of the workflow - - name: version_argo_tasks - description: Version of argo-tasks to use + - name: 'workflow_status' + description: 'Status of the workflow' + - name: 'workflow_parameters' + description: 'Parameters of the workflow' + - name: 'version_argo_tasks' + description: 'Version of argo-tasks to use' default: 'v4' script: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(inputs.parameters.version_argo_tasks)}}' - command: [node] + command: + - 'node' source: | let parameters = {}; {{= inputs.parameters.workflow_parameters }}.forEach((pair) => (parameters[pair.name] = pair.value)); diff --git a/templates/common/get.location.yml b/templates/common/get.location.yml index c33430076..82f144dc2 100644 --- a/templates/common/get.location.yml +++ b/templates/common/get.location.yml @@ -1,37 +1,38 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: # Template to output the locaton of the workflow S3 artifact root directory. - name: tpl-get-location + name: 'tpl-get-location' spec: templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' inputs: parameters: - - name: version_argo_tasks - description: Version of argo-tasks to use + - name: 'version_argo_tasks' + description: 'Version of argo-tasks to use' default: 'v4' outputs: parameters: - - name: location + - name: 'location' valueFrom: path: '/tmp/location' - - name: bucket + - name: 'bucket' valueFrom: path: '/tmp/bucket' - - name: key + - name: 'key' valueFrom: path: '/tmp/key' script: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(inputs.parameters.version_argo_tasks)}}' - command: [node] + command: + - 'node' source: | const fs = require('node:fs'); diff --git a/templates/gdal/gdalmerge.yaml b/templates/gdal/gdalmerge.yaml index c13b3ae43..db3c03d4e 100644 --- a/templates/gdal/gdalmerge.yaml +++ b/templates/gdal/gdalmerge.yaml @@ -1,59 +1,59 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: tpl-gdalmerge + name: 'tpl-gdalmerge' labels: - linz.govt.nz/category: util + linz.govt.nz/category: 'util' spec: nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' container: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:{{=sprig.trim(inputs.parameters.version_topo_imagery)}} - imagePullPolicy: Always - command: [gdal_merge.py] + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:{{=sprig.trim(inputs.parameters.version_topo_imagery)}}' + imagePullPolicy: 'Always' + command: + - 'gdal_merge.py' args: - [ - '-of', - 'GTiff', - '-ps', - '1', - '-1', - '-co', - 'COMPRESS=lerc', - '-co', - 'MAX_Z_ERROR=0', - '-co', - 'NUM_THREADS=ALL_CPUS', - '-o', - '{{inputs.parameters.merged_filename}}', - '{{= sprig.replace("s3://", "/vsis3_streaming/", inputs.parameters.base_filename)}}', - '{{= sprig.replace("s3://", "/vsis3_streaming/", inputs.parameters.additional_filename)}}', - ] + - '-of' + - 'GTiff' + - '-ps' + - '1' + - '-1' + - '-co' + - 'COMPRESS=lerc' + - '-co' + - 'MAX_Z_ERROR=0' + - '-co' + - 'NUM_THREADS=ALL_CPUS' + - '-o' + - '{{inputs.parameters.merged_filename}}' + - '{{= sprig.replace("s3://", "/vsis3_streaming/", inputs.parameters.base_filename)}}' + - '{{= sprig.replace("s3://", "/vsis3_streaming/", inputs.parameters.additional_filename)}}' + resources: requests: - memory: 12.0Gi + memory: '12.0Gi' inputs: parameters: - - name: base_filename + - name: 'base_filename' description: 'File to use as base layer, e.g. "s3://bucket/8m/CB13.tiff"' - - name: additional_filename + - name: 'additional_filename' description: 'File to use as top layer, e.g. "s3://bucket/1m/CB13.tiff". Use VRT files to layer multiple files. Note: VRT files must contain /vsis3/ or /vsis3_streaming/ paths instead of s3://.' - - name: version_topo_imagery + - name: 'version_topo_imagery' description: 'Specify a version of the topo-imagery image to use, e.g. "v7" or "latest"' default: 'v7' - - name: merged_filename + - name: 'merged_filename' description: 'Optional file name for output file, e.g. "CB13.tiff". Defaults to base_filename basename.' default: '{{= sprig.last(sprig.splitList("/", inputs.parameters.base_filename))}}' outputs: artifacts: - - name: merged_tiff + - name: 'merged_tiff' archive: none: {} path: '{{inputs.parameters.merged_filename}}' diff --git a/templates/topo-imagery/create-collection.yaml b/templates/topo-imagery/create-collection.yaml index 811e49722..073798c5a 100644 --- a/templates/topo-imagery/create-collection.yaml +++ b/templates/topo-imagery/create-collection.yaml @@ -1,100 +1,100 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: # Template from https://github.com/linz/topo-imagery # see https://github.com/linz/topo-imagery/blob/master/scripts/collection_from_items.py - name: tpl-ti-create-collection + name: 'tpl-ti-create-collection' spec: templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' inputs: parameters: - - name: version_topo_imagery + - name: 'version_topo_imagery' description: 'Specify a version of the topo-imagery image to use, e.g. "v4.8" or "latest"' default: 'v7' - - name: collection_id + - name: 'collection_id' description: 'Collection ID of existing dataset, or else a generated collection ID' default: '' - - name: linz_slug + - name: 'linz_slug' description: 'Dataset identifying slug' default: '' - - name: odr_url + - name: 'odr_url' description: '(Optional) If an existing dataset add the S3 path to the dataset here to load existing metadata e.g. "s3://nz-imagery/taranaki/new-plymouth_2017_0.1m/rgb/2193/"' default: '' - - name: location + - name: 'location' description: 'Location for output files' default: '' - - name: current_datetime + - name: 'current_datetime' description: 'Current date and time of the workflow' default: '' - - name: category + - name: 'category' description: 'Geospatial category of the dataset' default: '' - - name: region + - name: 'region' description: 'Region of the dataset' default: '' - - name: gsd + - name: 'gsd' description: 'Dataset GSD in metres, e.g., "0.3" for 30 centimetres' default: '' - - name: geographic_description + - name: 'geographic_description' description: '(Optional) Additional dataset description, to be used in the title in place of the Region, e.g. "Hamilton"' default: '' - - name: event + - name: 'event' description: '(Optional) Event name if dataset has been captured in association with an event, e.g. "Top of the South Floods"' default: '' - - name: historic_survey_number + - name: 'historic_survey_number' description: '(Optional) Survey Number associated with historical datasets, e.g. "SN8844"' default: '' - - name: lifecycle + - name: 'lifecycle' description: 'The release lifecycle status of the dataset, e.g. "completed or "ongoing"' default: '' - - name: add_title_suffix + - name: 'add_title_suffix' description: 'Add a title suffix to the collection title based on the lifecycle. For example, "[TITLE] - Preview"' default: 'false' - - name: producer + - name: 'producer' description: 'The producer of the source dataset, e.g. aerial or bathymetric survey company' default: '' - - name: producer_list + - name: 'producer_list' description: '(Optional) List of imagery producers, separated by semicolon (;). Has no effect unless a semicolon delimited list is entered.' default: '' - - name: licensor + - name: 'licensor' description: 'The licensor of the dataset, e.g. local or regional council, government agency, satellite provider' default: '' - - name: licensor_list + - name: 'licensor_list' description: '(Optional) List of imagery licensors, separated by semicolon (;). Has no effect unless a semicolon delimited list is entered.' default: '' - - name: create_capture_dates + - name: 'create_capture_dates' description: 'Add a capture-dates.geojson.gz file to the collection assets' default: 'false' outputs: artifacts: - - name: capture-area + - name: 'capture-area' path: '/tmp/capture-area.geojson' optional: true archive: @@ -103,7 +103,7 @@ spec: container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:{{=sprig.trim(inputs.parameters.version_topo_imagery)}}' args: - - python + - 'python' - '/app/scripts/collection_from_items.py' - '--uri' - '{{=sprig.trimSuffix("/", inputs.parameters.location)}}' @@ -145,5 +145,5 @@ spec: - '{{inputs.parameters.current_datetime}}' resources: requests: - memory: 7.8Gi - cpu: 15000m + memory: '7.8Gi' + cpu: '15000m' diff --git a/templates/topo-imagery/create-hillshade.yaml b/templates/topo-imagery/create-hillshade.yaml index 62d464261..a5755611b 100644 --- a/templates/topo-imagery/create-hillshade.yaml +++ b/templates/topo-imagery/create-hillshade.yaml @@ -1,150 +1,150 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: # Template calling linz/topo-imagery for its main task (generate-hillshade) # see https://github.com/linz/topo-imagery/pull/1253 - name: tpl-create-hillshade + name: 'tpl-create-hillshade' labels: - linz.govt.nz/category: raster + linz.govt.nz/category: 'raster' spec: templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' inputs: parameters: - - name: source + - name: 'source' description: 's3 path to where the TIFF files to generate hillshade from are stored.' - - name: scale + - name: 'scale' description: 'Scale of the source data.' - - name: group + - name: 'group' description: 'Number of tiles to group together.' - - name: version_argo_tasks + - name: 'version_argo_tasks' description: 'Version of the argo-tasks image to use.' default: 'v4' - - name: version_topo_imagery + - name: 'version_topo_imagery' description: 'Version of the topo-imagery image to use.' default: 'v7' - - name: target + - name: 'target' description: 's3 path to where the hillshade tiles will be stored. The different hillshade types will be stored in subdirectories using the preset name.' - - name: preset + - name: 'preset' description: 'List of hillshade types to generate. Expect a list of strings.' default: '["greyscale", "igor"]' dag: tasks: - - name: tile-index-validate + - name: 'tile-index-validate' templateRef: - name: tpl-at-tile-index-validate - template: main + name: 'tpl-at-tile-index-validate' + template: 'main' arguments: parameters: - - name: scale + - name: 'scale' value: '{{inputs.parameters.scale}}' - - name: include + - name: 'include' value: '.tiff?$' - - name: source + - name: 'source' value: '{{=sprig.trim(inputs.parameters.source)}}' - - name: source_epsg + - name: 'source_epsg' value: '2193' - - name: validate + - name: 'validate' value: 'false' - - name: retile + - name: 'retile' value: 'true' - - name: preset + - name: 'preset' value: 'dem_lerc' - - name: version + - name: 'version' value: '{{= inputs.parameters.version_argo_tasks}}' - - name: group + - name: 'group' templateRef: - name: tpl-at-group - template: main + name: 'tpl-at-group' + template: 'main' arguments: artifacts: - - name: input + - name: 'input' from: '{{ tasks.tile-index-validate.outputs.artifacts.files }}' parameters: - - name: size + - name: 'size' value: '{{inputs.parameters.group}}' - - name: version + - name: 'version' value: '{{= inputs.parameters.version_argo_tasks}}' depends: 'tile-index-validate' - - name: hillshade-preset - template: hillshade-preset + - name: 'hillshade-preset' + template: 'hillshade-preset' arguments: parameters: - - name: group_id + - name: 'group_id' value: '{{item}}' - - name: target + - name: 'target' value: '{{inputs.parameters.target}}' - - name: preset + - name: 'preset' value: '{{inputs.parameters.preset}}' - - name: version_topo_imagery + - name: 'version_topo_imagery' value: '{{inputs.parameters.version_topo_imagery}}' artifacts: - - name: group_data + - name: 'group_data' from: '{{ tasks.group.outputs.artifacts.output }}' depends: 'group' withParam: '{{ tasks.group.outputs.parameters.output }}' - - name: hillshade-preset + - name: 'hillshade-preset' inputs: parameters: - - name: group_id - - name: target - - name: preset - - name: version_topo_imagery + - name: 'group_id' + - name: 'target' + - name: 'preset' + - name: 'version_topo_imagery' artifacts: - - name: group_data + - name: 'group_data' steps: - - - name: generate-hillshade - template: generate-hillshade + - - name: 'generate-hillshade' + template: 'generate-hillshade' arguments: parameters: - - name: group_id + - name: 'group_id' value: '{{inputs.parameters.group_id}}' - - name: target + - name: 'target' value: '{{inputs.parameters.target}}' - - name: preset + - name: 'preset' value: '{{item}}' - - name: version_topo_imagery + - name: 'version_topo_imagery' value: '{{inputs.parameters.version_topo_imagery}}' artifacts: - - name: group_data + - name: 'group_data' from: '{{inputs.artifacts.group_data}}' withParam: '{{inputs.parameters.preset}}' - - name: generate-hillshade + - name: 'generate-hillshade' nodeSelector: karpenter.sh/capacity-type: 'spot' inputs: parameters: - - name: group_id - - name: target - - name: preset - - name: version_topo_imagery + - name: 'group_id' + - name: 'target' + - name: 'preset' + - name: 'version_topo_imagery' artifacts: - - name: group_data - path: /tmp/input/ + - name: 'group_data' + path: '/tmp/input/' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:{{=sprig.trim(inputs.parameters.version_topo_imagery)}}' resources: requests: - memory: 7.8Gi - cpu: 15000m - ephemeral-storage: 29.5Gi + memory: '7.8Gi' + cpu: '15000m' + ephemeral-storage: '29.5Gi' volumeMounts: # This volume must be defined in the workflow calling this workflowTemplate - - name: ephemeral + - name: 'ephemeral' mountPath: '/tmp' args: - - python + - 'python' - '/app/scripts/generate_hillshade.py' - '--from-file' - '/tmp/input/{{inputs.parameters.group_id}}.json' diff --git a/templates/topo-imagery/standardise-validate.yaml b/templates/topo-imagery/standardise-validate.yaml index 666aa6537..5042dadd0 100644 --- a/templates/topo-imagery/standardise-validate.yaml +++ b/templates/topo-imagery/standardise-validate.yaml @@ -1,92 +1,92 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: # Template from https://github.com/linz/topo-imagery # see https://github.com/linz/topo-imagery/blob/master/scripts/standardise_validate.py - name: tpl-ti-standardise-validate + name: 'tpl-ti-standardise-validate' spec: templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' - entrypoint: main + entrypoint: 'main' templates: - - name: main + - name: 'main' inputs: parameters: - - name: version_topo_imagery + - name: 'version_topo_imagery' description: 'Specify a version of the topo-imagery image to use, e.g. "v4.8" or "latest"' default: 'v7' - - name: group_id + - name: 'group_id' description: 'The ID of the group of files to use as the input.' default: '' - - name: odr_url + - name: 'odr_url' description: '(Optional) If an existing dataset add the S3 path to the dataset here to load existing metadata e.g. "s3://nz-imagery/taranaki/new-plymouth_2017_0.1m/rgb/2193/"' default: '' - - name: target + - name: 'target' description: 'Location for output files' default: '' - - name: collection_id + - name: 'collection_id' description: 'Collection ID of existing dataset, or else a generated collection ID' default: '' - - name: compression + - name: 'compression' description: 'Compression type to use when standardising TIFFs, e.g. "webp" for imagery or "dem_lerc" for elevation data' default: '' - - name: current_datetime + - name: 'current_datetime' description: 'Current date and time of the workflow' default: '' - - name: start_datetime + - name: 'start_datetime' description: 'Dataset capture start date in numeric format YYYY-MM-DD, e.g. "2024-01-14"' default: '' - - name: end_datetime + - name: 'end_datetime' description: 'Dataset capture end date in numeric format YYYY-MM-DD, e.g. "2024-02-23"' default: '' - - name: create_capture_area + - name: 'create_capture_area' description: 'Create a capture area GeoJSON file for the standardised dataset' default: '' - - name: cutline + - name: 'cutline' description: '(Optional) location of a cutline file to cut the imagery to .fgb or .geojson' default: '' - - name: gsd + - name: 'gsd' description: 'Dataset GSD in metres, e.g., "0.3" for 30 centimetres' default: '' - - name: scale_to_resolution + - name: 'scale_to_resolution' description: 'Scale output TIFFs to x,y resolution (e.g. 1,1 - leave blank for no scaling)' default: '' - - name: source_epsg + - name: 'source_epsg' description: 'EPSG of the source files' default: '2193' - - name: target_epsg + - name: 'target_epsg' description: 'EPSG of the standardised output files' default: '2193' artifacts: - - name: group_data - path: /tmp/input/ + - name: 'group_data' + path: '/tmp/input/' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:{{=sprig.trim(inputs.parameters.version_topo_imagery)}}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - - python + - 'python' - '/app/scripts/standardise_validate.py' - '--from-file' - '/tmp/input/{{inputs.parameters.group_id}}.json' @@ -118,10 +118,10 @@ spec: - '{{=sprig.trim(inputs.parameters.scale_to_resolution)}}' resources: requests: - memory: 7.8Gi - cpu: 15000m - ephemeral-storage: 29.5Gi + memory: '7.8Gi' + cpu: '15000m' + ephemeral-storage: '29.5Gi' volumeMounts: # This volume must be defined in the workflow calling this workflowTemplate - - name: ephemeral + - name: 'ephemeral' mountPath: '/tmp' diff --git a/workflows/basemaps/create-config.yaml b/workflows/basemaps/create-config.yaml index 80b828b61..deda75554 100644 --- a/workflows/basemaps/create-config.yaml +++ b/workflows/basemaps/create-config.yaml @@ -1,59 +1,61 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: basemaps-config-create + name: 'basemaps-config-create' labels: - linz.govt.nz/category: basemaps + linz.govt.nz/category: 'basemaps' spec: - entrypoint: main + entrypoint: 'main' podMetadata: labels: - linz.govt.nz/category: basemaps + linz.govt.nz/category: 'basemaps' arguments: parameters: - - name: version_basemaps_cli + - name: 'version_basemaps_cli' value: 'v7' - - name: location + - name: 'location' value: 's3://bucket/path/to' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' dag: tasks: - - name: create-config - template: create-config + - name: 'create-config' + template: 'create-config' arguments: parameters: - - name: location + - name: 'location' value: '{{workflow.parameters.location}}' - - name: create-config + - name: 'create-config' inputs: parameters: - - name: location - description: Location of the imagery to create config for + - name: 'location' + description: 'Location of the imagery to create config for' container: - image: ghcr.io/linz/basemaps/cli:{{=sprig.trim(workflow.parameters.version_basemaps_cli)}} - command: [node, /app/node_modules/@basemaps/cogify/dist/index.cjs] + image: 'ghcr.io/linz/basemaps/cli:{{=sprig.trim(workflow.parameters.version_basemaps_cli)}}' + command: + - 'node' + - '/app/node_modules/@basemaps/cogify/dist/index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.basemaps.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.basemaps.json' args: - 'config' - '{{ inputs.parameters.location }}' outputs: parameters: - - name: url - description: Basemaps URL to view the imagery + - name: 'url' + description: 'Basemaps URL to view the imagery' valueFrom: path: '/tmp/cogify/config-url' - - name: config - description: Location of the config file + - name: 'config' + description: 'Location of the config file' valueFrom: path: '/tmp/cogify/config-path' diff --git a/workflows/basemaps/create-overview-all.yaml b/workflows/basemaps/create-overview-all.yaml index 7f87b53f2..183207d9e 100644 --- a/workflows/basemaps/create-overview-all.yaml +++ b/workflows/basemaps/create-overview-all.yaml @@ -1,81 +1,82 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: Workflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'Workflow' metadata: - name: create-all-overviews- - namespace: argo + name: 'create-all-overviews-' + namespace: 'argo' labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'raster' spec: parallelism: 20 nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main + entrypoint: 'main' podMetadata: labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'raster' arguments: parameters: - - name: version_basemaps_cli + - name: 'version_basemaps_cli' value: 'v6' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' dag: tasks: - - name: list-imagery - template: list-imagery - - name: create-overview - template: create-overview + - name: 'list-imagery' + template: 'list-imagery' + - name: 'create-overview' + template: 'create-overview' arguments: parameters: - - name: source + - name: 'source' value: '{{item}}' depends: 'list-imagery' withParam: '{{tasks.list-imagery.outputs.parameters.imagery}}' - - name: list-imagery + - name: 'list-imagery' container: - image: peterevans/curl-jq:latest - command: [sh, -c] + image: 'peterevans/curl-jq:latest' + command: + - 'sh' + - '-c' args: - [ - 'curl https://raw.githubusercontent.com/linz/basemaps-config/master/config/tileset/aerial.json | jq ''.layers[] | [.["2193"], .["3857"]] | flatten[]'' -r | sort | uniq | grep -v "null" | jq -ncR ''[inputs]'' -r > /tmp/imagery.json', - ] + - 'curl https://raw.githubusercontent.com/linz/basemaps-config/master/config/tileset/aerial.json | jq ''.layers[] | [.["2193"], .["3857"]] | flatten[]'' -r | sort | uniq | grep -v "null" | jq -ncR ''[inputs]'' -r > /tmp/imagery.json' + outputs: parameters: - - name: imagery + - name: 'imagery' valueFrom: path: '/tmp/imagery.json' - - name: create-overview + - name: 'create-overview' inputs: parameters: - - name: source + - name: 'source' container: - image: ghcr.io/linz/basemaps/cli:{{workflow.parameters.version_basemaps_cli}} + image: 'ghcr.io/linz/basemaps/cli:{{workflow.parameters.version_basemaps_cli}}' resources: requests: - cpu: 3000m - memory: 7.8Gi - command: [node, index.cjs] + cpu: '3000m' + memory: '7.8Gi' + command: + - 'node' + - 'index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.basemaps.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.basemaps.json' args: - [ - '-V', - 'create-overview', - '--source', - '{{inputs.parameters.source}}', - '--output', - '{{inputs.parameters.source}}', - ] + - '-V' + - 'create-overview' + - '--source' + - '{{inputs.parameters.source}}' + - '--output' + - '{{inputs.parameters.source}}' diff --git a/workflows/basemaps/create-overview.yaml b/workflows/basemaps/create-overview.yaml index 5d554ee44..a073bb249 100644 --- a/workflows/basemaps/create-overview.yaml +++ b/workflows/basemaps/create-overview.yaml @@ -1,66 +1,66 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: Workflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'Workflow' metadata: - name: basemaps-create-overview- - namespace: argo + name: 'basemaps-create-overview-' + namespace: 'argo' labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'raster' spec: - entrypoint: main + entrypoint: 'main' podMetadata: labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'raster' arguments: parameters: - - name: version_basemaps_cli + - name: 'version_basemaps_cli' value: 'v6' - - name: source + - name: 'source' value: '' - - name: output + - name: 'output' value: '' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' dag: tasks: - - name: create-overview - template: create-overview + - name: 'create-overview' + template: 'create-overview' arguments: parameters: - - name: source + - name: 'source' value: '{{workflow.parameters.source}}' - - name: output + - name: 'output' value: '{{workflow.parameters.output}}' - - name: create-overview + - name: 'create-overview' inputs: parameters: - - name: source - - name: output + - name: 'source' + - name: 'output' container: - image: ghcr.io/linz/basemaps/cli:{{workflow.parameters.version_basemaps_cli}} + image: 'ghcr.io/linz/basemaps/cli:{{workflow.parameters.version_basemaps_cli}}' resources: requests: - cpu: 3000m - memory: 7.8Gi - command: [node, index.cjs] + cpu: '3000m' + memory: '7.8Gi' + command: + - 'node' + - 'index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.basemaps.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.basemaps.json' args: - [ - '-V', - 'create-overview', - '--source', - '{{inputs.parameters.source}}', - '--output', - '{{inputs.parameters.output}}', - ] + - '-V' + - 'create-overview' + - '--source' + - '{{inputs.parameters.source}}' + - '--output' + - '{{inputs.parameters.output}}' diff --git a/workflows/basemaps/imagery-import-cogify.yml b/workflows/basemaps/imagery-import-cogify.yml index 26ffdd89b..4a9426690 100644 --- a/workflows/basemaps/imagery-import-cogify.yml +++ b/workflows/basemaps/imagery-import-cogify.yml @@ -1,72 +1,72 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: basemaps-imagery-import-cogify + name: 'basemaps-imagery-import-cogify' labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'raster' spec: parallelism: 100 - entrypoint: main - onExit: exit-handler + entrypoint: 'main' + onExit: 'exit-handler' synchronization: semaphore: configMapKeyRef: - name: semaphores - key: basemaps_import + name: 'semaphores' + key: 'basemaps_import' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' workflowMetadata: labelsFrom: linz.govt.nz/user-group: - expression: workflow.parameters.user_group + expression: 'workflow.parameters.user_group' linz.govt.nz/ticket: - expression: workflow.parameters.ticket + expression: 'workflow.parameters.ticket' linz.govt.nz/region: - expression: workflow.parameters.region + expression: 'workflow.parameters.region' podMetadata: labels: linz.govt.nz/user-group: '{{workflow.parameters.user_group}}' - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'raster' linz.govt.nz/ticket: '{{workflow.parameters.ticket}}' linz.govt.nz/region: '{{workflow.parameters.region}}' arguments: parameters: - - name: version_basemaps_cli - description: Version of the basemaps CLI docker container to use - value: v7 + - name: 'version_basemaps_cli' + description: 'Version of the basemaps CLI docker container to use' + value: 'v7' - - name: version_argo_tasks - description: Version of the basemaps CLI docker container to use - value: v4 + - name: 'version_argo_tasks' + description: 'Version of the basemaps CLI docker container to use' + value: 'v4' - - name: user_group - description: Group of users running the workflow + - name: 'user_group' + description: 'Group of users running the workflow' value: 'none' enum: - 'land' - 'sea' - 'none' - - name: ticket - description: Ticket ID e.g. 'AIP-55' + - name: 'ticket' + description: "Ticket ID e.g. 'AIP-55'" value: '' - - name: preset - description: Import preset configuration, WebP for 4 band RGBA LERC for 1 band DEM/DSM + - name: 'preset' + description: 'Import preset configuration, WebP for 4 band RGBA LERC for 1 band DEM/DSM' value: 'webp' enum: - 'webp' - 'lerc_10mm' - 'lerc_1mm' - - name: region - description: Region of the dataset + - name: 'region' + description: 'Region of the dataset' value: 'new-zealand' enum: - 'antarctica' @@ -90,39 +90,39 @@ spec: - 'wellington' - 'west-coast' - - name: source - description: Source imagery location "s3://linz-imagery" + - name: 'source' + description: 'Source imagery location "s3://linz-imagery"' value: 's3://linz-imagery-staging/test/sample/' - - name: require_stac_collection - description: Validate that a STAC collection.json exists with the source + - name: 'require_stac_collection' + description: 'Validate that a STAC collection.json exists with the source' value: 'true' enum: - 'true' - 'false' - - name: create_pull_request + - name: 'create_pull_request' description: 'Create pull request after importing imagery.' value: 'true' enum: - 'true' - 'false' - - name: config_type + - name: 'config_type' description: 'Type of Basemaps Config file to create in pull requests' value: 'raster' enum: - 'raster' - 'elevation' - - name: individual + - name: 'individual' description: 'Individual Config or Combined Config' value: 'combined' enum: - 'individual' - 'combined' - - name: category + - name: 'category' value: 'Rural Aerial Photos' enum: - 'Rural Aerial Photos' @@ -132,214 +132,216 @@ spec: - 'Event' - 'Elevation' - - name: target - description: Target location for output COGs + - name: 'target' + description: 'Target location for output COGs' value: 's3://linz-basemaps/' enum: - 's3://linz-basemaps/' - 's3://linz-basemaps-staging/' - 's3://linz-workflowsnp-scratch/' - - name: tile_matrix - description: Output tile matrix, ";" separated list + - name: 'tile_matrix' + description: 'Output tile matrix, ";" separated list' value: 'NZTM2000Quad;WebMercatorQuad' enum: - 'NZTM2000Quad' - 'WebMercatorQuad' - 'NZTM2000Quad;WebMercatorQuad' - - name: cutline - description: Path to cutline to apply + - name: 'cutline' + description: 'Path to cutline to apply' value: 's3://linz-basemaps-source/cutline/2020-05-07-cutline-nz-coasts-rural-and-urban.geojson' - - name: cutline_blend - description: Blending to use for cutline see gdal_translate#cblend + - name: 'cutline_blend' + description: 'Blending to use for cutline see gdal_translate#cblend' value: '20' - - name: group_size - description: How many items to pass to each create-cog job + - name: 'group_size' + description: 'How many items to pass to each create-cog job' value: '20' - - name: create_overview + - name: 'create_overview' description: 'Create overview after importing imagery.' value: 'true' enum: - 'true' - 'false' - - name: background + - name: 'background' description: 'Background RGBA hexstring to fill empty space in the COG. Format: "#rrggbbaa"' value: '' templates: # Main entrypoint into the workflow - - name: main + - name: 'main' retryStrategy: expression: 'false' inputs: parameters: - - name: source - - name: target - - name: preset - - name: tile_matrix - - name: cutline - - name: cutline_blend - - name: group_size - - name: require_stac_collection - - name: background + - name: 'source' + - name: 'target' + - name: 'preset' + - name: 'tile_matrix' + - name: 'cutline' + - name: 'cutline_blend' + - name: 'group_size' + - name: 'require_stac_collection' + - name: 'background' dag: tasks: # For each tile matrix specified run the "cogify" template - - name: cogify - template: cogify + - name: 'cogify' + template: 'cogify' withParam: "{{= toJson(sprig.splitList(';', inputs.parameters.tile_matrix)) }}" arguments: parameters: - - name: source + - name: 'source' value: '{{ inputs.parameters.source }}' - - name: target + - name: 'target' value: '{{ inputs.parameters.target }}' - - name: preset + - name: 'preset' value: '{{ inputs.parameters.preset }}' - - name: tile_matrix + - name: 'tile_matrix' value: '{{ item }}' - - name: cutline + - name: 'cutline' value: '{{ inputs.parameters.cutline }}' - - name: cutline_blend + - name: 'cutline_blend' value: '{{ inputs.parameters.cutline_blend }}' - - name: group_size + - name: 'group_size' value: '{{ inputs.parameters.group_size }}' - - name: require_stac_collection + - name: 'require_stac_collection' value: '{{ inputs.parameters.require_stac_collection }}' - - name: background + - name: 'background' value: '{{ inputs.parameters.background }}' - - name: create-pull-request - template: create-pull-request + - name: 'create-pull-request' + template: 'create-pull-request' arguments: parameters: - - name: target + - name: 'target' value: '{{ tasks.cogify.outputs.parameters.target }}' when: '{{workflow.parameters.create_pull_request}} == true' depends: 'cogify' # Generate COGs for a specific tile matrix from a given collection of source imagery - - name: cogify + - name: 'cogify' inputs: parameters: - - name: source - - name: target - - name: tile_matrix - - name: cutline - - name: cutline_blend - - name: group_size - - name: preset - - name: require_stac_collection - - name: background + - name: 'source' + - name: 'target' + - name: 'tile_matrix' + - name: 'cutline' + - name: 'cutline_blend' + - name: 'group_size' + - name: 'preset' + - name: 'require_stac_collection' + - name: 'background' dag: tasks: # generate a tile covering from the source imagery - - name: create-covering - template: create-covering + - name: 'create-covering' + template: 'create-covering' arguments: parameters: - - name: source + - name: 'source' value: '{{ inputs.parameters.source }}' - - name: target + - name: 'target' value: '{{ inputs.parameters.target }}' - - name: preset + - name: 'preset' value: '{{ inputs.parameters.preset }}' - - name: tile_matrix + - name: 'tile_matrix' value: '{{ inputs.parameters.tile_matrix }}' - - name: cutline + - name: 'cutline' value: '{{ inputs.parameters.cutline }}' - - name: cutline_blend + - name: 'cutline_blend' value: '{{ inputs.parameters.cutline_blend }}' - - name: require_stac_collection + - name: 'require_stac_collection' value: '{{ inputs.parameters.require_stac_collection }}' - - name: background + - name: 'background' value: '{{ inputs.parameters.background }}' # Group covering output into chunks to pass to create-cog - - name: group + - name: 'group' arguments: parameters: - - name: size + - name: 'size' value: '{{ inputs.parameters.group_size }}' - - name: version + - name: 'version' value: '{{= workflow.parameters.version_argo_tasks }}' artifacts: - - name: input + - name: 'input' from: '{{ tasks.create-covering.outputs.artifacts.tiles }}' templateRef: - name: tpl-at-group - template: main - depends: create-covering + name: 'tpl-at-group' + template: 'main' + depends: 'create-covering' # Create COGS from the grouped output of create-covering - - name: create-cog - depends: group - template: create-cog + - name: 'create-cog' + depends: 'group' + template: 'create-cog' withParam: '{{ tasks.group.outputs.parameters.output }}' arguments: parameters: - - name: covering_grouped_id + - name: 'covering_grouped_id' value: '{{ item }}' artifacts: - - name: covering_grouped + - name: 'covering_grouped' from: '{{ tasks.group.outputs.artifacts.output }}' # TODO: overviews are only supported in RGBA pipelines # once all COGs are created generate a more overviews to increase tile rendering performance - - name: create-overview - template: create-overview + - name: 'create-overview' + template: 'create-overview' when: '{{workflow.parameters.create_overview}} == true' - depends: create-cog + depends: 'create-cog' arguments: parameters: - - name: path + - name: 'path' value: '{{ tasks.create-covering.outputs.parameters.target }}' # Create a basemaps config to view the imagery directly - - name: create-config - template: create-config - depends: create-overview + - name: 'create-config' + template: 'create-config' + depends: 'create-overview' arguments: parameters: - - name: path + - name: 'path' value: '{{ tasks.create-covering.outputs.parameters.target }}' - - name: title + - name: 'title' value: '{{ tasks.create-covering.outputs.parameters.title }}' outputs: parameters: - - name: target - description: location of where the output tiffs were created + - name: 'target' + description: 'location of where the output tiffs were created' valueFrom: parameter: '{{ tasks.create-covering.outputs.parameters.target }}' # Generate a tile covering for input imagery - - name: create-covering + - name: 'create-covering' nodeSelector: karpenter.sh/capacity-type: 'spot' inputs: parameters: - - name: source - - name: target - - name: tile_matrix - - name: cutline - - name: cutline_blend - - name: preset - - name: require_stac_collection - - name: background + - name: 'source' + - name: 'target' + - name: 'tile_matrix' + - name: 'cutline' + - name: 'cutline_blend' + - name: 'preset' + - name: 'require_stac_collection' + - name: 'background' container: - image: ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }} + image: 'ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }}' resources: requests: - memory: 2Gi - command: [node, /app/node_modules/@basemaps/cogify/dist/index.cjs] + memory: '2Gi' + command: + - 'node' + - '/app/node_modules/@basemaps/cogify/dist/index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.basemaps.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.basemaps.json' args: - 'cover' - '--preset={{ inputs.parameters.preset }}' @@ -352,110 +354,116 @@ spec: - '{{= sprig.trim(inputs.parameters.source) }}' outputs: parameters: - - name: target - description: output path for where the covering was written + - name: 'target' + description: 'output path for where the covering was written' valueFrom: - path: /tmp/cogify/cover-target + path: '/tmp/cogify/cover-target' - - name: title - description: Title for the covering + - name: 'title' + description: 'Title for the covering' valueFrom: - path: /tmp/cogify/cover-title + path: '/tmp/cogify/cover-title' artifacts: - - name: tiles - path: /tmp/cogify/cover-items.json + - name: 'tiles' + path: '/tmp/cogify/cover-items.json' # Actually create COGs using gdal_translate on a large spot instances - - name: create-cog + - name: 'create-cog' nodeSelector: karpenter.sh/capacity-type: 'spot' inputs: artifacts: - - name: covering_grouped - path: /tmp/cogify/covering_grouped/ + - name: 'covering_grouped' + path: '/tmp/cogify/covering_grouped/' parameters: - - name: covering_grouped_id + - name: 'covering_grouped_id' container: resources: requests: - memory: 30Gi - cpu: 15000m # AWS gives 2x cpu cores = memory for most instances - ephemeral-storage: 98Gi # 2 pods per 200GB of storage - image: ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }} - command: [node, /app/node_modules/@basemaps/cogify/dist/index.cjs] + memory: '30Gi' + cpu: '15000m' # AWS gives 2x cpu cores = memory for most instances + ephemeral-storage: '98Gi' # 2 pods per 200GB of storage + image: 'ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }}' + command: + - 'node' + - '/app/node_modules/@basemaps/cogify/dist/index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.basemaps.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.basemaps.json' args: - 'create' - '--from-file={{= inputs.artifacts.covering_grouped.path }}{{inputs.parameters.covering_grouped_id}}.json' - '--concurrency=2' # Create a basemaps configuration file to view the imagery - - name: create-config + - name: 'create-config' inputs: parameters: - - name: path - description: Location of the imagery to create config for + - name: 'path' + description: 'Location of the imagery to create config for' container: - image: ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }} - command: [node, /app/node_modules/@basemaps/cogify/dist/index.cjs] + image: 'ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }}' + command: + - 'node' + - '/app/node_modules/@basemaps/cogify/dist/index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.basemaps.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.basemaps.json' args: - 'config' - '{{ inputs.parameters.path }}' outputs: parameters: - - name: url - description: Basemaps URL to view the imagery + - name: 'url' + description: 'Basemaps URL to view the imagery' valueFrom: path: '/tmp/cogify/config-url' - - name: config - description: Location of the config file + - name: 'config' + description: 'Location of the config file' valueFrom: path: '/tmp/cogify/config-path' # create additional overviews for any COGs found in the path - - name: create-overview + - name: 'create-overview' nodeSelector: karpenter.sh/capacity-type: 'spot' inputs: parameters: - - name: path - description: Location of the imagery to create overviews for + - name: 'path' + description: 'Location of the imagery to create overviews for' container: resources: requests: - memory: 7.8Gi - cpu: 15000m - image: ghcr.io/linz/basemaps/cli:v6 - command: [node, index.cjs] + memory: '7.8Gi' + cpu: '15000m' + image: 'ghcr.io/linz/basemaps/cli:v6' + command: + - 'node' + - 'index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.basemaps.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.basemaps.json' args: - '-V' - 'create-overview' - '--source={{= sprig.trim(inputs.parameters.path) }}' - '--output={{= sprig.trim(inputs.parameters.path) }}' - - name: create-pull-request + - name: 'create-pull-request' inputs: parameters: - - name: target + - name: 'target' container: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{workflow.parameters.version_argo_tasks}} + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{workflow.parameters.version_argo_tasks}}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.basemaps.json - - name: GITHUB_API_TOKEN + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.basemaps.json' + - name: 'GITHUB_API_TOKEN' valueFrom: secretKeyRef: - name: github-linz-li-bot-pat - key: pat + name: 'github-linz-li-bot-pat' + key: 'pat' args: - 'bmc' - 'create-pr' @@ -465,17 +473,17 @@ spec: - "--individual={{= workflow.parameters.individual == 'individual'? 'true' : 'false' }}" - '--category={{workflow.parameters.category}}' - - name: exit-handler + - name: 'exit-handler' retryStrategy: limit: '0' # `tpl-exit-handler` retries itself steps: - - - name: exit + - - name: 'exit' templateRef: - name: tpl-exit-handler - template: main + name: 'tpl-exit-handler' + template: 'main' arguments: parameters: - - name: workflow_status + - name: 'workflow_status' value: '{{workflow.status}}' - - name: workflow_parameters + - name: 'workflow_parameters' value: '{{workflow.parameters}}' diff --git a/workflows/basemaps/mapsheet-json.yaml b/workflows/basemaps/mapsheet-json.yaml index 9d9119926..6edbc4dc0 100644 --- a/workflows/basemaps/mapsheet-json.yaml +++ b/workflows/basemaps/mapsheet-json.yaml @@ -1,118 +1,123 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: basemaps-create-mapsheet-json + name: 'basemaps-create-mapsheet-json' labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: raster - linz.govt.nz/data-sub-type: maps + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'raster' + linz.govt.nz/data-sub-type: 'maps' spec: - entrypoint: main + entrypoint: 'main' podMetadata: labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: raster - linz.govt.nz/data-sub-type: maps + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'raster' + linz.govt.nz/data-sub-type: 'maps' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' value: 'v4' - - name: layer + - name: 'layer' value: '104687' - - name: config + - name: 'config' value: 's3://linz-basemaps/config/config-latest.json.gz' - - name: include + - name: 'include' value: '' - - name: exclude + - name: 'exclude' value: 'new-zealand' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' dag: tasks: - - name: fetch-layer - template: fetch-layer + - name: 'fetch-layer' + template: 'fetch-layer' arguments: parameters: - - name: layer + - name: 'layer' value: '{{workflow.parameters.layer}}' - - name: transform - template: transform + - name: 'transform' + template: 'transform' arguments: artifacts: - - name: geopackage + - name: 'geopackage' from: '{{tasks.fetch-layer.outputs.artifacts.geopackage}}' depends: 'fetch-layer' - - name: create-mapsheet - template: create-mapsheet + - name: 'create-mapsheet' + template: 'create-mapsheet' arguments: artifacts: - - name: flatGeobuf + - name: 'flatGeobuf' from: '{{tasks.transform.outputs.artifacts.flatGeobuf}}' depends: 'transform' - - name: fetch-layer + - name: 'fetch-layer' inputs: parameters: - - name: layer + - name: 'layer' container: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}} + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json - args: ['lds-fetch-layer', '{{=sprig.trim(inputs.parameters.layer)}}', '--target=/tmp/lds-layer'] + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' + args: + - 'lds-fetch-layer' + - '{{=sprig.trim(inputs.parameters.layer)}}' + - '--target=/tmp/lds-layer' outputs: artifacts: - - name: geopackage - path: /tmp/lds-layer/ + - name: 'geopackage' + path: '/tmp/lds-layer/' - - name: transform + - name: 'transform' inputs: artifacts: - - name: geopackage - path: /tmp/ + - name: 'geopackage' + path: '/tmp/' container: - image: ghcr.io/osgeo/gdal:alpine-small-3.7.0 - imagePullPolicy: IfNotPresent - command: [sh] - args: ['-c', 'ogr2ogr -f FlatGeobuf tmp/flatGeobuf.fgb tmp/*.gpkg'] + image: 'ghcr.io/osgeo/gdal:alpine-small-3.7.0' + imagePullPolicy: 'IfNotPresent' + command: + - 'sh' + args: + - '-c' + - 'ogr2ogr -f FlatGeobuf tmp/flatGeobuf.fgb tmp/*.gpkg' outputs: artifacts: - - name: flatGeobuf - path: /tmp/flatGeobuf.fgb + - name: 'flatGeobuf' + path: '/tmp/flatGeobuf.fgb' - - name: create-mapsheet + - name: 'create-mapsheet' inputs: artifacts: - - name: flatGeobuf - path: /tmp/flatGeobuf.fgb + - name: 'flatGeobuf' + path: '/tmp/flatGeobuf.fgb' container: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}} + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - [ - 'bmc', - 'create-mapsheet', - '--path=/tmp/flatGeobuf.fgb', - '--bm-config={{=sprig.trim(workflow.parameters.config)}}', - '--output=/tmp/mapsheet.json', - '--include={{=sprig.trim(workflow.parameters.include)}}', - '--exclude={{=sprig.trim(workflow.parameters.exclude)}}', - ] + - 'bmc' + - 'create-mapsheet' + - '--path=/tmp/flatGeobuf.fgb' + - '--bm-config={{=sprig.trim(workflow.parameters.config)}}' + - '--output=/tmp/mapsheet.json' + - '--include={{=sprig.trim(workflow.parameters.include)}}' + - '--exclude={{=sprig.trim(workflow.parameters.exclude)}}' + outputs: artifacts: - - name: mapsheet - path: /tmp/mapsheet.json + - name: 'mapsheet' + path: '/tmp/mapsheet.json' archive: {} volumes: - - name: ephemeral + - name: 'ephemeral' emptyDir: {} diff --git a/workflows/basemaps/topo-maps-standardising.yml b/workflows/basemaps/topo-maps-standardising.yml index db76ca866..4f3d69b92 100644 --- a/workflows/basemaps/topo-maps-standardising.yml +++ b/workflows/basemaps/topo-maps-standardising.yml @@ -1,209 +1,211 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: topo-raster-maps-standardising + name: 'topo-raster-maps-standardising' labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'raster' spec: parallelism: 50 - entrypoint: main - onExit: exit-handler + entrypoint: 'main' + onExit: 'exit-handler' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' podMetadata: labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'raster' arguments: parameters: - - name: version_basemaps_cli - description: Version of the basemaps CLI docker container to use - value: v7 + - name: 'version_basemaps_cli' + description: 'Version of the basemaps CLI docker container to use' + value: 'v7' - - name: version_argo_tasks - description: Version of the argo tasks CLI docker container to use - value: v4 + - name: 'version_argo_tasks' + description: 'Version of the argo tasks CLI docker container to use' + value: 'v4' - - name: source - description: s3 directory of the NZ Topo Raster Map Series imagery to process + - name: 'source' + description: 's3 directory of the NZ Topo Raster Map Series imagery to process' value: 's3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/' enum: - 's3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/' - 's3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo250_GeoTif_Gridless/' - - name: target - description: s3 directory into which to save the generated directory structure of STAC files and COGs + - name: 'target' + description: 's3 directory into which to save the generated directory structure of STAC files and COGs' value: 's3://linz-topographic/' enum: - 's3://linz-basemaps-staging/' - 's3://linz-workflows-scratch/' - 's3://linz-topographic/' - - name: map_series - description: topo50 or topo250 + - name: 'map_series' + description: 'topo50 or topo250' value: 'topo50' enum: - 'topo50' - 'topo250' - - name: latest_only - description: Only process the latest version of each map sheet + - name: 'latest_only' + description: 'Only process the latest version of each map sheet' value: 'false' enum: - 'true' - 'false' - - name: group_size - description: How many items to pass to each create-cog job + - name: 'group_size' + description: 'How many items to pass to each create-cog job' value: '50' - - name: copy_option - description: --no-clobber Skip overwriting existing files. --force Overwrite all files. --force-no-clobber Overwrite only changed files, skip unchanged files. + - name: 'copy_option' + description: '--no-clobber Skip overwriting existing files. --force Overwrite all files. --force-no-clobber Overwrite only changed files, skip unchanged files.' value: '--force-no-clobber' enum: - '--no-clobber' - '--force' - '--force-no-clobber' - - name: aws_role_config_path - description: s3 URL or comma-separated list of s3 URLs allowing the workflow to write to a target(s) + - name: 'aws_role_config_path' + description: 's3 URL or comma-separated list of s3 URLs allowing the workflow to write to a target(s)' value: 's3://linz-bucket-config/config-write.topographic.json,s3://linz-bucket-config/config.basemaps.json' templates: # Main entrypoint into the workflow - - name: main + - name: 'main' retryStrategy: expression: 'false' inputs: parameters: - - name: source - - name: target - - name: map_series - - name: latest_only - - name: group_size + - name: 'source' + - name: 'target' + - name: 'map_series' + - name: 'latest_only' + - name: 'group_size' dag: tasks: # For each tile matrix specified run the "cogify" template - - name: get-location + - name: 'get-location' templateRef: - name: tpl-get-location - template: main + name: 'tpl-get-location' + template: 'main' - - name: topo-stac-creation - template: topo-stac-creation + - name: 'topo-stac-creation' + template: 'topo-stac-creation' arguments: parameters: - - name: source + - name: 'source' value: '{{ inputs.parameters.source }}' - - name: target + - name: 'target' value: '{{ tasks.get-location.outputs.parameters.location }}' - - name: map_series + - name: 'map_series' value: '{{ inputs.parameters.map_series }}' - - name: latest-only + - name: 'latest-only' value: '{{ inputs.parameters.latest_only }}' - depends: get-location + depends: 'get-location' - - name: group + - name: 'group' arguments: parameters: - - name: size + - name: 'size' value: '{{ inputs.parameters.group_size }}' - - name: version + - name: 'version' value: '{{= workflow.parameters.version_argo_tasks }}' artifacts: - - name: input + - name: 'input' from: '{{ tasks.topo-stac-creation.outputs.artifacts.tiles }}' templateRef: - name: tpl-at-group - template: main - depends: topo-stac-creation + name: 'tpl-at-group' + template: 'main' + depends: 'topo-stac-creation' - - name: topo-cog-creation - template: topo-cog-creation + - name: 'topo-cog-creation' + template: 'topo-cog-creation' withParam: '{{ tasks.group.outputs.parameters.output }}' arguments: parameters: - - name: grouped_id + - name: 'grouped_id' value: '{{ item }}' artifacts: - - name: grouped + - name: 'grouped' from: '{{ tasks.group.outputs.artifacts.output }}' - depends: group + depends: 'group' - - name: create-config - template: create-config + - name: 'create-config' + template: 'create-config' when: '{{ item.epsg }} == 2193' withParam: '{{ tasks.topo-stac-creation.outputs.parameters.targets }}' arguments: parameters: - - name: location + - name: 'location' value: '{{ item.url }}' - depends: topo-stac-creation && topo-cog-creation + depends: 'topo-stac-creation && topo-cog-creation' - - name: copy + - name: 'copy' templateRef: - name: copy - template: main + name: 'copy' + template: 'main' when: '{{ inputs.parameters.latest_only }} == false' arguments: parameters: - - name: source + - name: 'source' value: '{{ tasks.get-location.outputs.parameters.location }}{{ inputs.parameters.map_series }}/' - - name: target + - name: 'target' value: '{{ inputs.parameters.target }}{{ inputs.parameters.map_series }}/' - - name: include + - name: 'include' value: '\.tiff?$|\.json$' - - name: exclude + - name: 'exclude' value: '' - - name: flatten + - name: 'flatten' value: 'false' - - name: group + - name: 'group' value: '1000' - - name: group_size + - name: 'group_size' value: '100Gi' - depends: topo-stac-creation && topo-cog-creation + depends: 'topo-stac-creation && topo-cog-creation' - - name: copy-latest + - name: 'copy-latest' templateRef: - name: copy - template: main + name: 'copy' + template: 'main' arguments: parameters: - - name: source + - name: 'source' value: '{{ tasks.get-location.outputs.parameters.location }}{{ inputs.parameters.map_series }}_latest/' - - name: target + - name: 'target' value: '{{ inputs.parameters.target }}{{ inputs.parameters.map_series }}_latest/' - - name: include + - name: 'include' value: '\.tiff?$|\.json$' - - name: exclude + - name: 'exclude' value: '' - - name: flatten + - name: 'flatten' value: 'false' - - name: group + - name: 'group' value: '1000' - - name: group_size + - name: 'group_size' value: '100Gi' - depends: topo-stac-creation && topo-cog-creation + depends: 'topo-stac-creation && topo-cog-creation' # List of the topographic tiles from source and import into target bucket - - name: topo-stac-creation + - name: 'topo-stac-creation' inputs: parameters: - - name: source - - name: target - - name: map_series - - name: latest-only + - name: 'source' + - name: 'target' + - name: 'map_series' + - name: 'latest-only' container: - image: ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }} - command: [node, /app/node_modules/@basemaps/cogify/dist/index.cjs] + image: 'ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }}' + command: + - 'node' + - '/app/node_modules/@basemaps/cogify/dist/index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - 'topo' - '--target={{ inputs.parameters.target }}' @@ -212,77 +214,81 @@ spec: - '{{ inputs.parameters.source }}' outputs: artifacts: - - name: tiles - path: /tmp/topo-stac-creation/tiles.json - - name: broken_tiffs - path: /tmp/topo-stac-creation/broken-tiffs.json + - name: 'tiles' + path: '/tmp/topo-stac-creation/tiles.json' + - name: 'broken_tiffs' + path: '/tmp/topo-stac-creation/broken-tiffs.json' parameters: - - name: targets + - name: 'targets' valueFrom: - path: /tmp/topo-stac-creation/targets.json + path: '/tmp/topo-stac-creation/targets.json' - - name: topo-cog-creation + - name: 'topo-cog-creation' nodeSelector: karpenter.sh/capacity-type: 'spot' inputs: artifacts: - - name: grouped - path: /tmp/topo-cog-creation/grouped/ + - name: 'grouped' + path: '/tmp/topo-cog-creation/grouped/' parameters: - - name: grouped_id + - name: 'grouped_id' container: resources: requests: - memory: 30Gi - cpu: 15000m # AWS gives 2x cpu cores = memory for most instances - ephemeral-storage: 98Gi # 2 pods per 200GB of storage - image: ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }} - command: [node, /app/node_modules/@basemaps/cogify/dist/index.cjs] + memory: '30Gi' + cpu: '15000m' # AWS gives 2x cpu cores = memory for most instances + ephemeral-storage: '98Gi' # 2 pods per 200GB of storage + image: 'ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }}' + command: + - 'node' + - '/app/node_modules/@basemaps/cogify/dist/index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - 'create' - '--from-file={{= inputs.artifacts.grouped.path }}{{inputs.parameters.grouped_id}}.json' - '--concurrency=2' # Create a basemaps configuration file to view the imagery - - name: create-config + - name: 'create-config' inputs: parameters: - - name: location - description: Location of the imagery to create config for + - name: 'location' + description: 'Location of the imagery to create config for' container: - image: ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }} - command: [node, /app/node_modules/@basemaps/cogify/dist/index.cjs] + image: 'ghcr.io/linz/basemaps/cli:{{ workflow.parameters.version_basemaps_cli }}' + command: + - 'node' + - '/app/node_modules/@basemaps/cogify/dist/index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - 'config' - '{{ inputs.parameters.location }}' outputs: parameters: - - name: url - description: Basemaps URL to view the imagery + - name: 'url' + description: 'Basemaps URL to view the imagery' valueFrom: path: '/tmp/cogify/config-url' - - name: config - description: Location of the config file + - name: 'config' + description: 'Location of the config file' valueFrom: path: '/tmp/cogify/config-path' - - name: exit-handler + - name: 'exit-handler' retryStrategy: limit: '0' # `tpl-exit-handler` retries itself steps: - - - name: exit + - - name: 'exit' templateRef: - name: tpl-exit-handler - template: main + name: 'tpl-exit-handler' + template: 'main' arguments: parameters: - - name: workflow_status + - name: 'workflow_status' value: '{{ workflow.status }}' - - name: workflow_parameters + - name: 'workflow_parameters' value: '{{ workflow.parameters }}' diff --git a/workflows/basemaps/vector-etl.yaml b/workflows/basemaps/vector-etl.yaml index 0f7b18856..880d7771b 100644 --- a/workflows/basemaps/vector-etl.yaml +++ b/workflows/basemaps/vector-etl.yaml @@ -1,101 +1,103 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: basemaps-vector-etl + name: 'basemaps-vector-etl' labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: vector + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'vector' spec: - entrypoint: main - onExit: exit-handler + entrypoint: 'main' + onExit: 'exit-handler' podMetadata: labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: vector + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'vector' arguments: parameters: - - name: version_argo_tasks - description: Version of the Argo Tasks CLI docker container to use + - name: 'version_argo_tasks' + description: 'Version of the Argo Tasks CLI docker container to use' value: 'v4' - - name: version_basemaps_etl - description: Version of the Basemaps ETL eks container to use + - name: 'version_basemaps_etl' + description: 'Version of the Basemaps ETL eks container to use' value: 'v1' - - name: target - description: S3 Bucket to use for storing the output + - name: 'target' + description: 'S3 Bucket to use for storing the output' value: 'linz-basemaps' enum: - 'linz-basemaps' - 'linz-basemaps-staging' - - name: filename - description: output filename in s3 + - name: 'filename' + description: 'output filename in s3' value: 'topographic' - - name: title - description: title for the output data, which will populate in stac file + - name: 'title' + description: 'title for the output data, which will populate in stac file' value: 'Topographic' - - name: create_pull_request - description: Should a pull request be created in linz/basemaps-config + - name: 'create_pull_request' + description: 'Should a pull request be created in linz/basemaps-config' value: 'true' enum: - 'true' - 'false' - - name: individual + - name: 'individual' description: 'Individual Config or Combined Config' value: 'combined' enum: - 'individual' - 'combined' - - name: retry - description: Number of retry on failure vector-etl task + - name: 'retry' + description: 'Number of retry on failure vector-etl task' value: '1' - - name: config + - name: 'config' description: 'Config file that etl importing to compare' value: 's3://linz-basemaps/config/config-latest.json.gz' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' dag: tasks: - - name: check-updates - template: check-updates + - name: 'check-updates' + template: 'check-updates' - - name: vector-etl - template: vector-etl + - name: 'vector-etl' + template: 'vector-etl' depends: 'check-updates' when: '{{tasks.check-updates.outputs.parameters.updatesRequired}} == true' - - name: create-pull-request - template: create-pull-request + - name: 'create-pull-request' + template: 'create-pull-request' arguments: parameters: - - name: target + - name: 'target' value: '{{ tasks.vector-etl.outputs.parameters.target }}' when: '{{ workflow.parameters.create_pull_request }} == true && {{tasks.check-updates.outputs.parameters.updatesRequired}} == true' depends: 'vector-etl' - - name: check-updates + - name: 'check-updates' container: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:bm-etl-{{ workflow.parameters.version_basemaps_etl }} - command: [node, /app/index.js] + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:bm-etl-{{ workflow.parameters.version_basemaps_etl }}' + command: + - 'node' + - '/app/index.js' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - 'check-updates' - '--filename={{ workflow.parameters.filename }}' @@ -103,29 +105,31 @@ spec: - '--output=/tmp/' outputs: parameters: - - name: updates + - name: 'updates' valueFrom: path: '/tmp/updates.json' - - name: updatesRequired + - name: 'updatesRequired' valueFrom: path: '/tmp/updatesRequired' - - name: vector-etl + - name: 'vector-etl' retryStrategy: limit: '{{workflow.parameters.retry}}' retryPolicy: 'Always' nodeSelector: karpenter.sh/capacity-type: 'spot' container: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:bm-etl-{{ workflow.parameters.version_basemaps_etl }} + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/eks:bm-etl-{{ workflow.parameters.version_basemaps_etl }}' resources: requests: - memory: 7.8Gi - cpu: 15000m - command: [node, /app/index.js] + memory: '7.8Gi' + cpu: '15000m' + command: + - 'node' + - '/app/index.js' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.basemaps.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.basemaps.json' args: - 'etl' - "{{= workflow.parameters.filename == 'topographic'? '--all' : '--layer=' + workflow.parameters.filename }}" @@ -136,24 +140,24 @@ spec: - '--commit' outputs: parameters: - - name: target + - name: 'target' valueFrom: path: '/tmp/target' - - name: create-pull-request + - name: 'create-pull-request' inputs: parameters: - - name: target + - name: 'target' container: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{ workflow.parameters.version_argo_tasks }} + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{ workflow.parameters.version_argo_tasks }}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.basemaps.json - - name: GITHUB_API_TOKEN + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.basemaps.json' + - name: 'GITHUB_API_TOKEN' valueFrom: secretKeyRef: - name: github-linz-li-bot-pat - key: pat + name: 'github-linz-li-bot-pat' + key: 'pat' args: - 'bmc' - 'create-pr' @@ -161,17 +165,17 @@ spec: - '--config-type=vector' - "--individual={{= workflow.parameters.individual == 'individual'? 'true' : 'false' }}" - - name: exit-handler + - name: 'exit-handler' retryStrategy: limit: '0' # `tpl-exit-handler` retries itself steps: - - - name: exit + - - name: 'exit' templateRef: - name: tpl-exit-handler - template: main + name: 'tpl-exit-handler' + template: 'main' arguments: parameters: - - name: workflow_status + - name: 'workflow_status' value: '{{workflow.status}}' - - name: workflow_parameters + - name: 'workflow_parameters' value: '{{workflow.parameters}}' diff --git a/workflows/cron/cron-national-dem.yaml b/workflows/cron/cron-national-dem.yaml index 16172c3c2..bd2d03eac 100644 --- a/workflows/cron/cron-national-dem.yaml +++ b/workflows/cron/cron-national-dem.yaml @@ -1,11 +1,11 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: CronWorkflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'CronWorkflow' metadata: - name: cron-national-dem + name: 'cron-national-dem' labels: - linz.govt.nz/category: raster - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'raster' + linz.govt.nz/data-type: 'raster' spec: schedule: '0 6 * * 1-5' # At 06:00 AM, Monday through Friday timezone: 'NZ' @@ -18,18 +18,18 @@ spec: workflowSpec: podMetadata: labels: - linz.govt.nz/category: raster - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'raster' + linz.govt.nz/data-type: 'raster' linz.govt.nz/region: 'new-zealand' workflowTemplateRef: - name: national-dem + name: 'national-dem' arguments: parameters: - name: 'config_file' value: 'https://raw.githubusercontent.com/linz/basemaps-config/master/config/tileset/elevation.json' - name: 'odr_url' value: 's3://nz-elevation/new-zealand/new-zealand/dem_1m/2193/' - - name: publish_to_odr + - name: 'publish_to_odr' value: 'true' - name: 'copy_option' value: '--force-no-clobber' diff --git a/workflows/cron/cron-stac-validate-fast.yaml b/workflows/cron/cron-stac-validate-fast.yaml index aba57037b..2977eacc8 100644 --- a/workflows/cron/cron-stac-validate-fast.yaml +++ b/workflows/cron/cron-stac-validate-fast.yaml @@ -1,10 +1,10 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: CronWorkflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'CronWorkflow' metadata: - name: cron-stac-validate-fast + name: 'cron-stac-validate-fast' labels: - linz.govt.nz/category: stac + linz.govt.nz/category: 'stac' spec: schedule: '0 05 * * *' # 5 AM every day timezone: 'NZ' @@ -14,26 +14,26 @@ spec: failedJobsHistoryLimit: 3 suspend: false workflowSpec: - entrypoint: main - onExit: exit-handler + entrypoint: 'main' + onExit: 'exit-handler' podMetadata: labels: - linz.govt.nz/category: stac + linz.govt.nz/category: 'stac' arguments: parameters: - - name: checksum_assets + - name: 'checksum_assets' value: 'false' - name: 'checksum_links' value: 'true' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' steps: - - - name: stac-validate-imagery + - - name: 'stac-validate-imagery' templateRef: - name: tpl-at-stac-validate - template: main + name: 'tpl-at-stac-validate' + template: 'main' arguments: parameters: - name: 'uri' @@ -42,10 +42,10 @@ spec: value: '{{workflow.parameters.checksum_assets}}' - name: 'checksum_links' value: '{{workflow.parameters.checksum_links}}' - - name: stac-validate-elevation + - name: 'stac-validate-elevation' templateRef: - name: tpl-at-stac-validate - template: main + name: 'tpl-at-stac-validate' + template: 'main' arguments: parameters: - name: 'uri' @@ -54,17 +54,17 @@ spec: value: '{{workflow.parameters.checksum_assets}}' - name: 'checksum_links' value: '{{workflow.parameters.checksum_links}}' - - name: exit-handler + - name: 'exit-handler' retryStrategy: limit: '0' # `tpl-exit-handler` retries itself steps: - - - name: exit + - - name: 'exit' templateRef: - name: tpl-exit-handler - template: main + name: 'tpl-exit-handler' + template: 'main' arguments: parameters: - - name: workflow_status + - name: 'workflow_status' value: '{{workflow.status}}' - - name: workflow_parameters + - name: 'workflow_parameters' value: '{{workflow.parameters}}' diff --git a/workflows/cron/cron-stac-validate-full.yaml b/workflows/cron/cron-stac-validate-full.yaml index f8a004877..e1a9dd4b0 100644 --- a/workflows/cron/cron-stac-validate-full.yaml +++ b/workflows/cron/cron-stac-validate-full.yaml @@ -1,10 +1,10 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: CronWorkflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'CronWorkflow' metadata: - name: cron-stac-validate-full + name: 'cron-stac-validate-full' labels: - linz.govt.nz/category: stac + linz.govt.nz/category: 'stac' spec: schedule: '0 05 1 * *' # 5 AM every 1st of the month timezone: 'NZ' @@ -14,18 +14,18 @@ spec: failedJobsHistoryLimit: 3 suspend: false workflowSpec: - entrypoint: main - onExit: exit-handler + entrypoint: 'main' + onExit: 'exit-handler' podMetadata: labels: - linz.govt.nz/category: stac + linz.govt.nz/category: 'stac' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' value: 'v4' - name: 'include' value: 'collection.json$' - - name: checksum_assets + - name: 'checksum_assets' value: 'true' - name: 'checksum_links' value: 'true' @@ -34,21 +34,21 @@ spec: - name: 'concurrency' value: '1' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' steps: - - - name: stac-validate-imagery + - - name: 'stac-validate-imagery' templateRef: - name: stac-validate-parallel - template: main + name: 'stac-validate-parallel' + template: 'main' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' value: '{{workflow.parameters.version_argo_tasks}}' - name: 'uri' value: 's3://nz-imagery/' - - name: include + - name: 'include' value: '{{workflow.parameters.include}}' - name: 'checksum_assets' value: '{{workflow.parameters.checksum_assets}}' @@ -58,17 +58,17 @@ spec: value: '{{workflow.parameters.recursive}}' - name: 'concurrency' value: '{{workflow.parameters.concurrency}}' - - name: stac-validate-elevation + - name: 'stac-validate-elevation' templateRef: - name: stac-validate-parallel - template: main + name: 'stac-validate-parallel' + template: 'main' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' value: '{{workflow.parameters.version_argo_tasks}}' - name: 'uri' value: 's3://nz-elevation/' - - name: include + - name: 'include' value: '{{workflow.parameters.include}}' - name: 'checksum_assets' value: '{{workflow.parameters.checksum_assets}}' @@ -78,17 +78,17 @@ spec: value: '{{workflow.parameters.recursive}}' - name: 'concurrency' value: '{{workflow.parameters.concurrency}}' - - name: exit-handler + - name: 'exit-handler' retryStrategy: limit: '0' # `tpl-exit-handler` retries itself steps: - - - name: exit + - - name: 'exit' templateRef: - name: tpl-exit-handler - template: main + name: 'tpl-exit-handler' + template: 'main' arguments: parameters: - - name: workflow_status + - name: 'workflow_status' value: '{{workflow.status}}' - - name: workflow_parameters + - name: 'workflow_parameters' value: '{{workflow.parameters}}' diff --git a/workflows/cron/cron-vector-etl-roads-addressing.yaml b/workflows/cron/cron-vector-etl-roads-addressing.yaml index 8215b3d40..3cff64a4c 100644 --- a/workflows/cron/cron-vector-etl-roads-addressing.yaml +++ b/workflows/cron/cron-vector-etl-roads-addressing.yaml @@ -1,11 +1,11 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: CronWorkflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'CronWorkflow' metadata: - name: cron-vector-etl-roads-addressing + name: 'cron-vector-etl-roads-addressing' labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: vector + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'vector' spec: schedule: '0 06 * * 3' # 6 AM every Wednesday timezone: 'NZ' @@ -17,10 +17,10 @@ spec: workflowSpec: podMetadata: labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: vector + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'vector' workflowTemplateRef: - name: basemaps-vector-etl + name: 'basemaps-vector-etl' arguments: parameters: - name: 'filename' diff --git a/workflows/cron/cron-vector-etl.yaml b/workflows/cron/cron-vector-etl.yaml index c4a573fde..7540e1f17 100644 --- a/workflows/cron/cron-vector-etl.yaml +++ b/workflows/cron/cron-vector-etl.yaml @@ -1,11 +1,11 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: CronWorkflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'CronWorkflow' metadata: - name: cron-vector-etl-topographic + name: 'cron-vector-etl-topographic' labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: vector + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'vector' spec: schedule: '0 06 * * 3' # 6 AM every Wednesday timezone: 'NZ' @@ -17,10 +17,10 @@ spec: workflowSpec: podMetadata: labels: - linz.govt.nz/category: basemaps - linz.govt.nz/data-type: vector + linz.govt.nz/category: 'basemaps' + linz.govt.nz/data-type: 'vector' workflowTemplateRef: - name: basemaps-vector-etl + name: 'basemaps-vector-etl' arguments: parameters: - name: 'filename' diff --git a/workflows/raster/copy.yaml b/workflows/raster/copy.yaml index ff74c8cae..f94f6a666 100644 --- a/workflows/raster/copy.yaml +++ b/workflows/raster/copy.yaml @@ -1,54 +1,54 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: copy + name: 'copy' labels: - linz.govt.nz/category: raster - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'raster' + linz.govt.nz/data-type: 'raster' spec: parallelism: 50 nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main - onExit: exit-handler + entrypoint: 'main' + onExit: 'exit-handler' synchronization: semaphore: configMapKeyRef: - name: semaphores - key: bulkcopy + name: 'semaphores' + key: 'bulkcopy' workflowMetadata: labelsFrom: linz.govt.nz/user-group: - expression: workflow.parameters.user_group + expression: 'workflow.parameters.user_group' linz.govt.nz/ticket: - expression: workflow.parameters.ticket + expression: 'workflow.parameters.ticket' linz.govt.nz/region: - expression: workflow.parameters.region + expression: 'workflow.parameters.region' podMetadata: labels: linz.govt.nz/user-group: '{{workflow.parameters.user_group}}' - linz.govt.nz/category: raster - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'raster' + linz.govt.nz/data-type: 'raster' linz.govt.nz/ticket: '{{workflow.parameters.ticket}}' linz.govt.nz/region: '{{workflow.parameters.region}}' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' value: 'v4' - - name: user_group - description: Group of users running the workflow + - name: 'user_group' + description: 'Group of users running the workflow' value: 'none' enum: - 'land' - 'sea' - 'none' - - name: ticket - description: Ticket ID e.g. 'AIP-55' + - name: 'ticket' + description: "Ticket ID e.g. 'AIP-55'" value: '' - - name: region - description: Region of the dataset + - name: 'region' + description: 'Region of the dataset' value: 'new-zealand' enum: - 'antarctica' @@ -71,106 +71,106 @@ spec: - 'waikato' - 'wellington' - 'west-coast' - - name: source + - name: 'source' value: 's3://linz-imagery-staging/test/sample/' - - name: target + - name: 'target' value: 's3://linz-imagery-staging/test/sample_target/' - - name: include + - name: 'include' value: '\.tiff?$|\.json$|\.tfw$|/capture-area\.geojson$|/capture-dates\.geojson$' - - name: exclude + - name: 'exclude' value: '' - - name: copy_option + - name: 'copy_option' value: '--no-clobber' enum: - '--no-clobber' - '--force' - '--force-no-clobber' - - name: flatten - description: Flatten the files in the target location + - name: 'flatten' + description: 'Flatten the files in the target location' value: 'false' enum: - 'true' - 'false' - - name: group + - name: 'group' value: '1000' - - name: group_size + - name: 'group_size' value: '100Gi' - - name: transform + - name: 'transform' value: 'f' - - name: aws_role_config_path - description: s3 URL or comma-separated list of s3 URLs allowing the workflow to write to a target(s) + - name: 'aws_role_config_path' + description: 's3 URL or comma-separated list of s3 URLs allowing the workflow to write to a target(s)' value: 's3://linz-bucket-config/config-write.elevation.json,s3://linz-bucket-config/config-write.imagery.json,s3://linz-bucket-config/config-write.topographic.json' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' inputs: parameters: - - name: source - - name: target - - name: include - - name: exclude - - name: flatten - - name: group - - name: group_size + - name: 'source' + - name: 'target' + - name: 'include' + - name: 'exclude' + - name: 'flatten' + - name: 'group' + - name: 'group_size' dag: tasks: - - name: create-manifest + - name: 'create-manifest' templateRef: - name: tpl-create-manifest - template: main + name: 'tpl-create-manifest' + template: 'main' arguments: parameters: - - name: source + - name: 'source' value: '{{inputs.parameters.source}}' - - name: target + - name: 'target' value: '{{inputs.parameters.target}}' - - name: include + - name: 'include' value: '{{inputs.parameters.include}}' - - name: exclude + - name: 'exclude' value: '{{inputs.parameters.exclude}}' - - name: flatten + - name: 'flatten' value: '{{inputs.parameters.flatten}}' - - name: group + - name: 'group' value: '{{inputs.parameters.group}}' - - name: group_size + - name: 'group_size' value: '{{inputs.parameters.group_size}}' - - name: version_argo_tasks + - name: 'version_argo_tasks' value: '{{workflow.parameters.version_argo_tasks}}' - - name: copy + - name: 'copy' templateRef: - name: tpl-copy - template: main + name: 'tpl-copy' + template: 'main' arguments: parameters: - - name: copy_option + - name: 'copy_option' value: '{{workflow.parameters.copy_option}}' - - name: file + - name: 'file' value: '{{item}}' - - name: version_argo_tasks + - name: 'version_argo_tasks' value: '{{workflow.parameters.version_argo_tasks}}' - - name: aws_role_config_path + - name: 'aws_role_config_path' value: '{{workflow.parameters.aws_role_config_path}}' depends: 'create-manifest' withParam: '{{tasks.create-manifest.outputs.parameters.files}}' - - name: exit-handler + - name: 'exit-handler' retryStrategy: limit: '0' # `tpl-exit-handler` retries itself steps: - - - name: exit + - - name: 'exit' templateRef: - name: tpl-exit-handler - template: main + name: 'tpl-exit-handler' + template: 'main' arguments: parameters: - - name: workflow_status + - name: 'workflow_status' value: '{{workflow.status}}' - - name: workflow_parameters + - name: 'workflow_parameters' value: '{{workflow.parameters}}' diff --git a/workflows/raster/merge-layers.yaml b/workflows/raster/merge-layers.yaml index 5fc77d861..8c5970665 100644 --- a/workflows/raster/merge-layers.yaml +++ b/workflows/raster/merge-layers.yaml @@ -1,69 +1,69 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: merge-layers + name: 'merge-layers' labels: - linz.govt.nz/category: raster - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'raster' + linz.govt.nz/data-type: 'raster' spec: parallelism: 50 nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main - onExit: exit-handler + entrypoint: 'main' + onExit: 'exit-handler' workflowMetadata: labels: - linz.govt.nz/region: workflow.parameters.region + linz.govt.nz/region: 'workflow.parameters.region' labelsFrom: linz.govt.nz/ticket: - expression: workflow.parameters.ticket + expression: 'workflow.parameters.ticket' podMetadata: labels: - linz.govt.nz/category: raster - linz.govt.nz/data-type: raster - linz.govt.nz/region: workflow.parameters.region + linz.govt.nz/category: 'raster' + linz.govt.nz/data-type: 'raster' + linz.govt.nz/region: 'workflow.parameters.region' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' description: 'Specify a version of the argo-tasks image to use, e.g. "v4.1" or "latest"' value: 'v4' - - name: version_basemaps_cli + - name: 'version_basemaps_cli' description: 'Specify a version of the basemaps-cli image to use, e.g. "v7.1" or "latest"' value: 'v7' - - name: version_topo_imagery + - name: 'version_topo_imagery' description: 'Specify a version of the topo-imagery image to use, e.g. "v4.8" or "latest"' value: 'v7' - - name: ticket + - name: 'ticket' description: 'Ticket ID, e.g. "TDE-1130"' value: '' - - name: top_layer_source + - name: 'top_layer_source' description: 'Location of the top layer (e.g. 1m DEM hillshade). Semi-colon separated list of paths.' value: 's3://nz-elevation/new-zealand/new-zealand/dem-hillshade-igor_1m/2193/' - - name: base_layer_source + - name: 'base_layer_source' description: 'Location of the base layers (e.g. 8m DEM hillshade) to fill any gaps in the top layer. Semi-colon separated list of paths.' value: 's3://nz-elevation/new-zealand/new-zealand-contour/dem-hillshade-igor_8m/2193/' - - name: odr_url + - name: 'odr_url' description: '(Optional) If an existing dataset add the S3 path to the dataset here to load existing metadata e.g. "s3://nz-elevation/new-zealand/new-zealand/dem-hillshade-igor/2193/"' value: 's3://nz-elevation/new-zealand/new-zealand/dem-hillshade-igor/2193/' - - name: group + - name: 'group' description: 'How many output tiles to process in each standardise-validate task "pod". Change if you have resource or performance issues when standardising a dataset.' value: '4' - - name: publish_to_odr + - name: 'publish_to_odr' description: 'Create a Pull Request for publishing to imagery or elevation ODR bucket' value: 'false' enum: - 'false' - 'true' - - name: copy_option + - name: 'copy_option' description: 'Do not overwrite existing files with "no-clobber", "force" overwriting files in the target location, or "force-no-clobber" overwriting only changed files, skipping unchanged files' value: '--force-no-clobber' enum: - '--no-clobber' - '--force' - '--force-no-clobber' - - name: geospatial_category + - name: 'geospatial_category' description: 'Geospatial category of the dataset, e.g. "dem-hillshade"' value: 'dem-hillshade-igor' enum: @@ -76,29 +76,29 @@ spec: - 'satellite-imagery' - 'dem-hillshade' - 'dem-hillshade-igor' - - name: create_capture_area + - name: 'create_capture_area' value: 'true' enum: - 'false' - 'true' - - name: create_capture_dates + - name: 'create_capture_dates' value: 'false' enum: - 'false' - 'true' - - name: gsd + - name: 'gsd' value: '1' - - name: scale_to_resolution + - name: 'scale_to_resolution' description: 'Scale output TIFFs to x,y resolution (e.g. 1,1 - leave blank for no scaling)' value: '1,1' - - name: source_epsg + - name: 'source_epsg' value: '2193' - - name: target_epsg + - name: 'target_epsg' value: '2193' - - name: include + - name: 'include' description: 'Regular expression pattern match for paths/files to include e.g ".tiff?$"' value: '.tiff?$' - - name: scale + - name: 'scale' description: 'Scale of the standardised output imagery' value: '50000' enum: @@ -109,286 +109,288 @@ spec: - '10000' - '50000' - 'None' - - name: target_bucket_name + - name: 'target_bucket_name' description: 'The ODR bucket name to publish to' value: 'nz-elevation' enum: - 'nz-imagery' - 'nz-elevation' - - name: region + - name: 'region' value: 'new-zealand' - - name: gdal_compression_preset + - name: 'gdal_compression_preset' value: 'dem_lerc' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' inputs: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' default: '{{workflow.parameters.version_argo_tasks}}' - - name: version_basemaps_cli + - name: 'version_basemaps_cli' default: '{{workflow.parameters.version_basemaps_cli}}' - - name: version_topo_imagery + - name: 'version_topo_imagery' default: '{{workflow.parameters.version_topo_imagery}}' - - name: ticket + - name: 'ticket' default: '{{workflow.parameters.ticket}}' - - name: top_layer_source + - name: 'top_layer_source' default: '{{workflow.parameters.top_layer_source}}' - - name: base_layer_source + - name: 'base_layer_source' default: '{{workflow.parameters.base_layer_source}}' - - name: odr_url + - name: 'odr_url' default: '{{workflow.parameters.odr_url}}' - - name: group + - name: 'group' default: '{{workflow.parameters.group}}' - - name: publish_to_odr + - name: 'publish_to_odr' default: '{{workflow.parameters.publish_to_odr}}' - - name: copy_option + - name: 'copy_option' default: '{{workflow.parameters.copy_option}}' - - name: geospatial_category + - name: 'geospatial_category' default: '{{workflow.parameters.geospatial_category}}' - - name: create_capture_area + - name: 'create_capture_area' default: '{{workflow.parameters.create_capture_area}}' - - name: create_capture_dates + - name: 'create_capture_dates' default: '{{workflow.parameters.create_capture_dates}}' - - name: gsd + - name: 'gsd' default: '{{workflow.parameters.gsd}}' - - name: scale_to_resolution + - name: 'scale_to_resolution' default: '{{workflow.parameters.scale_to_resolution}}' - - name: source_epsg + - name: 'source_epsg' default: '{{workflow.parameters.source_epsg}}' - - name: target_epsg + - name: 'target_epsg' default: '{{workflow.parameters.target_epsg}}' - - name: include + - name: 'include' default: '{{workflow.parameters.include}}' - - name: scale + - name: 'scale' default: '{{workflow.parameters.scale}}' - - name: target_bucket_name + - name: 'target_bucket_name' default: '{{workflow.parameters.target_bucket_name}}' - - name: region + - name: 'region' default: '{{workflow.parameters.region}}' - - name: gdal_compression_preset + - name: 'gdal_compression_preset' default: '{{workflow.parameters.gdal_compression_preset}}' dag: tasks: - - name: get-location + - name: 'get-location' templateRef: - name: tpl-get-location - template: main + name: 'tpl-get-location' + template: 'main' - - name: stac-setup + - name: 'stac-setup' templateRef: - name: tpl-at-stac-setup - template: main + name: 'tpl-at-stac-setup' + template: 'main' arguments: parameters: - - name: gsd + - name: 'gsd' value: '{{=sprig.trim(inputs.parameters.gsd)}}' - - name: region + - name: 'region' value: '{{=sprig.trim(inputs.parameters.region)}}' - - name: geospatial_category + - name: 'geospatial_category' value: '{{inputs.parameters.geospatial_category}}' - - name: odr_url + - name: 'odr_url' value: '{{=sprig.trim(inputs.parameters.odr_url)}}' - - name: version + - name: 'version' value: '{{=sprig.trim(inputs.parameters.version_argo_tasks)}}' - - name: tile-index-validate + - name: 'tile-index-validate' templateRef: - name: tpl-at-tile-index-validate - template: main + name: 'tpl-at-tile-index-validate' + template: 'main' arguments: parameters: - - name: scale + - name: 'scale' value: '{{inputs.parameters.scale}}' - - name: include + - name: 'include' value: '{{inputs.parameters.include}}' - - name: source # combine both base_layer_source and top_layer_source + - name: 'source' # combine both base_layer_source and top_layer_source value: '{{= inputs.parameters.base_layer_source == "" ? "" : sprig.trimSuffix("/", inputs.parameters.base_layer_source) + "/;" }}{{=sprig.trimSuffix("/", inputs.parameters.top_layer_source)}}/' - - name: source_epsg + - name: 'source_epsg' value: '{{=sprig.trim(inputs.parameters.source_epsg)}}' - - name: validate + - name: 'validate' value: 'false' - - name: retile + - name: 'retile' value: 'true' - - name: preset + - name: 'preset' value: '{{inputs.parameters.gdal_compression_preset}}' - - name: version + - name: 'version' value: '{{= inputs.parameters.version_argo_tasks}}' - - name: includeDerived + - name: 'includeDerived' value: 'true' depends: 'get-location.Succeeded && stac-setup.Succeeded' - - name: group + - name: 'group' templateRef: - name: tpl-at-group - template: main + name: 'tpl-at-group' + template: 'main' arguments: artifacts: - - name: input + - name: 'input' from: '{{ tasks.tile-index-validate.outputs.artifacts.files }}' parameters: - - name: size + - name: 'size' value: '{{inputs.parameters.group}}' - - name: version + - name: 'version' value: '{{= inputs.parameters.version_argo_tasks}}' depends: 'tile-index-validate.Succeeded' - - name: standardise-validate + - name: 'standardise-validate' templateRef: - name: tpl-ti-standardise-validate - template: main + name: 'tpl-ti-standardise-validate' + template: 'main' arguments: parameters: - - name: group_id + - name: 'group_id' value: '{{item}}' - - name: collection_id + - name: 'collection_id' value: '{{tasks.stac-setup.outputs.parameters.collection_id}}' - - name: current_datetime + - name: 'current_datetime' value: '{{tasks.stac-setup.finishedAt}}' - - name: target + - name: 'target' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/{{inputs.parameters.geospatial_category}}/flat/' - - name: compression + - name: 'compression' value: '{{inputs.parameters.gdal_compression_preset}}' - - name: odr_url + - name: 'odr_url' value: '{{=sprig.trim(inputs.parameters.odr_url)}}' - - name: scale_to_resolution + - name: 'scale_to_resolution' value: '{{=sprig.trim(inputs.parameters.scale_to_resolution)}}' - - name: source_epsg + - name: 'source_epsg' value: '{{=sprig.trim(inputs.parameters.source_epsg)}}' - - name: target_epsg + - name: 'target_epsg' value: '{{=sprig.trim(inputs.parameters.target_epsg)}}' - - name: version_topo_imagery + - name: 'version_topo_imagery' value: '{{= inputs.parameters.version_topo_imagery}}' - - name: create_capture_area + - name: 'create_capture_area' value: '{{inputs.parameters.create_capture_area}}' - - name: create_capture_dates + - name: 'create_capture_dates' value: '{{inputs.parameters.create_capture_dates}}' - - name: gsd + - name: 'gsd' value: '{{=sprig.trim(inputs.parameters.gsd)}}' artifacts: - - name: group_data + - name: 'group_data' from: '{{ tasks.group.outputs.artifacts.output }}' depends: 'group.Succeeded' withParam: '{{ tasks.group.outputs.parameters.output }}' - - name: create-collection + - name: 'create-collection' templateRef: - name: tpl-ti-create-collection - template: main + name: 'tpl-ti-create-collection' + template: 'main' arguments: parameters: - - name: collection_id + - name: 'collection_id' value: '{{tasks.stac-setup.outputs.parameters.collection_id}}' - - name: linz_slug + - name: 'linz_slug' value: '{{tasks.stac-setup.outputs.parameters.linz_slug}}' - - name: location + - name: 'location' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/{{inputs.parameters.geospatial_category}}/flat/' - - name: current_datetime + - name: 'current_datetime' value: '{{tasks.stac-setup.finishedAt}}' - - name: category + - name: 'category' value: '{{inputs.parameters.geospatial_category}}' - - name: region + - name: 'region' value: '{{=sprig.trim(inputs.parameters.region)}}' - - name: gsd + - name: 'gsd' value: '{{=sprig.trim(inputs.parameters.gsd)}}' - - name: lifecycle + - name: 'lifecycle' value: 'ongoing' - - name: producer + - name: 'producer' value: 'Toitū Te Whenua Land Information New Zealand' - - name: licensor + - name: 'licensor' value: 'Toitū Te Whenua Land Information New Zealand' - - name: create_capture_area + - name: 'create_capture_area' value: '{{inputs.parameters.create_capture_area}}' - - name: create_capture_dates + - name: 'create_capture_dates' value: '{{inputs.parameters.create_capture_dates}}' - - name: version_topo_imagery + - name: 'version_topo_imagery' value: '{{= inputs.parameters.version_topo_imagery}}' depends: 'standardise-validate.Succeeded' - - name: stac-validate + - name: 'stac-validate' templateRef: - name: tpl-at-stac-validate - template: main + name: 'tpl-at-stac-validate' + template: 'main' arguments: parameters: - - name: uri + - name: 'uri' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/{{inputs.parameters.geospatial_category}}/flat/collection.json' artifacts: - - name: stac-result + - name: 'stac-result' raw: data: '{{tasks.stac-validate.outputs.result}}' depends: 'create-collection' - - name: create-config + - name: 'create-config' when: "'{{inputs.parameters.target_epsg}}' =~ '2193|3857'" arguments: parameters: - - name: location + - name: 'location' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/{{inputs.parameters.geospatial_category}}/' - - name: bucket + - name: 'bucket' value: '{{tasks.get-location.outputs.parameters.bucket}}' - - name: key + - name: 'key' value: '{{tasks.get-location.outputs.parameters.key}}/{{inputs.parameters.geospatial_category}}' - template: create-config + template: 'create-config' depends: 'standardise-validate' - - name: publish-odr + - name: 'publish-odr' templateRef: - name: publish-odr - template: main + name: 'publish-odr' + template: 'main' when: "'{{inputs.parameters.publish_to_odr}}' == 'true'" arguments: parameters: - - name: source + - name: 'source' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/{{inputs.parameters.geospatial_category}}/flat/' - - name: target_bucket_name + - name: 'target_bucket_name' value: '{{inputs.parameters.target_bucket_name}}' - - name: copy_option + - name: 'copy_option' value: '{{inputs.parameters.copy_option}}' - - name: ticket + - name: 'ticket' value: '{{=sprig.trim(inputs.parameters.ticket)}}' depends: 'stac-validate.Succeeded && create-config.Succeeded' outputs: parameters: - - name: target + - name: 'target' valueFrom: parameter: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/{{inputs.parameters.geospatial_category}}/' default: '' - - name: create-config + - name: 'create-config' inputs: parameters: - - name: location + - name: 'location' description: 'Location of the imagery to create config for' - - name: bucket - - name: key + - name: 'bucket' + - name: 'key' container: image: 'ghcr.io/linz/basemaps/cli:{{=sprig.trim(workflow.parameters.version_basemaps_cli)}}' - command: [node, /app/node_modules/@basemaps/cogify/dist/index.cjs] + command: + - 'node' + - '/app/node_modules/@basemaps/cogify/dist/index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.basemaps.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.basemaps.json' args: - 'config' - '{{=sprig.trimSuffix("/", inputs.parameters.location)}}/flat/' outputs: parameters: - - name: url + - name: 'url' description: 'Basemaps URL to view the imagery' valueFrom: path: '/tmp/cogify/config-url' - - name: config + - name: 'config' description: 'Location of the config file' valueFrom: path: '/tmp/cogify/config-path' artifacts: - - name: url + - name: 'url' path: '/tmp/cogify/config-url' s3: bucket: '{{inputs.parameters.bucket}}' @@ -396,21 +398,21 @@ spec: archive: none: {} - - name: exit-handler + - name: 'exit-handler' retryStrategy: limit: '0' # `tpl-exit-handler` retries itself steps: - - - name: exit + - - name: 'exit' templateRef: - name: tpl-exit-handler - template: main + name: 'tpl-exit-handler' + template: 'main' arguments: parameters: - - name: workflow_status + - name: 'workflow_status' value: '{{workflow.status}}' - - name: workflow_parameters + - name: 'workflow_parameters' value: '{{workflow.parameters}}' volumes: - - name: ephemeral + - name: 'ephemeral' emptyDir: {} diff --git a/workflows/raster/national-dem.yaml b/workflows/raster/national-dem.yaml index 675ad92c3..2e88ddf87 100644 --- a/workflows/raster/national-dem.yaml +++ b/workflows/raster/national-dem.yaml @@ -1,59 +1,59 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: national-dem + name: 'national-dem' labels: - linz.govt.nz/category: raster - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'raster' + linz.govt.nz/data-type: 'raster' spec: parallelism: 50 nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main - onExit: exit-handler + entrypoint: 'main' + onExit: 'exit-handler' workflowMetadata: labels: linz.govt.nz/region: 'new-zealand' labelsFrom: linz.govt.nz/ticket: - expression: workflow.parameters.ticket + expression: 'workflow.parameters.ticket' podMetadata: labels: - linz.govt.nz/category: raster - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'raster' + linz.govt.nz/data-type: 'raster' linz.govt.nz/region: 'new-zealand' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' description: 'Specify a version of the argo-tasks image to use, e.g. "v4.1" or "latest"' value: 'v4' - - name: version_basemaps_cli + - name: 'version_basemaps_cli' description: 'Specify a version of the basemaps-cli image to use, e.g. "v7.1" or "latest"' value: 'v7' - - name: version_topo_imagery + - name: 'version_topo_imagery' description: 'Specify a version of the topo-imagery image to use, e.g. "v4.8" or "latest"' value: 'v7' - - name: ticket + - name: 'ticket' description: 'Ticket ID, e.g. "TDE-1130"' value: '' - - name: config_file + - name: 'config_file' description: 'Location of the configuration file listing the source datasets to merge.' value: 'https://raw.githubusercontent.com/linz/basemaps-config/master/config/tileset/elevation.json' - - name: odr_url + - name: 'odr_url' description: '(Optional) If an existing dataset add the S3 path to the dataset here to load existing metadata e.g. "s3://nz-elevation/new-zealand/new-zealand/dem_1m/2193/"' value: '' - - name: group + - name: 'group' description: 'How many output tiles to process in each standardise-validate task "pod". Change if you have resource or performance issues when standardising a dataset.' value: '2' - - name: publish_to_odr + - name: 'publish_to_odr' description: 'Create a Pull Request for publishing to imagery or elevation ODR bucket' value: 'false' enum: - 'false' - 'true' - - name: copy_option + - name: 'copy_option' description: 'Do not overwrite existing files with "no-clobber", "force" overwriting files in the target location, or "force-no-clobber" overwriting only changed files, skipping unchanged files' value: '--force-no-clobber' enum: @@ -62,296 +62,297 @@ spec: - '--force-no-clobber' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' dag: tasks: - - name: get-location + - name: 'get-location' templateRef: - name: tpl-get-location - template: main + name: 'tpl-get-location' + template: 'main' - - name: create-mapsheet - template: create-mapsheet + - name: 'create-mapsheet' + template: 'create-mapsheet' arguments: parameters: - - name: config_file + - name: 'config_file' value: '{{workflow.parameters.config_file}}' - - name: compare + - name: 'compare' # Workaround --compare with empty value will fail so we add the flag inside the input parameter value: '{{= workflow.parameters.odr_url == "" ? "" : "--compare=" + sprig.trimSuffix("/", workflow.parameters.odr_url) + "/collection.json" }}' - - name: bucket + - name: 'bucket' value: '{{tasks.get-location.outputs.parameters.bucket}}' - - name: key + - name: 'key' value: '{{tasks.get-location.outputs.parameters.key}}' depends: 'get-location.Succeeded' - - name: group + - name: 'group' templateRef: - name: tpl-at-group - template: main + name: 'tpl-at-group' + template: 'main' arguments: artifacts: - - name: input + - name: 'input' from: '{{ tasks.create-mapsheet.outputs.artifacts.files }}' parameters: - - name: size + - name: 'size' value: '{{workflow.parameters.group}}' - - name: version + - name: 'version' value: '{{= workflow.parameters.version_argo_tasks}}' depends: 'create-mapsheet' when: "{{= len(sprig.fromJson(tasks['create-mapsheet'].outputs.parameters.file_list)) > 0 }}" - - name: stac-setup + - name: 'stac-setup' templateRef: - name: tpl-at-stac-setup - template: main + name: 'tpl-at-stac-setup' + template: 'main' arguments: parameters: - - name: gsd + - name: 'gsd' value: '1' - - name: region + - name: 'region' value: 'new zealand' - - name: geospatial_category + - name: 'geospatial_category' value: 'dem' - - name: odr_url + - name: 'odr_url' value: '{{=sprig.trim(workflow.parameters.odr_url)}}' - - name: version + - name: 'version' value: '{{=sprig.trim(workflow.parameters.version_argo_tasks)}}' - - name: add_date_in_survey_path + - name: 'add_date_in_survey_path' value: 'false' depends: 'group.Succeeded' - - name: standardise-validate + - name: 'standardise-validate' templateRef: - name: tpl-ti-standardise-validate - template: main + name: 'tpl-ti-standardise-validate' + template: 'main' arguments: parameters: - - name: group_id + - name: 'group_id' value: '{{item}}' - - name: collection_id + - name: 'collection_id' value: '{{tasks.stac-setup.outputs.parameters.collection_id}}' - - name: current_datetime + - name: 'current_datetime' value: '{{tasks.stac-setup.finishedAt}}' - - name: target + - name: 'target' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/flat/' - - name: odr_url + - name: 'odr_url' value: '{{=sprig.trim(workflow.parameters.odr_url)}}' - - name: compression + - name: 'compression' value: 'dem_lerc' - - name: create_capture_area + - name: 'create_capture_area' value: 'true' - - name: gsd + - name: 'gsd' value: '1' - - name: source_epsg + - name: 'source_epsg' value: '2193' - - name: target_epsg + - name: 'target_epsg' value: '2193' - - name: version_topo_imagery + - name: 'version_topo_imagery' value: '{{= workflow.parameters.version_topo_imagery}}' artifacts: - - name: group_data + - name: 'group_data' from: '{{ tasks.group.outputs.artifacts.output }}' depends: 'stac-setup.Succeeded && get-location.Succeeded' withParam: '{{ tasks.group.outputs.parameters.output }}' - - name: create-collection + - name: 'create-collection' templateRef: - name: tpl-ti-create-collection - template: main + name: 'tpl-ti-create-collection' + template: 'main' arguments: parameters: - - name: collection_id + - name: 'collection_id' value: '{{tasks.stac-setup.outputs.parameters.collection_id}}' - - name: linz_slug + - name: 'linz_slug' value: '{{tasks.stac-setup.outputs.parameters.linz_slug}}' - - name: location + - name: 'location' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/flat/' - - name: current_datetime + - name: 'current_datetime' value: '{{tasks.stac-setup.finishedAt}}' - - name: odr_url + - name: 'odr_url' value: '{{=sprig.trim(workflow.parameters.odr_url)}}' - - name: category + - name: 'category' value: 'dem' - - name: region + - name: 'region' value: 'new-zealand' - - name: gsd + - name: 'gsd' value: '1' - - name: lifecycle + - name: 'lifecycle' value: 'ongoing' - - name: producer + - name: 'producer' value: 'Toitū Te Whenua Land Information New Zealand' - - name: licensor + - name: 'licensor' value: 'Toitū Te Whenua Land Information New Zealand' - - name: create_capture_dates + - name: 'create_capture_dates' value: 'true' - - name: version_topo_imagery + - name: 'version_topo_imagery' value: '{{= workflow.parameters.version_topo_imagery}}' depends: 'standardise-validate.Succeeded' - - name: stac-validate-all + - name: 'stac-validate-all' templateRef: - name: tpl-at-stac-validate - template: main + name: 'tpl-at-stac-validate' + template: 'main' arguments: parameters: - - name: uri + - name: 'uri' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/flat/collection.json' artifacts: - - name: stac-result + - name: 'stac-result' raw: data: '{{tasks.stac-validate-all.outputs.result}}' depends: 'create-collection.Succeeded' when: "'{{workflow.parameters.odr_url}}' == ''" - - name: stac-validate-only-updated + - name: 'stac-validate-only-updated' templateRef: - name: stac-validate-parallel # Needs to validate a list of STAC documents - template: main + name: 'stac-validate-parallel' # Needs to validate a list of STAC documents + template: 'main' arguments: parameters: - - name: uri + - name: 'uri' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/flat/' - - name: recursive + - name: 'recursive' value: 'false' # The Collection might have some Items that are only in the published location - - name: checksum_links + - name: 'checksum_links' value: 'false' - - name: checksum_assets + - name: 'checksum_assets' value: 'true' - - name: include + - name: 'include' value: '\.json$' # FIXME: For now, assuming we have only STAC as JSON files in the scratch location - - name: concurrency + - name: 'concurrency' value: '50' depends: 'create-collection.Succeeded' when: "'{{workflow.parameters.odr_url}}' != ''" - - name: create-config + - name: 'create-config' arguments: parameters: - - name: location + - name: 'location' value: '{{tasks.get-location.outputs.parameters.location}}' - - name: bucket + - name: 'bucket' value: '{{tasks.get-location.outputs.parameters.bucket}}' - - name: key + - name: 'key' value: '{{tasks.get-location.outputs.parameters.key}}' - template: create-config + template: 'create-config' depends: 'standardise-validate' - - name: publish-odr + - name: 'publish-odr' templateRef: - name: publish-odr - template: main + name: 'publish-odr' + template: 'main' when: "'{{workflow.parameters.publish_to_odr}}' == 'true'" arguments: parameters: - - name: source + - name: 'source' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/flat/' - - name: target_bucket_name + - name: 'target_bucket_name' value: 'nz-elevation' - - name: copy_option + - name: 'copy_option' value: '{{workflow.parameters.copy_option}}' - - name: ticket + - name: 'ticket' value: '{{=sprig.trim(workflow.parameters.ticket)}}' depends: '(stac-validate-all.Succeeded || stac-validate-only-updated.Succeeded) && create-config.Succeeded' outputs: parameters: - - name: target + - name: 'target' valueFrom: parameter: '{{tasks.get-location.outputs.parameters.location}}' default: '' # END TEMPLATE `main` - - name: create-mapsheet + - name: 'create-mapsheet' inputs: parameters: - - name: config_file - - name: compare - - name: bucket - - name: key + - name: 'config_file' + - name: 'compare' + - name: 'bucket' + - name: 'key' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}}' resources: requests: - cpu: 3000m - memory: 7.8Gi + cpu: '3000m' + memory: '7.8Gi' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - [ - 'mapsheet-coverage', - '--verbose', - '--location', - '{{inputs.parameters.config_file}}', - '--epsg-code', - '2193', - '{{inputs.parameters.compare}}', - ] + - 'mapsheet-coverage' + - '--verbose' + - '--location' + - '{{inputs.parameters.config_file}}' + - '--epsg-code' + - '2193' + - '{{inputs.parameters.compare}}' + outputs: parameters: - - name: file_list + - name: 'file_list' valueFrom: - path: /tmp/mapsheet-coverage/file-list.json + path: '/tmp/mapsheet-coverage/file-list.json' artifacts: # List of tiff files that need to be processed - - name: files - path: /tmp/mapsheet-coverage/file-list.json + - name: 'files' + path: '/tmp/mapsheet-coverage/file-list.json' optional: true archive: none: {} - - name: layers_source - path: /tmp/mapsheet-coverage/layers-source.geojson.gz + - name: 'layers_source' + path: '/tmp/mapsheet-coverage/layers-source.geojson.gz' optional: true archive: none: {} - - name: layers_combined - path: /tmp/mapsheet-coverage/layers-combined.geojson.gz + - name: 'layers_combined' + path: '/tmp/mapsheet-coverage/layers-combined.geojson.gz' optional: true archive: none: {} # Provenance information for the collection - - name: capture_dates - path: /tmp/mapsheet-coverage/capture-dates.geojson + - name: 'capture_dates' + path: '/tmp/mapsheet-coverage/capture-dates.geojson' s3: bucket: '{{inputs.parameters.bucket}}' key: '{{=sprig.trimSuffix("/", inputs.parameters.key)}}/flat/capture-dates.geojson' archive: none: {} - - name: create-config + - name: 'create-config' inputs: parameters: - - name: location + - name: 'location' description: 'Location of the imagery to create config for' - - name: bucket - - name: key + - name: 'bucket' + - name: 'key' container: image: 'ghcr.io/linz/basemaps/cli:{{=sprig.trim(workflow.parameters.version_basemaps_cli)}}' - command: [node, /app/node_modules/@basemaps/cogify/dist/index.cjs] + command: + - 'node' + - '/app/node_modules/@basemaps/cogify/dist/index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.basemaps.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.basemaps.json' args: - 'config' - '{{=sprig.trimSuffix("/", inputs.parameters.location)}}/flat/' outputs: parameters: - - name: url + - name: 'url' description: 'Basemaps URL to view the imagery' valueFrom: path: '/tmp/cogify/config-url' - - name: config + - name: 'config' description: 'Location of the config file' valueFrom: path: '/tmp/cogify/config-path' artifacts: - - name: url + - name: 'url' path: '/tmp/cogify/config-url' s3: bucket: '{{inputs.parameters.bucket}}' @@ -359,21 +360,21 @@ spec: archive: none: {} - - name: exit-handler + - name: 'exit-handler' retryStrategy: limit: '0' # `tpl-exit-handler` retries itself steps: - - - name: exit + - - name: 'exit' templateRef: - name: tpl-exit-handler - template: main + name: 'tpl-exit-handler' + template: 'main' arguments: parameters: - - name: workflow_status + - name: 'workflow_status' value: '{{workflow.status}}' - - name: workflow_parameters + - name: 'workflow_parameters' value: '{{workflow.parameters}}' volumes: - - name: ephemeral + - name: 'ephemeral' emptyDir: {} diff --git a/workflows/raster/publish-odr.yaml b/workflows/raster/publish-odr.yaml index 9ab974b9c..9800e3d63 100644 --- a/workflows/raster/publish-odr.yaml +++ b/workflows/raster/publish-odr.yaml @@ -1,44 +1,44 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: publish-odr + name: 'publish-odr' labels: - linz.govt.nz/category: raster - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'raster' + linz.govt.nz/data-type: 'raster' spec: parallelism: 50 nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main - onExit: exit-handler + entrypoint: 'main' + onExit: 'exit-handler' synchronization: semaphore: configMapKeyRef: - name: semaphores - key: bulkcopy + name: 'semaphores' + key: 'bulkcopy' workflowMetadata: labelsFrom: linz.govt.nz/ticket: - expression: workflow.parameters.ticket + expression: 'workflow.parameters.ticket' linz.govt.nz/region: - expression: workflow.parameters.region + expression: 'workflow.parameters.region' podMetadata: labels: - linz.govt.nz/category: raster - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'raster' + linz.govt.nz/data-type: 'raster' linz.govt.nz/ticket: '{{workflow.parameters.ticket}}' linz.govt.nz/region: '{{workflow.parameters.region}}' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' description: 'Specify a version of the argo-tasks image to use, e.g. "v4.1" or "latest"' value: 'v4' - - name: ticket + - name: 'ticket' description: 'Ticket ID, e.g. "LI-1570"' value: '' - - name: region + - name: 'region' description: 'Region of the dataset from the list below' value: 'new-zealand' enum: @@ -62,17 +62,17 @@ spec: - 'waikato' - 'wellington' - 'west-coast' - - name: source + - name: 'source' description: 'S3 location of the source dataset' value: 's3://linz-imagery-staging/test/sample/' - - name: target_bucket_name + - name: 'target_bucket_name' description: 'The ODR bucket name to publish to' value: '' enum: - 'nz-elevation' - 'nz-imagery' - '' - - name: copy_option + - name: 'copy_option' description: 'Do not overwrite existing files with "no-clobber", or "force" overwriting files in the target location' value: '--no-clobber' enum: @@ -81,64 +81,64 @@ spec: - '--force-no-clobber' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' inputs: parameters: - - name: copy_option - - name: source - - name: target_bucket_name - - name: ticket + - name: 'copy_option' + - name: 'source' + - name: 'target_bucket_name' + - name: 'ticket' dag: tasks: - - name: generate-path + - name: 'generate-path' templateRef: - name: tpl-at-generate-path - template: main + name: 'tpl-at-generate-path' + template: 'main' arguments: parameters: - - name: version + - name: 'version' value: '{{workflow.parameters.version_argo_tasks}}' - - name: target_bucket_name + - name: 'target_bucket_name' value: '{{inputs.parameters.target_bucket_name}}' - - name: source + - name: 'source' value: '{{inputs.parameters.source}}' - - name: push-to-github + - name: 'push-to-github' templateRef: - name: tpl-push-to-github - template: main + name: 'tpl-push-to-github' + template: 'main' arguments: parameters: - - name: source + - name: 'source' value: '{{inputs.parameters.source}}' - - name: target + - name: 'target' value: '{{tasks.generate-path.outputs.parameters.target}}' - - name: version_argo_tasks + - name: 'version_argo_tasks' value: '{{workflow.parameters.version_argo_tasks}}' - - name: repository + - name: 'repository' value: "{{=sprig.trimPrefix('nz-', inputs.parameters.target_bucket_name)}}" - - name: ticket + - name: 'ticket' value: '{{=sprig.trim(inputs.parameters.ticket)}}' - - name: copy_option + - name: 'copy_option' value: '{{inputs.parameters.copy_option}}' depends: 'generate-path' - - name: exit-handler + - name: 'exit-handler' retryStrategy: limit: '0' # `tpl-exit-handler` retries itself steps: - - - name: exit + - - name: 'exit' templateRef: - name: tpl-exit-handler - template: main + name: 'tpl-exit-handler' + template: 'main' arguments: parameters: - - name: workflow_status + - name: 'workflow_status' value: '{{workflow.status}}' - - name: workflow_parameters + - name: 'workflow_parameters' value: '{{workflow.parameters}}' diff --git a/workflows/raster/standardising.yaml b/workflows/raster/standardising.yaml index de4a84806..0c951dcb4 100644 --- a/workflows/raster/standardising.yaml +++ b/workflows/raster/standardising.yaml @@ -1,60 +1,60 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: imagery-standardising + name: 'imagery-standardising' labels: - linz.govt.nz/category: raster - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'raster' + linz.govt.nz/data-type: 'raster' spec: parallelism: 50 nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main - onExit: exit-handler + entrypoint: 'main' + onExit: 'exit-handler' synchronization: semaphore: configMapKeyRef: - name: semaphores - key: standardising + name: 'semaphores' + key: 'standardising' workflowMetadata: labelsFrom: linz.govt.nz/user-group: - expression: workflow.parameters.user_group + expression: 'workflow.parameters.user_group' linz.govt.nz/ticket: - expression: workflow.parameters.ticket + expression: 'workflow.parameters.ticket' linz.govt.nz/region: - expression: workflow.parameters.region + expression: 'workflow.parameters.region' podMetadata: labels: linz.govt.nz/user-group: '{{workflow.parameters.user_group}}' - linz.govt.nz/category: raster - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'raster' + linz.govt.nz/data-type: 'raster' linz.govt.nz/ticket: '{{workflow.parameters.ticket}}' linz.govt.nz/region: '{{workflow.parameters.region}}' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' description: 'Specify a version of the argo-tasks image to use, e.g. "v4.1" or "latest"' value: 'v4' - - name: version_basemaps_cli + - name: 'version_basemaps_cli' description: 'Specify a version of the basemaps-cli image to use, e.g. "v7.1" or "latest"' value: 'v7' - - name: version_topo_imagery + - name: 'version_topo_imagery' description: 'Specify a version of the topo-imagery image to use, e.g. "v4.8" or "latest"' value: 'v7' - - name: user_group - description: Group of users running the workflow + - name: 'user_group' + description: 'Group of users running the workflow' value: 'none' enum: - 'land' - 'sea' - 'none' - - name: ticket + - name: 'ticket' description: 'Ticket ID, e.g. "LI-1570"' value: '' - - name: region + - name: 'region' description: 'Region of the dataset from the list below' value: 'new-zealand' enum: @@ -78,13 +78,13 @@ spec: - 'waikato' - 'wellington' - 'west-coast' - - name: source + - name: 'source' description: 'S3 location of the source dataset(s). Separate multiple sources with a semicolon (;) for merging. Target imagery will be layered (bottom to top) in order specified (left to right)' value: 's3://linz-imagery-staging/test/sample/' - - name: include + - name: 'include' description: 'Regular expression pattern match for paths/files to include e.g ".tiff?$"' value: '.tiff?$' - - name: scale + - name: 'scale' description: 'Scale of the standardised output imagery' value: '500' enum: @@ -95,47 +95,47 @@ spec: - '10000' - '50000' - 'None' - - name: validate + - name: 'validate' description: 'Validate the tiles match LINZ map sheet tile index and there are no duplicate tiles' value: 'true' enum: - 'false' - 'true' - - name: retile + - name: 'retile' description: 'Retile the dataset to the output scale specified in the "scale" parameter' value: 'false' enum: - 'true' - 'false' - - name: source_epsg + - name: 'source_epsg' description: 'EPSG of the source files' value: '2193' - - name: target_epsg + - name: 'target_epsg' description: 'EPSG of the standardised output files' value: '2193' - - name: group + - name: 'group' description: 'How many output tiles to process in each standardise-validate task "pod". Change if you have resource or performance issues when standardising a dataset.' value: '50' - - name: compression + - name: 'compression' description: 'Compression type to use when standardising TIFFs, e.g. "webp" for imagery or "dem_lerc" for elevation data' value: 'webp' enum: - 'webp' - 'lzw' - 'dem_lerc' - - name: create_capture_area + - name: 'create_capture_area' description: 'Create a capture area GeoJSON file for the standardised dataset' value: 'true' enum: - 'false' - 'true' - - name: cutline + - name: 'cutline' description: '(Optional) location of a cutline file to cut the imagery to .fgb or .geojson' value: '' - - name: odr_url + - name: 'odr_url' description: '(Optional) If an existing dataset add the S3 path to the dataset here to load existing metadata e.g. "s3://nz-imagery/taranaki/new-plymouth_2017_0.1m/rgb/2193/"' value: '' - - name: category + - name: 'category' description: 'Geospatial category of the dataset' value: 'urban-aerial-photos' enum: @@ -146,156 +146,154 @@ spec: - 'dem' - 'dsm' - 'satellite-imagery' - - name: gsd + - name: 'gsd' description: 'Dataset GSD in metres, e.g., "0.3" for 30 centimetres' - - name: producer + - name: 'producer' description: 'The producer of the source dataset, e.g. aerial or bathymetric survey company' value: 'Unknown' enum: - [ - 'Unknown', - 'AAM NZ', - 'Aerial Surveys', - 'Beca', - 'Chang Guang Satellite Technology', - 'Christchurch Helicopters', - 'Dimap', - 'European Space Agency', - 'GeoSmart', - 'Landpro', - 'Maxar', - 'NZ Aerial Mapping', - 'Ocean Infinity', - 'Recon', - 'RPS', - 'SKYCAN', - 'SKYVUW', - 'Terralink International', - 'UAV Mapping NZ', - 'University of Canterbury', - 'Woolpert', - ] - - name: producer_list + - 'Unknown' + - 'AAM NZ' + - 'Aerial Surveys' + - 'Beca' + - 'Chang Guang Satellite Technology' + - 'Christchurch Helicopters' + - 'Dimap' + - 'European Space Agency' + - 'GeoSmart' + - 'Landpro' + - 'Maxar' + - 'NZ Aerial Mapping' + - 'Ocean Infinity' + - 'Recon' + - 'RPS' + - 'SKYCAN' + - 'SKYVUW' + - 'Terralink International' + - 'UAV Mapping NZ' + - 'University of Canterbury' + - 'Woolpert' + + - name: 'producer_list' description: '(Optional) List of imagery producers, separated by semicolon (;). Has no effect unless a semicolon delimited list is entered.' value: '' - - name: licensor + - name: 'licensor' description: 'The licensor of the dataset, e.g. local or regional council, government agency, satellite provider' value: 'Unknown' enum: - [ - 'Unknown', - 'Ashburton District Council', - 'Auckland Council', - 'BOPLASS', - 'Bay of Plenty Regional Council', - 'Buller District Council', - 'Canterbury Aerial Imagery Consortium (CAI)', - 'Carterton District Council', - "Central Hawke's Bay District Council", - 'Central Otago District Council', - 'Chang Guang Satellite Technology', - 'Chatham Islands Council', - 'Christchurch City Council', - 'Clutha District Council', - 'CoLAB', - 'Department of Conservation', - 'Dunedin City Council', - 'Environment Canterbury', - 'Environment Southland', - 'Far North District Council', - 'Gisborne District Council', - 'Gore District Council', - 'Greater Wellington Regional Council', - 'Grey District Council', - 'Hamilton City Council', - 'Hastings District Council', - 'Hauraki District Council', - "Hawke's Bay Local Authority Shared Services (HB LASS)", - "Hawke's Bay Regional Council", - 'Horizons Regional Council', - 'Horowhenua District Council', - 'Hurunui District Council', - 'Hutt City Council', - 'Invercargill City Council', - 'Kaikōura District Council', - 'Kaipara District Council', - 'Kawerau District Council', - 'Kāpiti Coast District Council', - 'Mackenzie District Council', - 'Manawatū District Council', - 'Manawatū-Whanganui LASS', - 'Marlborough District Council', - 'Masterton District Council', - 'Matamata-Piako District Council', - 'Maxar Technologies', - 'Ministry of Business, Innovation and Employment', - 'Ministry of Primary Industries', - 'NZ Aerial Mapping', - 'Napier City Council', - 'National Emergency Management Agency', - 'National Institute of Water and Atmospheric Research', - 'Nelson City Council', - 'New Plymouth District Council', - 'Northland Regional Council', - 'Ōpōtiki District Council', - 'Ōtorohanga District Council', - 'Otago Regional Council', - 'Palmerston North City Council', - 'Planet', - 'Porirua City Council', - 'Queenstown-Lakes District Council', - 'Rangitīkei District Council', - 'Regional Software Holdings Limited', - 'Rotorua District Council', - 'Ruapehu District Council', - 'Selwyn District Council', - 'Sinergise', - 'South Taranaki District Council', - 'South Waikato District Council', - 'South Wairarapa District Council', - 'Southland District Council', - 'Stratford District Council', - 'Taranaki Regional Council', - 'Tararua District Council', - 'Tasman District Council', - 'Taupō District Council', - 'Tauranga City Council', - 'Terralink International', - 'Thames-Coromandel District Council', - 'Timaru District Council', - 'Toitū Te Whenua Land Information New Zealand', - 'Upper Hutt City Council', - 'Waikato District Council', - 'Waikato Regional Aerial Photography Service (WRAPS)', - 'Waikato Regional Council', - 'Waimakariri District Council', - 'Waimate District Council', - 'Waipā District Council', - 'Wairoa District Council', - 'Waitaki District Council', - 'Waitomo District Council', - 'Waka Kotahi', - 'Wellington City Council', - 'West Coast Regional Council', - 'Western Bay of Plenty District Council', - 'Westland District Council', - 'Whakatāne District Council', - 'Whanganui District Council', - 'Whangārei District Council', - ] - - name: licensor_list + - 'Unknown' + - 'Ashburton District Council' + - 'Auckland Council' + - 'BOPLASS' + - 'Bay of Plenty Regional Council' + - 'Buller District Council' + - 'Canterbury Aerial Imagery Consortium (CAI)' + - 'Carterton District Council' + - "Central Hawke's Bay District Council" + - 'Central Otago District Council' + - 'Chang Guang Satellite Technology' + - 'Chatham Islands Council' + - 'Christchurch City Council' + - 'Clutha District Council' + - 'CoLAB' + - 'Department of Conservation' + - 'Dunedin City Council' + - 'Environment Canterbury' + - 'Environment Southland' + - 'Far North District Council' + - 'Gisborne District Council' + - 'Gore District Council' + - 'Greater Wellington Regional Council' + - 'Grey District Council' + - 'Hamilton City Council' + - 'Hastings District Council' + - 'Hauraki District Council' + - "Hawke's Bay Local Authority Shared Services (HB LASS)" + - "Hawke's Bay Regional Council" + - 'Horizons Regional Council' + - 'Horowhenua District Council' + - 'Hurunui District Council' + - 'Hutt City Council' + - 'Invercargill City Council' + - 'Kaikōura District Council' + - 'Kaipara District Council' + - 'Kawerau District Council' + - 'Kāpiti Coast District Council' + - 'Mackenzie District Council' + - 'Manawatū District Council' + - 'Manawatū-Whanganui LASS' + - 'Marlborough District Council' + - 'Masterton District Council' + - 'Matamata-Piako District Council' + - 'Maxar Technologies' + - 'Ministry of Business, Innovation and Employment' + - 'Ministry of Primary Industries' + - 'NZ Aerial Mapping' + - 'Napier City Council' + - 'National Emergency Management Agency' + - 'National Institute of Water and Atmospheric Research' + - 'Nelson City Council' + - 'New Plymouth District Council' + - 'Northland Regional Council' + - 'Ōpōtiki District Council' + - 'Ōtorohanga District Council' + - 'Otago Regional Council' + - 'Palmerston North City Council' + - 'Planet' + - 'Porirua City Council' + - 'Queenstown-Lakes District Council' + - 'Rangitīkei District Council' + - 'Regional Software Holdings Limited' + - 'Rotorua District Council' + - 'Ruapehu District Council' + - 'Selwyn District Council' + - 'Sinergise' + - 'South Taranaki District Council' + - 'South Waikato District Council' + - 'South Wairarapa District Council' + - 'Southland District Council' + - 'Stratford District Council' + - 'Taranaki Regional Council' + - 'Tararua District Council' + - 'Tasman District Council' + - 'Taupō District Council' + - 'Tauranga City Council' + - 'Terralink International' + - 'Thames-Coromandel District Council' + - 'Timaru District Council' + - 'Toitū Te Whenua Land Information New Zealand' + - 'Upper Hutt City Council' + - 'Waikato District Council' + - 'Waikato Regional Aerial Photography Service (WRAPS)' + - 'Waikato Regional Council' + - 'Waimakariri District Council' + - 'Waimate District Council' + - 'Waipā District Council' + - 'Wairoa District Council' + - 'Waitaki District Council' + - 'Waitomo District Council' + - 'Waka Kotahi' + - 'Wellington City Council' + - 'West Coast Regional Council' + - 'Western Bay of Plenty District Council' + - 'Westland District Council' + - 'Whakatāne District Council' + - 'Whanganui District Council' + - 'Whangārei District Council' + + - name: 'licensor_list' description: '(Optional) List of imagery licensors, separated by semicolon (;). Has no effect unless a semicolon delimited list is entered.' value: '' - - name: start_datetime + - name: 'start_datetime' description: 'Dataset capture start date in numeric format YYYY-MM-DD, e.g. "2024-01-14"' value: 'YYYY-MM-DD' - - name: end_datetime + - name: 'end_datetime' description: 'Dataset capture end date in numeric format YYYY-MM-DD, e.g. "2024-02-23"' value: 'YYYY-MM-DD' - - name: geographic_description + - name: 'geographic_description' description: '(Optional) Additional dataset description, to be used in the title in place of the Region, e.g. "Hamilton"' value: '' - - name: lifecycle + - name: 'lifecycle' description: 'The release lifecycle status of the dataset, e.g. "completed or "ongoing"' value: 'completed' enum: @@ -304,19 +302,19 @@ spec: - 'ongoing' - 'completed' - 'deprecated' - - name: event + - name: 'event' description: '(Optional) Event name if dataset has been captured in association with an event, e.g. "Top of the South Floods"' value: '' - - name: historic_survey_number + - name: 'historic_survey_number' description: '(Optional) Survey Number associated with historical datasets, e.g. "SN8844"' value: '' - - name: publish_to_odr + - name: 'publish_to_odr' description: 'Create a Pull Request for publishing to imagery or elevation ODR bucket' value: 'false' enum: - 'false' - 'true' - - name: target_bucket_name + - name: 'target_bucket_name' description: 'The ODR bucket name to publish to' value: '' enum: @@ -324,7 +322,7 @@ spec: - 'nz-imagery' - 'nz-elevation' - '' - - name: copy_option + - name: 'copy_option' description: 'Do not overwrite existing files with "no-clobber", "force" overwriting files in the target location, or "force-no-clobber" overwriting only changed files, skipping unchanged files' value: '--no-clobber' enum: @@ -333,302 +331,304 @@ spec: - '--force-no-clobber' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' inputs: parameters: - - name: source + - name: 'source' dag: tasks: - - name: stac-setup + - name: 'stac-setup' templateRef: - name: tpl-at-stac-setup - template: main + name: 'tpl-at-stac-setup' + template: 'main' arguments: parameters: - - name: start_datetime + - name: 'start_datetime' value: '{{=sprig.trim(workflow.parameters.start_datetime)}}' - - name: end_datetime + - name: 'end_datetime' value: '{{=sprig.trim(workflow.parameters.end_datetime)}}' - - name: gsd + - name: 'gsd' value: '{{=sprig.trim(workflow.parameters.gsd)}}' - - name: region + - name: 'region' value: '{{=sprig.trim(workflow.parameters.region)}}' - - name: geographic_description + - name: 'geographic_description' value: '{{=sprig.trim(workflow.parameters.geographic_description)}}' - - name: geospatial_category + - name: 'geospatial_category' value: '{{=sprig.trim(workflow.parameters.category)}}' - - name: odr_url + - name: 'odr_url' value: '{{=sprig.trim(workflow.parameters.odr_url)}}' - - name: version + - name: 'version' value: '{{=sprig.trim(workflow.parameters.version_argo_tasks)}}' - - name: tile-index-validate + - name: 'tile-index-validate' templateRef: - name: tpl-at-tile-index-validate - template: main + name: 'tpl-at-tile-index-validate' + template: 'main' arguments: parameters: - - name: scale + - name: 'scale' value: '{{workflow.parameters.scale}}' - - name: include + - name: 'include' value: '{{workflow.parameters.include}}' - - name: source + - name: 'source' value: '{{=sprig.trim(inputs.parameters.source)}}' - - name: source_epsg + - name: 'source_epsg' value: '{{=sprig.trim(workflow.parameters.source_epsg)}}' - - name: validate + - name: 'validate' value: '{{= workflow.parameters.validate}}' - - name: retile + - name: 'retile' value: '{{= workflow.parameters.retile}}' - - name: preset + - name: 'preset' value: '{{= workflow.parameters.compression}}' - - name: version + - name: 'version' value: '{{= workflow.parameters.version_argo_tasks}}' - - name: group + - name: 'group' templateRef: - name: tpl-at-group - template: main + name: 'tpl-at-group' + template: 'main' arguments: artifacts: - - name: input + - name: 'input' from: '{{ tasks.tile-index-validate.outputs.artifacts.files }}' parameters: - - name: size + - name: 'size' value: '{{workflow.parameters.group}}' - - name: version + - name: 'version' value: '{{= workflow.parameters.version_argo_tasks}}' depends: 'tile-index-validate' - - name: standardise-validate + - name: 'standardise-validate' templateRef: - name: tpl-ti-standardise-validate - template: main + name: 'tpl-ti-standardise-validate' + template: 'main' arguments: parameters: - - name: group_id + - name: 'group_id' value: '{{item}}' - - name: collection_id + - name: 'collection_id' value: '{{tasks.stac-setup.outputs.parameters.collection_id}}' - - name: current_datetime + - name: 'current_datetime' value: '{{tasks.stac-setup.finishedAt}}' - - name: target + - name: 'target' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/flat/' - - name: odr_url + - name: 'odr_url' value: '{{=sprig.trim(workflow.parameters.odr_url)}}' - - name: compression + - name: 'compression' value: '{{= workflow.parameters.compression}}' - - name: start_datetime + - name: 'start_datetime' value: '{{=sprig.trim(workflow.parameters.start_datetime)}}' - - name: end_datetime + - name: 'end_datetime' value: '{{=sprig.trim(workflow.parameters.end_datetime)}}' - - name: create_capture_area + - name: 'create_capture_area' value: '{{=sprig.trim(workflow.parameters.create_capture_area)}}' - - name: cutline + - name: 'cutline' value: '{{=sprig.trim(workflow.parameters.cutline)}}' - - name: gsd + - name: 'gsd' value: '{{=sprig.trim(workflow.parameters.gsd)}}' - - name: source_epsg + - name: 'source_epsg' value: '{{=sprig.trim(workflow.parameters.source_epsg)}}' - - name: target_epsg + - name: 'target_epsg' value: '{{=sprig.trim(workflow.parameters.target_epsg)}}' - - name: version_topo_imagery + - name: 'version_topo_imagery' value: '{{= workflow.parameters.version_topo_imagery}}' artifacts: - - name: group_data + - name: 'group_data' from: '{{ tasks.group.outputs.artifacts.output }}' depends: 'group && stac-setup && get-location' withParam: '{{ tasks.group.outputs.parameters.output }}' - - name: create-collection + - name: 'create-collection' templateRef: - name: tpl-ti-create-collection - template: main + name: 'tpl-ti-create-collection' + template: 'main' arguments: parameters: - - name: collection_id + - name: 'collection_id' value: '{{tasks.stac-setup.outputs.parameters.collection_id}}' - - name: linz_slug + - name: 'linz_slug' value: '{{tasks.stac-setup.outputs.parameters.linz_slug}}' - - name: location + - name: 'location' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/flat/' - - name: current_datetime + - name: 'current_datetime' value: '{{tasks.stac-setup.finishedAt}}' - - name: odr_url + - name: 'odr_url' value: '{{=sprig.trim(workflow.parameters.odr_url)}}' - - name: category + - name: 'category' value: '{{=sprig.trim(workflow.parameters.category)}}' - - name: region + - name: 'region' value: '{{=sprig.trim(workflow.parameters.region)}}' - - name: gsd + - name: 'gsd' value: '{{=sprig.trim(workflow.parameters.gsd)}}' - - name: geographic_description + - name: 'geographic_description' value: '{{=sprig.trim(workflow.parameters.geographic_description)}}' - - name: event + - name: 'event' value: '{{=sprig.trim(workflow.parameters.event)}}' - - name: historic_survey_number + - name: 'historic_survey_number' value: '{{=sprig.trim(workflow.parameters.historic_survey_number)}}' - - name: lifecycle + - name: 'lifecycle' value: '{{=sprig.trim(workflow.parameters.lifecycle)}}' - - name: add_title_suffix + - name: 'add_title_suffix' value: 'true' - - name: producer + - name: 'producer' value: '{{workflow.parameters.producer}}' - - name: producer_list + - name: 'producer_list' value: '{{=sprig.trim(workflow.parameters.producer_list)}}' - - name: licensor + - name: 'licensor' value: '{{workflow.parameters.licensor}}' - - name: licensor_list + - name: 'licensor_list' value: '{{=sprig.trim(workflow.parameters.licensor_list)}}' - - name: version_topo_imagery + - name: 'version_topo_imagery' value: '{{= workflow.parameters.version_topo_imagery}}' depends: 'standardise-validate' - - name: stac-validate-all + - name: 'stac-validate-all' templateRef: - name: tpl-at-stac-validate - template: main + name: 'tpl-at-stac-validate' + template: 'main' arguments: parameters: - - name: uri + - name: 'uri' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/flat/collection.json' artifacts: - - name: stac-result + - name: 'stac-result' raw: data: '{{tasks.stac-validate-all.outputs.result}}' depends: 'create-collection.Succeeded' when: "'{{workflow.parameters.odr_url}}' == ''" - - name: stac-validate-only-updated + - name: 'stac-validate-only-updated' templateRef: - name: stac-validate-parallel # Needs to validate a list of STAC documents - template: main + name: 'stac-validate-parallel' # Needs to validate a list of STAC documents + template: 'main' arguments: parameters: - - name: uri + - name: 'uri' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/flat/' - - name: recursive + - name: 'recursive' value: 'false' # The Collection might have some Items that are only in the published location - - name: checksum_links + - name: 'checksum_links' value: 'false' - - name: checksum_assets + - name: 'checksum_assets' value: 'true' - - name: include + - name: 'include' value: '\.json$' # FIXME: For now, assuming we have only STAC as JSON files in the scratch location - - name: concurrency + - name: 'concurrency' value: '50' depends: 'create-collection.Succeeded' when: "'{{workflow.parameters.odr_url}}' != ''" - - name: get-location + - name: 'get-location' templateRef: - name: tpl-get-location - template: main + name: 'tpl-get-location' + template: 'main' - - name: create-overview + - name: 'create-overview' when: "'{{workflow.parameters.target_epsg}}' =~ '2193|3857' && '{{workflow.parameters.compression}}' != 'dem_lerc'" arguments: parameters: - - name: location + - name: 'location' value: '{{tasks.get-location.outputs.parameters.location}}' - template: create-overview + template: 'create-overview' depends: 'standardise-validate' - - name: create-config + - name: 'create-config' when: "'{{workflow.parameters.target_epsg}}' =~ '2193|3857'" arguments: parameters: - - name: location + - name: 'location' value: '{{tasks.get-location.outputs.parameters.location}}' - - name: bucket + - name: 'bucket' value: '{{tasks.get-location.outputs.parameters.bucket}}' - - name: key + - name: 'key' value: '{{tasks.get-location.outputs.parameters.key}}' - template: create-config + template: 'create-config' depends: 'create-overview' - - name: publish-odr + - name: 'publish-odr' templateRef: - name: publish-odr - template: main + name: 'publish-odr' + template: 'main' when: "'{{workflow.parameters.publish_to_odr}}' == 'true'" arguments: parameters: - - name: source + - name: 'source' value: '{{=sprig.trimSuffix("/", tasks["get-location"].outputs.parameters.location)}}/flat/' - - name: target_bucket_name + - name: 'target_bucket_name' value: '{{workflow.parameters.target_bucket_name}}' - - name: copy_option + - name: 'copy_option' value: '{{workflow.parameters.copy_option}}' - - name: ticket + - name: 'ticket' value: '{{=sprig.trim(workflow.parameters.ticket)}}' depends: '(stac-validate-all || stac-validate-only-updated) && create-config' outputs: parameters: - - name: target + - name: 'target' valueFrom: parameter: '{{tasks.get-location.outputs.parameters.location}}' # END TEMPLATE `main` - - name: create-overview + - name: 'create-overview' inputs: parameters: - - name: location + - name: 'location' container: # Basemaps v7+ has removed overview creation image: 'ghcr.io/linz/basemaps/cli:v6' resources: requests: - cpu: 3000m - memory: 7.8Gi - command: [node, index.cjs] + cpu: '3000m' + memory: '7.8Gi' + command: + - 'node' + - 'index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - [ - '-V', - 'create-overview', - '--source', - '{{=sprig.trimSuffix("/", inputs.parameters.location)}}/flat/', - '--output', - '{{=sprig.trimSuffix("/", inputs.parameters.location)}}/flat/', - ] + - '-V' + - 'create-overview' + - '--source' + - '{{=sprig.trimSuffix("/", inputs.parameters.location)}}/flat/' + - '--output' + - '{{=sprig.trimSuffix("/", inputs.parameters.location)}}/flat/' - - name: create-config + - name: 'create-config' inputs: parameters: - - name: location + - name: 'location' description: 'Location of the imagery to create config for' - - name: bucket - - name: key + - name: 'bucket' + - name: 'key' container: image: 'ghcr.io/linz/basemaps/cli:{{=sprig.trim(workflow.parameters.version_basemaps_cli)}}' - command: [node, /app/node_modules/@basemaps/cogify/dist/index.cjs] + command: + - 'node' + - '/app/node_modules/@basemaps/cogify/dist/index.cjs' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.basemaps.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.basemaps.json' args: - 'config' - '{{=sprig.trimSuffix("/", inputs.parameters.location)}}/flat/' outputs: parameters: - - name: url + - name: 'url' description: 'Basemaps URL to view the imagery' valueFrom: path: '/tmp/cogify/config-url' - - name: config + - name: 'config' description: 'Location of the config file' valueFrom: path: '/tmp/cogify/config-path' artifacts: - - name: url + - name: 'url' path: '/tmp/cogify/config-url' s3: bucket: '{{inputs.parameters.bucket}}' @@ -636,21 +636,21 @@ spec: archive: none: {} - - name: exit-handler + - name: 'exit-handler' retryStrategy: limit: '0' # `tpl-exit-handler` retries itself steps: - - - name: exit + - - name: 'exit' templateRef: - name: tpl-exit-handler - template: main + name: 'tpl-exit-handler' + template: 'main' arguments: parameters: - - name: workflow_status + - name: 'workflow_status' value: '{{workflow.status}}' - - name: workflow_parameters + - name: 'workflow_parameters' value: '{{workflow.parameters}}' volumes: - - name: ephemeral + - name: 'ephemeral' emptyDir: {} diff --git a/workflows/raster/tests.yaml b/workflows/raster/tests.yaml index cb3557663..645ed0426 100644 --- a/workflows/raster/tests.yaml +++ b/workflows/raster/tests.yaml @@ -1,33 +1,34 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: Workflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'Workflow' metadata: - generateName: test-topo-imagery- + generateName: 'test-topo-imagery-' labels: - linz.govt.nz/category: test - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'test' + linz.govt.nz/data-type: 'raster' spec: - serviceAccountName: workflow-runner-sa + serviceAccountName: 'workflow-runner-sa' podGC: - strategy: OnPodCompletion # Delete pod once its finished - entrypoint: test-script + strategy: 'OnPodCompletion' # Delete pod once its finished + entrypoint: 'test-script' podMetadata: labels: - linz.govt.nz/category: test - linz.govt.nz/data-type: raster + linz.govt.nz/category: 'test' + linz.govt.nz/data-type: 'raster' arguments: parameters: - - name: version_topo_imagery + - name: 'version_topo_imagery' value: 'latest' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: test-script + - name: 'test-script' script: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:{{workflow.parameters.version_topo_imagery}} - args: [python] + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:{{workflow.parameters.version_topo_imagery}}' + args: + - 'python' source: | # Put your code below diff --git a/workflows/stac/stac-validate-parallel.yaml b/workflows/stac/stac-validate-parallel.yaml index a25d905f8..156352b0a 100644 --- a/workflows/stac/stac-validate-parallel.yaml +++ b/workflows/stac/stac-validate-parallel.yaml @@ -1,131 +1,130 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: stac-validate-parallel + name: 'stac-validate-parallel' labels: - linz.govt.nz/category: stac + linz.govt.nz/category: 'stac' spec: parallelism: 20 nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main + entrypoint: 'main' podMetadata: labels: - linz.govt.nz/category: stac + linz.govt.nz/category: 'stac' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' value: 'v4' - - name: uri + - name: 'uri' description: 'Path(s) to the STAC file(s).' value: 's3://linz-imagery-staging/test/stac-validate/' - - name: include + - name: 'include' value: 'collection.json$' - - name: checksum_assets + - name: 'checksum_assets' description: 'Validate the file:checksum of each asset if it exists' value: 'false' enum: - 'true' - 'false' - - name: checksum_links + - name: 'checksum_links' description: 'Validate the file:checksum of each link if it exists' value: 'true' enum: - 'false' - 'true' - - name: recursive + - name: 'recursive' description: 'Follow and validate STAC links' value: 'true' enum: - 'false' - 'true' - - name: concurrency + - name: 'concurrency' description: 'Number of parent STAC documents to be validated concurrently. Used to group the files by aws-list.' value: '1' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' retryStrategy: expression: 'false' inputs: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' value: '{{workflow.parameters.version_argo_tasks}}' - - name: include + - name: 'include' value: '{{workflow.parameters.include}}' - - name: uri + - name: 'uri' value: '{{workflow.parameters.uri}}' - - name: checksum_assets + - name: 'checksum_assets' value: '{{workflow.parameters.checksum_assets}}' - - name: checksum_links + - name: 'checksum_links' value: '{{workflow.parameters.checksum_links}}' - - name: recursive + - name: 'recursive' value: '{{workflow.parameters.recursive}}' - - name: concurrency + - name: 'concurrency' value: '{{workflow.parameters.concurrency}}' dag: tasks: - - name: aws-list - template: aws-list + - name: 'aws-list' + template: 'aws-list' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' value: '{{inputs.parameters.version_argo_tasks}}' - - name: include + - name: 'include' value: '{{inputs.parameters.include}}' - - name: uri + - name: 'uri' value: '{{inputs.parameters.uri}}' - - name: group + - name: 'group' value: '{{inputs.parameters.concurrency}}' - - name: stac-validate + - name: 'stac-validate' templateRef: - name: tpl-at-stac-validate - template: main + name: 'tpl-at-stac-validate' + template: 'main' arguments: parameters: - - name: uri + - name: 'uri' value: '{{item}}' - - name: checksum_assets + - name: 'checksum_assets' value: '{{inputs.parameters.checksum_assets}}' - - name: checksum_links + - name: 'checksum_links' value: '{{inputs.parameters.checksum_links}}' - - name: recursive + - name: 'recursive' value: '{{inputs.parameters.recursive}}' - depends: aws-list + depends: 'aws-list' withParam: '{{tasks.aws-list.outputs.parameters.files}}' - - name: aws-list + - name: 'aws-list' inputs: parameters: - - name: version_argo_tasks - - name: include - - name: uri - - name: group + - name: 'version_argo_tasks' + - name: 'include' + - name: 'uri' + - name: 'group' retryStrategy: limit: '2' # force retrying this specific task container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(inputs.parameters.version_argo_tasks)}}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - [ - 'list', - '--verbose', - '--include', - '{{=sprig.trim(inputs.parameters.include)}}', - '--group', - '{{=sprig.trim(inputs.parameters.group)}}', - '--output', - '/tmp/file_list.json', - '{{=sprig.trim(inputs.parameters.uri)}}', - ] + - 'list' + - '--verbose' + - '--include' + - '{{=sprig.trim(inputs.parameters.include)}}' + - '--group' + - '{{=sprig.trim(inputs.parameters.group)}}' + - '--output' + - '/tmp/file_list.json' + - '{{=sprig.trim(inputs.parameters.uri)}}' + outputs: parameters: - - name: files + - name: 'files' valueFrom: - path: /tmp/file_list.json + path: '/tmp/file_list.json' diff --git a/workflows/test/env.yaml b/workflows/test/env.yaml index 97ce72363..93d613909 100644 --- a/workflows/test/env.yaml +++ b/workflows/test/env.yaml @@ -1,20 +1,21 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: Workflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'Workflow' metadata: - generateName: test-env- + generateName: 'test-env-' labels: - linz.govt.nz/category: test + linz.govt.nz/category: 'test' spec: - entrypoint: env + entrypoint: 'env' podMetadata: labels: - linz.govt.nz/category: test + linz.govt.nz/category: 'test' templates: - - name: env + - name: 'env' container: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:latest + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:latest' # Use `command` if you want to print the environment variables outside the virtualenv, or `args` if you want # to print the variables inside the virtualenv. - command: [env] + command: + - 'env' diff --git a/workflows/test/flatten.yaml b/workflows/test/flatten.yaml index f4d1b6957..543bcbb76 100644 --- a/workflows/test/flatten.yaml +++ b/workflows/test/flatten.yaml @@ -1,27 +1,27 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: test-flatten + name: 'test-flatten' labels: - linz.govt.nz/category: test + linz.govt.nz/category: 'test' spec: nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main + entrypoint: 'main' podMetadata: labels: - linz.govt.nz/category: test + linz.govt.nz/category: 'test' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' value: 'latest' - - name: uri + - name: 'uri' value: '' - - name: filter + - name: 'filter' value: '\.tiff?$|\.json$' - - name: copy_option + - name: 'copy_option' value: '--no-clobber' enum: - '--no-clobber' @@ -29,65 +29,67 @@ spec: - '--force-no-clobber' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' dag: tasks: - - name: flatten - template: flatten + - name: 'flatten' + template: 'flatten' arguments: parameters: - - name: uri + - name: 'uri' value: '{{workflow.parameters.uri}}' - - name: filter + - name: 'filter' value: '{{workflow.parameters.filter}}' - - name: flatten-copy - template: flatten-copy + - name: 'flatten-copy' + template: 'flatten-copy' arguments: parameters: - - name: file + - name: 'file' value: '{{item}}' depends: 'flatten' withParam: '{{tasks.flatten.outputs.parameters.files}}' - - name: flatten + - name: 'flatten' inputs: parameters: - - name: uri - - name: filter + - name: 'uri' + - name: 'filter' container: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}} + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - [ - 'flatten', - '--verbose', - '--include', - '{{=sprig.trim(inputs.parameters.filter)}}', - '--group', - '1000', - '--group-size', - '100Gi', - '--output', - '/tmp/file_list.json', - '{{=sprig.trim(inputs.parameters.uri)}}', - ] + - 'flatten' + - '--verbose' + - '--include' + - '{{=sprig.trim(inputs.parameters.filter)}}' + - '--group' + - '1000' + - '--group-size' + - '100Gi' + - '--output' + - '/tmp/file_list.json' + - '{{=sprig.trim(inputs.parameters.uri)}}' + outputs: parameters: - - name: files + - name: 'files' valueFrom: - path: /tmp/file_list.json - - name: flatten-copy + path: '/tmp/file_list.json' + - name: 'flatten-copy' inputs: parameters: - - name: file + - name: 'file' container: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}} + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{=sprig.trim(workflow.parameters.version_argo_tasks)}}' resources: requests: - memory: 7.8Gi - cpu: 2000m - args: ['copy', '{{workflow.parameters.copy_option}}', '{{inputs.parameters.file}}'] + memory: '7.8Gi' + cpu: '2000m' + args: + - 'copy' + - '{{workflow.parameters.copy_option}}' + - '{{inputs.parameters.file}}' diff --git a/workflows/test/generate-path.yaml b/workflows/test/generate-path.yaml index eb776af68..5859bff68 100644 --- a/workflows/test/generate-path.yaml +++ b/workflows/test/generate-path.yaml @@ -1,47 +1,47 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: test-generate-target + name: 'test-generate-target' labels: - linz.govt.nz/category: test + linz.govt.nz/category: 'test' spec: nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main + entrypoint: 'main' podMetadata: labels: - linz.govt.nz/category: test + linz.govt.nz/category: 'test' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' value: 'v4' - - name: source + - name: 'source' value: 's3://linz-imagery-staging/test/sample/' - - name: target_bucket_name + - name: 'target_bucket_name' value: 'nz-imagery' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' inputs: parameters: - - name: source - - name: target_bucket_name + - name: 'source' + - name: 'target_bucket_name' dag: tasks: - - name: generate-path + - name: 'generate-path' templateRef: - name: tpl-at-generate-path - template: main + name: 'tpl-at-generate-path' + template: 'main' arguments: parameters: - - name: version + - name: 'version' value: '{{workflow.parameters.version_argo_tasks}}' - - name: target_bucket_name + - name: 'target_bucket_name' value: '{{inputs.parameters.target_bucket_name}}' - - name: source + - name: 'source' value: '{{inputs.parameters.source}}' diff --git a/workflows/test/list.arm.yaml b/workflows/test/list.arm.yaml index 65ab76dd6..2f9885247 100644 --- a/workflows/test/list.arm.yaml +++ b/workflows/test/list.arm.yaml @@ -1,43 +1,43 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json # Example of using ARM + Spot instances for processing -apiVersion: argoproj.io/v1alpha1 -kind: Workflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'Workflow' metadata: - generateName: test-aws-list-arm64 + generateName: 'test-aws-list-arm64' labels: - linz.govt.nz/category: test + linz.govt.nz/category: 'test' spec: - entrypoint: main + entrypoint: 'main' podMetadata: labels: - linz.govt.nz/category: test + linz.govt.nz/category: 'test' arguments: parameters: - - name: version_argo_tasks - description: container tag to use + - name: 'version_argo_tasks' + description: 'container tag to use' value: 'latest' - - name: uri - description: location to list + - name: 'uri' + description: 'location to list' value: 's3://linz-imagery-staging/test/sample/' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' dag: tasks: - - name: aws-list - template: aws-list + - name: 'aws-list' + template: 'aws-list' arguments: parameters: - - name: uri + - name: 'uri' value: '{{workflow.parameters.uri}}' - - name: include + - name: 'include' value: '.tiff?$' - - name: aws-list + - name: 'aws-list' # Request a SPOT & ARM64 nodeSelector: @@ -57,13 +57,13 @@ spec: inputs: parameters: - - name: uri - - name: include + - name: 'uri' + - name: 'include' container: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{= workflow.parameters.version_argo_tasks }} + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{= workflow.parameters.version_argo_tasks }}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - 'list' - '--verbose' @@ -73,6 +73,6 @@ spec: - '{{inputs.parameters.uri}}' outputs: parameters: - - name: files + - name: 'files' valueFrom: - path: /tmp/file_list.json + path: '/tmp/file_list.json' diff --git a/workflows/test/list.yaml b/workflows/test/list.yaml index a70700b8d..b80ae7867 100644 --- a/workflows/test/list.yaml +++ b/workflows/test/list.yaml @@ -1,49 +1,49 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: Workflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'Workflow' metadata: - generateName: test-aws-list- + generateName: 'test-aws-list-' labels: - linz.govt.nz/category: test + linz.govt.nz/category: 'test' spec: - serviceAccountName: workflow-runner-sa - entrypoint: main + serviceAccountName: 'workflow-runner-sa' + entrypoint: 'main' podMetadata: labels: - linz.govt.nz/category: test + linz.govt.nz/category: 'test' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' value: 'latest' - - name: uri + - name: 'uri' value: 's3://linz-imagery-staging/test/sample/' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' dag: tasks: - - name: aws-list - template: aws-list + - name: 'aws-list' + template: 'aws-list' arguments: parameters: - - name: uri + - name: 'uri' value: '{{workflow.parameters.uri}}' - - name: include + - name: 'include' value: '.tiff?$' - - name: aws-list + - name: 'aws-list' inputs: parameters: - - name: uri - - name: include + - name: 'uri' + - name: 'include' container: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{workflow.parameters.version_argo_tasks}} + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{workflow.parameters.version_argo_tasks}}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - 'list' - '--verbose' @@ -53,6 +53,6 @@ spec: - '{{inputs.parameters.uri}}' outputs: parameters: - - name: files + - name: 'files' valueFrom: - path: /tmp/file_list.json + path: '/tmp/file_list.json' diff --git a/workflows/test/sleep.yml b/workflows/test/sleep.yml index de9fe2484..371d998be 100644 --- a/workflows/test/sleep.yml +++ b/workflows/test/sleep.yml @@ -1,18 +1,18 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: Workflow +apiVersion: 'argoproj.io/v1alpha1' +kind: 'Workflow' metadata: - generateName: test-sleep- + generateName: 'test-sleep-' labels: - linz.govt.nz/category: test + linz.govt.nz/category: 'test' spec: - entrypoint: sleep + entrypoint: 'sleep' podMetadata: labels: - linz.govt.nz/category: test + linz.govt.nz/category: 'test' templates: - - name: sleep + - name: 'sleep' nodeSelector: karpenter.sh/capacity-type: 'on-demand' tolerations: @@ -23,7 +23,9 @@ spec: container: resources: requests: - memory: 3.9Gi - cpu: 2000m - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:latest - command: ['sleep', '3600'] + memory: '3.9Gi' + cpu: '2000m' + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:latest' + command: + - 'sleep' + - '3600' diff --git a/workflows/util/create-thumbnails.yaml b/workflows/util/create-thumbnails.yaml index a9d1ba8f8..183238c98 100644 --- a/workflows/util/create-thumbnails.yaml +++ b/workflows/util/create-thumbnails.yaml @@ -1,100 +1,99 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/argoproj/argo-workflows/v3.5.5/api/jsonschema/schema.json -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +apiVersion: 'argoproj.io/v1alpha1' +kind: 'WorkflowTemplate' metadata: - name: create-thumbnails + name: 'create-thumbnails' labels: - linz.govt.nz/category: util - linz.govt.nz/data-type: raster - linz.govt.nz/data-sub-type: maps + linz.govt.nz/category: 'util' + linz.govt.nz/data-type: 'raster' + linz.govt.nz/data-sub-type: 'maps' spec: parallelism: 50 nodeSelector: karpenter.sh/capacity-type: 'spot' - entrypoint: main + entrypoint: 'main' arguments: parameters: - - name: version_argo_tasks + - name: 'version_argo_tasks' value: 'v4' - - name: version_topo_imagery + - name: 'version_topo_imagery' value: 'v4' - - name: source + - name: 'source' value: 's3://linz-topographic-upload/maps/topo50/' - - name: include + - name: 'include' value: '.*.tiff?$' templateDefaults: container: - imagePullPolicy: Always + imagePullPolicy: 'Always' image: '' templates: - - name: main + - name: 'main' dag: tasks: - - name: aws-list - template: aws-list + - name: 'aws-list' + template: 'aws-list' - - name: get-location + - name: 'get-location' templateRef: - name: tpl-get-location - template: main + name: 'tpl-get-location' + template: 'main' - - name: thumbnails - template: thumbnails + - name: 'thumbnails' + template: 'thumbnails' arguments: parameters: - - name: target + - name: 'target' value: '{{tasks.get-location.outputs.parameters.location}}thumbnails/' artifacts: - - name: files + - name: 'files' from: '{{ tasks.aws-list.outputs.artifacts.files }}' depends: 'aws-list && get-location' # END TEMPLATE `main` - - name: aws-list + - name: 'aws-list' container: - image: 019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{workflow.parameters.version_argo_tasks}} + image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/argo-tasks:{{workflow.parameters.version_argo_tasks}}' env: - - name: AWS_ROLE_CONFIG_PATH - value: s3://linz-bucket-config/config.json + - name: 'AWS_ROLE_CONFIG_PATH' + value: 's3://linz-bucket-config/config.json' args: - [ - 'list', - '--verbose', - '--include', - '{{workflow.parameters.include}}', - '--group', - '2000', - '--output', - '/tmp/file_list.json', - '{{workflow.parameters.source}}', - ] + - 'list' + - '--verbose' + - '--include' + - '{{workflow.parameters.include}}' + - '--group' + - '2000' + - '--output' + - '/tmp/file_list.json' + - '{{workflow.parameters.source}}' + outputs: artifacts: - - name: files - path: /tmp/file_list.json + - name: 'files' + path: '/tmp/file_list.json' - - name: thumbnails + - name: 'thumbnails' nodeSelector: karpenter.sh/capacity-type: 'spot' inputs: parameters: - - name: target + - name: 'target' artifacts: - - name: files - path: /tmp/file_list.json + - name: 'files' + path: '/tmp/file_list.json' container: image: '019359803926.dkr.ecr.ap-southeast-2.amazonaws.com/topo-imagery:{{=sprig.trim(workflow.parameters.version_topo_imagery)}}' resources: requests: - memory: 7.8Gi - cpu: 15000m - ephemeral-storage: 3Gi + memory: '7.8Gi' + cpu: '15000m' + ephemeral-storage: '3Gi' volumeMounts: - - name: ephemeral + - name: 'ephemeral' mountPath: '/tmp' args: - - python + - 'python' - '/app/scripts/thumbnails.py' - '--from-file' - '/tmp/file_list.json' @@ -102,5 +101,5 @@ spec: - '{{inputs.parameters.target}}' volumes: - - name: ephemeral + - name: 'ephemeral' emptyDir: {}