Skip to content

Commit 292e42c

Browse files
jtherrmannjtherrmann
committed
rename bucket_prefix to job_id in job specs
1 parent e3dfab0 commit 292e42c

20 files changed

+41
-38
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1010
- `ARIA_S1_GUNW` job type to hyp3-edc-uat deployment.
1111

1212
### Changed
13+
- The reserved `bucket_prefix` job spec parameter has been renamed to `job_id` and can be referenced as `Ref::job_id` within each step's `command` field.
1314
- The `job_id` parameter of the `ARIA_RAIDER` job type has been renamed to `gunw_job_id`.
1415
- The `AUTORIFT_ITS_LIVE` job type now accepts Sentinel-1 burst products.
1516
- The `OriginAccessIdentityId` has been renamed to `BucketReadPricipals` and now accepts multiple values.

‎apps/render_cf.py

+10-7
Original file line numberDiff line numberDiff line change
@@ -117,15 +117,20 @@ def parse_map_statement(map_statement: str) -> tuple[str, str]:
117117
raise ValueError(f"expected 'for', got '{tokens[0]}': {map_statement}")
118118
if tokens[2] != 'in':
119119
raise ValueError(f"expected 'in', got '{tokens[2]}': {map_statement}")
120-
return tokens[1], tokens[3]
120+
item, items = tokens[1], tokens[3]
121+
if item == 'job_id':
122+
raise ValueError(f"map statement contains reserved parameter name 'job_id': {map_statement}")
123+
return item, items
121124

122125

123126
def get_batch_job_parameters(job_spec: dict, step: dict, map_item: str | None = None) -> dict:
124-
job_params = {'bucket_prefix', *job_spec['parameters'].keys()}
127+
job_params = set(job_spec['parameters'].keys())
125128
step_params = get_batch_param_names_for_job_step(step)
126129
batch_params = {}
127130
for param in step_params:
128-
if param == map_item:
131+
if param == 'job_id':
132+
batch_params['job_id.$'] = '$.job_id'
133+
elif param == map_item:
129134
batch_params[f'{map_item}.$'] = "States.Format('{}', $$.Map.Item.Value)"
130135
else:
131136
if param not in job_params:
@@ -226,10 +231,8 @@ def validate_job_spec(job_type: str, job_spec: dict) -> None:
226231
if actual_fields != expected_fields:
227232
raise ValueError(f'{job_type} has fields {actual_fields} but should have {expected_fields}')
228233

229-
reserved_params = {'bucket_prefix'}
230-
reserved_params_in_spec = reserved_params.intersection(set(job_spec['parameters'].keys()))
231-
if reserved_params_in_spec:
232-
raise ValueError(f'{job_type} contains reserved parameter names: {sorted(reserved_params_in_spec)}')
234+
if 'job_id' in job_spec['parameters'].keys():
235+
raise ValueError(f"{job_type} contains reserved parameter name 'job_id'")
233236

234237
expected_param_fields = ['api_schema']
235238
for param_name, param_dict in job_spec['parameters'].items():

‎apps/step-function.json.j2

-6
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,6 @@
2929
}
3030
],
3131
"ResultPath": "$.results.execution_started",
32-
"Next": "ADD_PREFIX_TO_JOB_PARAMETERS"
33-
},
34-
"ADD_PREFIX_TO_JOB_PARAMETERS": {
35-
"Type": "Pass",
36-
"InputPath": "$.job_id",
37-
"ResultPath": "$.batch_job_parameters.bucket_prefix",
3832
"Next": "SET_BATCH_OVERRIDES"
3933
},
4034
"SET_BATCH_OVERRIDES": {

‎job_spec/ARIA_AUTORIFT.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ AUTORIFT:
4848
- --bucket
4949
- '!Ref Bucket'
5050
- --bucket-prefix
51-
- Ref::bucket_prefix
51+
- Ref::job_id
5252
- --parameter-file
5353
- Ref::parameter_file
5454
- --naming-scheme

‎job_spec/ARIA_RAIDER.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ ARIA_RAIDER:
3232
- --bucket
3333
- '!Ref Bucket'
3434
- --bucket-prefix
35-
- Ref::bucket_prefix
35+
- Ref::job_id
3636
- --weather-model
3737
- Ref::weather_model
3838
- --input-bucket-prefix

‎job_spec/ARIA_S1_GUNW.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ ARIA_S1_GUNW:
5959
- --bucket
6060
- '!Ref Bucket'
6161
- --bucket-prefix
62-
- Ref::bucket_prefix
62+
- Ref::job_id
6363
- --reference-scenes
6464
- Ref::reference
6565
- --secondary-scenes
@@ -83,7 +83,7 @@ ARIA_S1_GUNW:
8383
- --bucket
8484
- '!Ref Bucket'
8585
- --bucket-prefix
86-
- Ref::bucket_prefix
86+
- Ref::job_id
8787
- --weather-model
8888
- HRRR
8989
timeout: 10800 # 3 hr

‎job_spec/AUTORIFT.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ AUTORIFT:
4545
- --bucket
4646
- '!Ref Bucket'
4747
- --bucket-prefix
48-
- Ref::bucket_prefix
48+
- Ref::job_id
4949
- --parameter-file
5050
- '/vsicurl/http://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_landice_0120m.shp'
5151
- --naming-scheme

‎job_spec/AUTORIFT_ITS_LIVE.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ AUTORIFT:
6363
- --bucket
6464
- '!Ref Bucket'
6565
- --bucket-prefix
66-
- Ref::bucket_prefix
66+
- Ref::job_id
6767
- --parameter-file
6868
- Ref::parameter_file
6969
- --publish-bucket

‎job_spec/INSAR_GAMMA.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ INSAR_GAMMA:
9090
- --bucket
9191
- '!Ref Bucket'
9292
- --bucket-prefix
93-
- Ref::bucket_prefix
93+
- Ref::job_id
9494
- --include-look-vectors
9595
- Ref::include_look_vectors
9696
- --include-los-displacement

‎job_spec/INSAR_ISCE.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ INSAR_ISCE:
9797
- --bucket
9898
- '!Ref Bucket'
9999
- --bucket-prefix
100-
- Ref::bucket_prefix
100+
- Ref::job_id
101101
- --reference-scenes
102102
- Ref::granules
103103
- --secondary-scenes
@@ -135,7 +135,7 @@ INSAR_ISCE:
135135
- --bucket
136136
- '!Ref Bucket'
137137
- --bucket-prefix
138-
- Ref::bucket_prefix
138+
- Ref::job_id
139139
- --weather-model
140140
- Ref::weather_model
141141
timeout: 10800

‎job_spec/INSAR_ISCE_BURST.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ INSAR_ISCE_BURST:
5252
- --bucket
5353
- '!Ref Bucket'
5454
- --bucket-prefix
55-
- Ref::bucket_prefix
55+
- Ref::job_id
5656
- --apply-water-mask
5757
- Ref::apply_water_mask
5858
- --looks

‎job_spec/INSAR_ISCE_MULTI_BURST.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ INSAR_ISCE_MULTI_BURST:
6565
- --bucket
6666
- '!Ref Bucket'
6767
- --bucket-prefix
68-
- Ref::bucket_prefix
68+
- Ref::job_id
6969
- --apply-water-mask
7070
- Ref::apply_water_mask
7171
- --looks

‎job_spec/OPERA_DISP_TMS.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ OPERA_DISP_TMS:
6565
- --bucket
6666
- '!Ref Bucket'
6767
- --bucket-prefix
68-
- Ref::bucket_prefix
68+
- Ref::job_id
6969
timeout: 1800 # 30 min
7070
compute_environment: Default
7171
vcpu: 1
@@ -82,7 +82,7 @@ OPERA_DISP_TMS:
8282
- --bucket
8383
- '!Ref Bucket'
8484
- --bucket-prefix
85-
- Ref::bucket_prefix
85+
- Ref::job_id
8686
timeout: 21600 # 6 hr
8787
compute_environment: Default
8888
vcpu: 1

‎job_spec/RTC_GAMMA.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ RTC_GAMMA:
111111
- --bucket
112112
- '!Ref Bucket'
113113
- --bucket-prefix
114-
- Ref::bucket_prefix
114+
- Ref::job_id
115115
- --resolution
116116
- Ref::resolution
117117
- --dem-name

‎job_spec/S1_CORRECTION_ITS_LIVE.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ S1_CORRECTION_TEST:
3737
- --bucket
3838
- '!Ref Bucket'
3939
- --bucket-prefix
40-
- Ref::bucket_prefix
40+
- Ref::job_id
4141
- --buffer
4242
- Ref::buffer
4343
- --parameter-file

‎job_spec/SRG_GSLC.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ SRG_GSLC:
5353
- --bucket
5454
- '!Ref Bucket'
5555
- --bucket-prefix
56-
- Ref::bucket_prefix
56+
- Ref::job_id
5757
- Ref::granules
5858
timeout: 10800
5959
compute_environment: SrgGslc

‎job_spec/SRG_TIME_SERIES.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ SRG_TIME_SERIES:
5555
- --bucket
5656
- '!Ref Bucket'
5757
- --bucket-prefix
58-
- Ref::bucket_prefix
58+
- Ref::job_id
5959
- --use-gslc-prefix
6060
- Ref::granule
6161
timeout: 10800
@@ -76,7 +76,7 @@ SRG_TIME_SERIES:
7676
- --bucket
7777
- '!Ref Bucket'
7878
- --bucket-prefix
79-
- Ref::bucket_prefix
79+
- Ref::job_id
8080
- --use-gslc-prefix
8181
timeout: 21600 # 6 hr
8282
compute_environment: Default

‎job_spec/WATER_MAP.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ WATER_MAP:
124124
- --bucket
125125
- '!Ref Bucket'
126126
- --bucket-prefix
127-
- Ref::bucket_prefix
127+
- Ref::job_id
128128
- --resolution
129129
- Ref::resolution
130130
- --speckle-filter
@@ -161,7 +161,7 @@ WATER_MAP:
161161
- --bucket
162162
- '!Ref Bucket'
163163
- --bucket-prefix
164-
- Ref::bucket_prefix
164+
- Ref::job_id
165165
- --max-vv-threshold
166166
- Ref::max_vv_threshold
167167
- --max-vh-threshold
@@ -184,7 +184,7 @@ WATER_MAP:
184184
- --bucket
185185
- '!Ref Bucket'
186186
- --bucket-prefix
187-
- Ref::bucket_prefix
187+
- Ref::job_id
188188
- --estimator
189189
- Ref::flood_depth_estimator
190190
- --water-level-sigma

‎job_spec/WATER_MAP_EQ.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ WATER_MAP_EQ:
6666
- --bucket
6767
- '!Ref Bucket'
6868
- --bucket-prefix
69-
- Ref::bucket_prefix
69+
- Ref::job_id
7070
- --resolution
7171
- Ref::resolution
7272
- --speckle-filter
@@ -101,7 +101,7 @@ WATER_MAP_EQ:
101101
- --bucket
102102
- '!Ref Bucket'
103103
- --bucket-prefix
104-
- Ref::bucket_prefix
104+
- Ref::job_id
105105
- --max-vv-threshold
106106
- Ref::max_vv_threshold
107107
- --hand-threshold

‎tests/test_render_cf.py

+7-2
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,20 @@ def test_parse_map_statement():
2020
with pytest.raises(ValueError, match="expected 'in', got 'ib': for item ib items"):
2121
render_cf.parse_map_statement('for item ib items')
2222

23+
with pytest.raises(
24+
ValueError, match="map statement contains reserved parameter name 'job_id': for job_id in items"
25+
):
26+
render_cf.parse_map_statement('for job_id in items')
27+
2328

2429
def test_get_batch_job_parameters():
2530
job_spec: dict = {'parameters': {'param1': {}, 'param2': {}, 'param3': {}, 'param4': {}}}
2631

27-
step = {'command': ['foo', 'Ref::param2', 'Ref::param3', 'bar', 'Ref::bucket_prefix']}
32+
step = {'command': ['foo', 'Ref::param2', 'Ref::param3', 'bar', 'Ref::job_id']}
2833
assert render_cf.get_batch_job_parameters(job_spec, step) == {
2934
'param2.$': '$.batch_job_parameters.param2',
3035
'param3.$': '$.batch_job_parameters.param3',
31-
'bucket_prefix.$': '$.batch_job_parameters.bucket_prefix',
36+
'job_id.$': '$.job_id',
3237
}
3338

3439
step = {'command': ['foo', 'Ref::param2', 'Ref::param3', 'bar', 'Ref::param5']}

0 commit comments

Comments
 (0)