Skip to content

Commit f3a1d95

Browse files
awarde96awarde96
authored
Merge pull request #74 from ecmwf/feature/allow_multi_point
Properly deal wiht multiple points in timeseries array
2 parents 806141d + 995843d commit f3a1d95

File tree

3 files changed

+197
-133
lines changed

3 files changed

+197
-133
lines changed

‎covjsonkit/decoder/TimeSeries.py‎

Lines changed: 103 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -60,60 +60,115 @@ def to_geopandas(self):
6060
# function to convert covjson to xarray dataset
6161
def to_xarray(self):
6262
dims = ["x", "y", "z", "number", "datetime", "t"]
63-
dataarraydict = {}
64-
65-
# Get coordinates
66-
coords = self.get_domains()
67-
x = coords[0]["axes"]["x"]["values"]
68-
y = coords[0]["axes"]["y"]["values"]
69-
z = coords[0]["axes"]["z"]["values"]
70-
steps = coords[0]["axes"]["t"]["values"]
71-
steps = [step.replace("Z", "") for step in steps]
72-
steps = pd.to_datetime(steps)
73-
# steps = list(range(len(steps)))
74-
75-
num = []
76-
datetime = []
77-
for coverage in self.covjson["coverages"]:
78-
num.append(coverage["mars:metadata"]["number"])
79-
datetime.append(coverage["mars:metadata"]["Forecast date"])
80-
81-
nums = list(set(num))
82-
datetime = list(set(datetime))
63+
ds = []
8364

65+
# Get coordinates for all domains
66+
all_coords = self.get_domains()
67+
68+
unique_coords = set() # To track unique coordinate tuples
69+
unique_domains = [] # To store unique domains
70+
71+
for domain in self.domains:
72+
# Extract coordinate values
73+
x = domain["axes"]["x"]["values"][0]
74+
y = domain["axes"]["y"]["values"][0]
75+
z = domain["axes"]["z"]["values"][0]
76+
t = tuple(domain["axes"]["t"]["values"]) # Use tuple for hashable type
77+
78+
# Create a unique identifier for the domain
79+
coord_tuple = (x, y, z, t)
80+
81+
# Check if this coordinate combination is already seen
82+
if coord_tuple not in unique_coords:
83+
unique_coords.add(coord_tuple) # Mark as seen
84+
unique_domains.append(domain) # Add to unique domains
85+
86+
all_coords = unique_domains
8487
param_values = {}
8588

89+
# Initialize parameter values for all parameters
8690
for parameter in self.parameters:
8791
param_values[parameter] = []
88-
for i, num in enumerate(nums):
89-
param_values[parameter].append([])
90-
for j, date in enumerate(datetime):
91-
param_values[parameter][i].append([])
92-
for k, step in enumerate(steps):
93-
for coverage in self.covjson["coverages"]:
94-
if (
95-
coverage["mars:metadata"]["number"] == num
96-
and coverage["mars:metadata"]["Forecast date"] == date
97-
):
98-
param_values[parameter][i][j] = coverage["ranges"][parameter]["values"]
9992

100-
for parameter in self.parameters:
101-
param_coords = {"x": x, "y": y, "z": z, "number": nums, "datetime": datetime, "t": steps}
102-
dataarray = xr.DataArray(
103-
[[[param_values[parameter]]]],
104-
dims=dims,
105-
coords=param_coords,
106-
name=parameter,
107-
)
108-
109-
dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"]
110-
dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"]
111-
dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"]
112-
dataarraydict[dataarray.attrs["long_name"]] = dataarray
113-
114-
ds = xr.Dataset(dataarraydict)
93+
# Process each coordinate domain
94+
for domain_idx, coords in enumerate(all_coords):
95+
dataarraydict = {}
96+
x = coords["axes"]["x"]["values"]
97+
y = coords["axes"]["y"]["values"]
98+
z = coords["axes"]["z"]["values"]
99+
steps = coords["axes"]["t"]["values"]
100+
steps = [step.replace("Z", "") for step in steps]
101+
steps = pd.to_datetime(steps)
102+
103+
num = []
104+
datetime = []
105+
for coverage in self.covjson["coverages"]:
106+
num.append(coverage["mars:metadata"]["number"])
107+
datetime.append(coverage["mars:metadata"]["Forecast date"])
108+
109+
nums = list(set(num))
110+
datetime = list(set(datetime))
111+
112+
# Extract parameter values for the current domain
113+
for parameter in self.parameters:
114+
if len(param_values[parameter]) <= domain_idx:
115+
param_values[parameter].append([])
116+
117+
for i, num in enumerate(nums):
118+
if len(param_values[parameter][domain_idx]) <= i:
119+
param_values[parameter][domain_idx].append([])
120+
121+
for j, date in enumerate(datetime):
122+
if len(param_values[parameter][domain_idx][i]) <= j:
123+
param_values[parameter][domain_idx][i].append([])
124+
125+
for k, step in enumerate(steps):
126+
for coverage in self.covjson["coverages"]:
127+
# print(coverage["domain"])
128+
# print(coverage["mars:metadata"]["number"], num)
129+
# print(coverage["mars:metadata"]["Forecast date"], date)
130+
# print(coverage["domain"]["axes"]["x"]["values"][0], x)
131+
# print(coverage["domain"]["axes"]["y"]["values"][0], y)
132+
# print(coverage["domain"]["axes"]["z"]["values"][0], z)
133+
if (
134+
coverage["mars:metadata"]["number"] == num
135+
and coverage["mars:metadata"]["Forecast date"] == date
136+
and coverage["domain"]["axes"]["x"]["values"] == x
137+
and coverage["domain"]["axes"]["y"]["values"] == y
138+
and coverage["domain"]["axes"]["z"]["values"] == z
139+
):
140+
param_values[parameter][domain_idx][i][j] = coverage["ranges"][parameter]["values"]
141+
142+
for parameter in self.parameters:
143+
param_coords = {
144+
"x": x,
145+
"y": y,
146+
"z": z,
147+
"number": nums,
148+
"datetime": datetime,
149+
"t": steps,
150+
}
151+
dataarray = xr.DataArray(
152+
[[[param_values[parameter][domain_idx]]]],
153+
dims=dims,
154+
coords=param_coords,
155+
name=f"{parameter}_domain_{domain_idx}",
156+
)
157+
158+
dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"]
159+
dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"]
160+
dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"]
161+
dataarraydict[dataarray.attrs["long_name"]] = dataarray
162+
163+
ds.append(xr.Dataset(dataarraydict))
164+
165+
# Combine all DataArrays into a Dataset
115166
for mars_metadata in self.mars_metadata[0]:
116-
if mars_metadata != "date" and mars_metadata != "step":
117-
ds.attrs[mars_metadata] = self.mars_metadata[0][mars_metadata]
167+
for dss in ds:
168+
if mars_metadata != "date" and mars_metadata != "step":
169+
dss.attrs[mars_metadata] = self.mars_metadata[0][mars_metadata]
170+
171+
if len(ds) == 1:
172+
return ds[0]
118173

119174
return ds

‎covjsonkit/encoder/TimeSeries.py‎

Lines changed: 93 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -129,30 +129,41 @@ def from_polytope(self, result):
129129

130130
logging.debug("The parameters added were: %s", self.parameters) # noqa: E501
131131

132+
points = len(coords[fields["dates"][0]]["composite"])
133+
132134
for date in fields["dates"]:
133-
coordinates[date] = {
134-
"x": [coords[date]["composite"][0][0]],
135-
"y": [coords[date]["composite"][0][1]],
136-
"z": [levels[0]],
137-
}
138-
coordinates[date]["t"] = []
139-
for level in fields["levels"]:
140-
for num in fields["number"]:
141-
for para in fields["param"]:
142-
for step in fields["step"]:
143-
date_format = "%Y%m%dT%H%M%S"
144-
new_date = pd.Timestamp(date).strftime(date_format)
145-
start_time = datetime.strptime(new_date, date_format)
146-
# add current date to list by converting it to iso format
147-
try:
148-
int(step)
149-
except ValueError:
150-
step = step[0]
151-
stamp = start_time + timedelta(hours=int(step))
152-
coordinates[date]["t"].append(stamp.isoformat() + "Z")
135+
coordinates[date] = []
136+
for i, point in enumerate(range(points)):
137+
coordinates[date].append(
138+
{
139+
"x": [coords[date]["composite"][i][0]],
140+
"y": [coords[date]["composite"][i][1]],
141+
"z": [levels[0]],
142+
}
143+
)
144+
# coordinates[date] = {
145+
# "x": [coords[date]["composite"][0][0]],
146+
# "y": [coords[date]["composite"][0][1]],
147+
# "z": [levels[0]],
148+
# }
149+
coordinates[date][i]["t"] = []
150+
for level in fields["levels"]:
151+
for num in fields["number"]:
152+
for para in fields["param"]:
153+
for step in fields["step"]:
154+
date_format = "%Y%m%dT%H%M%S"
155+
new_date = pd.Timestamp(date).strftime(date_format)
156+
start_time = datetime.strptime(new_date, date_format)
157+
# add current date to list by converting it to iso format
158+
try:
159+
int(step)
160+
except ValueError:
161+
step = step[0]
162+
stamp = start_time + timedelta(hours=int(step))
163+
coordinates[date][i]["t"].append(stamp.isoformat() + "Z")
164+
break
153165
break
154166
break
155-
break
156167

157168
end = time.time()
158169
delta = end - start
@@ -165,23 +176,24 @@ def from_polytope(self, result):
165176
start = time.time()
166177
logging.debug("Coverage creation: %s", start) # noqa: E501
167178

168-
for date in fields["dates"]:
169-
for level in fields["levels"]:
170-
for num in fields["number"]:
171-
val_dict = {}
172-
for para in fields["param"]:
173-
val_dict[para] = []
174-
for step in fields["step"]:
175-
key = (date, level, num, para, step)
176-
# for k, v in range_dict.items():
177-
# if k == key:
178-
# val_dict[para].append(v[0])
179-
val_dict[para].append(range_dict[key][0])
180-
mm = mars_metadata.copy()
181-
mm["number"] = num
182-
mm["Forecast date"] = date
183-
del mm["step"]
184-
self.add_coverage(mm, coordinates[date], val_dict)
179+
for i, point in enumerate(range(points)):
180+
for date in fields["dates"]:
181+
for level in fields["levels"]:
182+
for num in fields["number"]:
183+
val_dict = {}
184+
for para in fields["param"]:
185+
val_dict[para] = []
186+
for step in fields["step"]:
187+
key = (date, level, num, para, step)
188+
# for k, v in range_dict.items():
189+
# if k == key:
190+
# val_dict[para].append(v[0])
191+
val_dict[para].append(range_dict[key][i])
192+
mm = mars_metadata.copy()
193+
mm["number"] = num
194+
mm["Forecast date"] = date
195+
del mm["step"]
196+
self.add_coverage(mm, coordinates[date][i], val_dict)
185197

186198
end = time.time()
187199
delta = end - start
@@ -224,12 +236,6 @@ def from_polytope_step(self, result):
224236
}
225237
)
226238

227-
# print("Fields: ", fields)
228-
# print("Coords: ", coords)
229-
# print("Mars Metadata: ", mars_metadata)
230-
# print("Range Dict: ", range_dict)
231-
# print("Range Dict keys: ", range_dict.keys())
232-
233239
coordinates = {}
234240

235241
levels = fields["levels"]
@@ -240,58 +246,61 @@ def from_polytope_step(self, result):
240246

241247
logging.debug("The parameters added were: %s", self.parameters) # noqa: E501
242248

249+
points = len(coords[fields["dates"][0]]["composite"])
250+
243251
for step in fields["step"]:
244-
coordinates[fields["dates"][0]] = {
245-
"x": [coords[fields["dates"][0]]["composite"][0][0]],
246-
"y": [coords[fields["dates"][0]]["composite"][0][1]],
247-
"z": [levels[0]],
248-
}
249-
coordinates[fields["dates"][0]]["t"] = []
250-
for level in fields["levels"]:
251-
for num in fields["number"]:
252-
for para in fields["param"]:
253-
for date in fields["dates"]:
254-
for times in fields["times"]:
255-
# date_format = "%Y%m%dT%H%M%S"
256-
# new_date = pd.Timestamp(date).strftime(date_format)
257-
# start_time = datetime.strptime(new_date, date_format)
258-
# add current date to list by converting it to iso format
259-
# stamp = start_time + timedelta(hours=int(step))
260-
datetime = pd.Timestamp(date) + times
261-
coordinates[fields["dates"][0]]["t"].append(str(datetime).split("+")[0] + "Z")
252+
coordinates[fields["dates"][0]] = []
253+
for i, point in enumerate(range(points)):
254+
coordinates[fields["dates"][0]].append(
255+
{
256+
"x": [coords[fields["dates"][0]]["composite"][i][0]],
257+
"y": [coords[fields["dates"][0]]["composite"][i][1]],
258+
"z": [levels[0]],
259+
}
260+
)
261+
coordinates[fields["dates"][0]][i]["t"] = []
262+
for level in fields["levels"]:
263+
for num in fields["number"]:
264+
for para in fields["param"]:
265+
for date in fields["dates"]:
266+
for times in fields["times"]:
267+
# date_format = "%Y%m%dT%H%M%S"
268+
# new_date = pd.Timestamp(date).strftime(date_format)
269+
# start_time = datetime.strptime(new_date, date_format)
270+
# add current date to list by converting it to iso format
271+
# stamp = start_time + timedelta(hours=int(step))
272+
datetime = pd.Timestamp(date) + times
273+
coordinates[fields["dates"][0]][i]["t"].append(str(datetime).split("+")[0] + "Z")
274+
break
262275
break
263276
break
264-
break
265277

266278
end = time.time()
267279
delta = end - start
268280
logging.debug("Coords creation: %s", end) # noqa: E501
269281
logging.debug("Coords creation: %s", delta) # noqa: E501
270282

271-
# print(coordinates)
272-
273-
# logging.debug("The values returned from walking tree: %s", range_dict) # noqa: E501
274-
# logging.debug("The coordinates returned from walking tree: %s", coordinates) # noqa: E501
275-
276283
start = time.time()
277284
logging.debug("Coverage creation: %s", start) # noqa: E501
278285

279-
for level in fields["levels"]:
280-
for num in fields["number"]:
281-
val_dict = {}
282-
for para in fields["param"]:
283-
val_dict[para] = []
284-
for date in fields["dates"]:
285-
key = (date, level, num, para)
286-
# for k, v in range_dict.items():
287-
# if k == key:
288-
# val_dict[para].append(v[0])
289-
val_dict[para].extend(range_dict[key][0])
290-
mm = mars_metadata.copy()
291-
mm["number"] = num
292-
mm["Forecast date"] = date
293-
# del mm["step"]
294-
self.add_coverage(mm, coordinates[fields["dates"][0]], val_dict)
286+
for i, point in enumerate(range(points)):
287+
for level in fields["levels"]:
288+
for num in fields["number"]:
289+
val_dict = {}
290+
for para in fields["param"]:
291+
val_dict[para] = []
292+
for date in fields["dates"]:
293+
key = (date, level, num, para)
294+
# for k, v in range_dict.items():
295+
# if k == key:
296+
# val_dict[para].append(v[0])
297+
val_dict[para].extend(range_dict[key][i])
298+
mm = mars_metadata.copy()
299+
mm["number"] = num
300+
mm["Forecast date"] = date
301+
# del mm["step"]
302+
print(val_dict)
303+
self.add_coverage(mm, coordinates[fields["dates"][0]][i], val_dict)
295304

296305
end = time.time()
297306
delta = end - start

‎covjsonkit/version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
__version__ = "0.1.7"
1+
__version__ = "0.1.8"

0 commit comments

Comments
 (0)