Skip to content
Snippets Groups Projects

Aggregate

Merged Alexander Hinz requested to merge hinza/data_progs:aggregate into meteo
1 file
+ 13
19
Compare changes
  • Side-by-side
  • Inline
@@ -42,7+42,7 @@
elif u > 0:
x = 270 - np.degrees(np.arctan(v / u))
elif u < 0:
x = 90 - np.degrees(np.arctan(v / u))
return x
@@ -71,7+71,7 @@
mask = df_flag == 0
# mask all flags != 9
df_flag[df_flag != 0] = np.nan
df_flag = np.where(df_flag == 0, df_flag, np.nan)
# mask all flags != 91
df_flag_rule_1[~(df_flag_rule_1 == 1)] = np.nan
df_flag_rule_1 = np.where(df_flag_rule_1 == 1, df_flag_rule_1, np.nan)
# mask all flags != 92
df_flag_rule_2[~(df_flag_rule_2 == 2)] = np.nan
df_flag_rule_2 = np.where(df_flag_rule_2 == 2, df_flag_rule_2, np.nan)
# mask all NODATA values
mask &= df_input != NODATA
@@ -91,18 +91,14 @@ def aggregatePeriod(df, pflags1, pflags2, aggregations):
# Rules to make a 91 flag in the aggregation
# Rule 1: if all flags are 91, then the aggregated flags should also be 91
ps_rule1 = np.isfinite(df_flag_rule_1).sum(axis=0) * 100 / df_flag_rule_1.shape[0]
flags[ps_rule1 == 100] = 91
flags[(df_flag_rule_1 == 1).all(axis=0)] = 91
# Rule 2: if the number of 92 flags are between at least one to 1/3 of all, then the aggregated flag should be 91
ps_rule2 = np.isfinite(df_flag_rule_2).sum(axis=0) * 100 / df_flag_rule_2.shape[0]
try:
flags[(1 / df_flag_rule_2.shape[0] <= ps_rule2) & (ps_rule2 <= 100 / 3)] = 91
except ZeroDivisionError:
pass
flags[((df_flag_rule_2 == 2).sum(axis=0) / df_flag_rule_2.shape[0] > 0) &
((df_flag_rule_2 == 2).sum(axis=0) / df_flag_rule_2.shape[0] < .3)] = 91
# aggregate data
out_data = np.full(df_input.shape[1], NODATA, dtype="float64")
agg_data = np.full(df_input.shape[1], NODATA, dtype="float64")
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
for agg in np.unique(aggregations):
@@ -110,17 +106,15 @@ def aggregatePeriod(df, pflags1, pflags2, aggregations):
if agg == "CIRCULAR_MAX_DIST":
for i, col in zip(np.where(idx)[0], columns[idx]):
wsi = columns.get_loc("WS" + col[2:])
# data[i] = wind(df_input[:, i], df_input[:, wsi])
out_data[i] = wind(df_input[:, i], df_input[:, wsi])
agg_data[i] = wind(df_input[:, i], df_input[:, wsi])
else:
# data.loc[:, idx] = AGGREGATIONS[agg](df_input[:, idx], axis=0)
out_data[idx] = AGGREGATIONS[agg](df_input[:, idx], axis=0)
agg_data[idx] = AGGREGATIONS[agg](df_input[:, idx], axis=0)
# merge data and flags back together - again, performance matters
final_out = np.zeros((len(flags) + len(out_data),))
final_out[::2] = out_data
final_out[1::2] = flags
out = np.zeros((len(flags) + len(agg_data),))
out[::2] = agg_data
out[1::2] = flags
return pd.Series(final_out).replace(np.nan, NODATA)
return pd.Series(out).replace(np.nan, NODATA)
def aggregateData(df, freq, config):
Loading