Skip to content
Snippets Groups Projects
Commit e1dca675 authored by David Schäfer's avatar David Schäfer
Browse files

less string usage

parent 8b7947fb
No related branches found
No related tags found
No related merge requests found
...@@ -23,6 +23,15 @@ from saqc.lib.tools import toSequence ...@@ -23,6 +23,15 @@ from saqc.lib.tools import toSequence
# todo: accelerated func applies # todo: accelerated func applies
class Heap:
INDEX = "initial_index"
DATA = "original_data"
FLAGGER = "original_flagger"
FREQ = "frequency"
METHOD = "reshape_method"
DROP = "drop_flags"
def harmWrapper(harm=True, heap={}): def harmWrapper(harm=True, heap={}):
# NOTE: # NOTE:
# (1) - harmonization will ALWAYS flag flagger.BAD all the np.nan values and afterwards DROP ALL # (1) - harmonization will ALWAYS flag flagger.BAD all the np.nan values and afterwards DROP ALL
...@@ -50,8 +59,6 @@ def harmWrapper(harm=True, heap={}): ...@@ -50,8 +59,6 @@ def harmWrapper(harm=True, heap={}):
reshape_missing_flag=None, reshape_missing_flag=None,
reshape_shift_comment=False, reshape_shift_comment=False,
drop_flags=None, drop_flags=None,
# outsort_drop_susp=True,
# outsort_drop_list=None,
data_missing_value=np.nan, data_missing_value=np.nan,
**kwargs **kwargs
): ):
...@@ -70,18 +77,16 @@ def harmWrapper(harm=True, heap={}): ...@@ -70,18 +77,16 @@ def harmWrapper(harm=True, heap={}):
# now we send the flags frame in its current shape to the future: # now we send the flags frame in its current shape to the future:
# heap.update({field: {'original_data': flags_col.assign(data_values=dat_col)}}) # heap.update({field: {'original_data': flags_col.assign(data_values=dat_col)}})
heap[field] = { heap[field] = {
"original_data": dat_col, Heap.DATA: dat_col,
"original_flagger": flagger_merged, Heap.FLAGGER: flagger_merged,
"freq": freq, Heap.FREQ: freq,
"reshape_method": reshape_method, Heap.METHOD: reshape_method,
"drop_flags": drop_flags, Heap.DROP: drop_flags,
# "drop_susp": outsort_drop_susp,
# "drop_list": outsort_drop_list,
} }
# furthermore we need to memorize the initial timestamp to ensure output format will equal input format. # furthermore we need to memorize the initial timestamp to ensure output format will equal input format.
if "initial_ts" not in heap.keys(): if Heap.INDEX not in heap.keys():
heap.update({"initial_ts": dat_col.index}) heap.update({Heap.INDEX: dat_col.index})
# now we can manipulate it without loosing information gathered before harmonization # now we can manipulate it without loosing information gathered before harmonization
dat_col, flagger_merged_clean = _outsortCrap( dat_col, flagger_merged_clean = _outsortCrap(
...@@ -95,7 +100,7 @@ def harmWrapper(harm=True, heap={}): ...@@ -95,7 +100,7 @@ def harmWrapper(harm=True, heap={}):
method=inter_method, method=inter_method,
order=inter_order, order=inter_order,
agg_method=inter_agg, agg_method=inter_agg,
total_range=(heap["initial_ts"][0], heap["initial_ts"][-1]), total_range=(heap[Heap.INDEX][0], heap[Heap.INDEX][-1]),
downcast_interpolation=inter_downcast, downcast_interpolation=inter_downcast,
) )
...@@ -134,11 +139,11 @@ def harmWrapper(harm=True, heap={}): ...@@ -134,11 +139,11 @@ def harmWrapper(harm=True, heap={}):
return data, flagger return data, flagger
# get some deharm configuration infos from the heap: # get some deharm configuration infos from the heap:
freq = heap[field]["freq"] freq = heap[field][Heap.FREQ]
redrop_flags = heap[field]["drop_flags"] redrop_flags = heap[field][Heap.DROP]
# redrop_susp = heap[field]["drop_susp"] # redrop_susp = heap[field]["drop_susp"]
# redrop_list = heap[field]["drop_list"] # redrop_list = heap[field]["drop_list"]
resolve_method = harm_2_deharm[heap[field]["reshape_method"]] resolve_method = harm_2_deharm[heap[field][Heap.METHOD]]
# retrieve data and flags from the merged saqc-conform data frame (and by that get rid of blow-up entries). # retrieve data and flags from the merged saqc-conform data frame (and by that get rid of blow-up entries).
dat_col, flagger_merged = _fromMerged(data, flagger, field) dat_col, flagger_merged = _fromMerged(data, flagger, field)
...@@ -148,7 +153,7 @@ def harmWrapper(harm=True, heap={}): ...@@ -148,7 +153,7 @@ def harmWrapper(harm=True, heap={}):
drops, flagger_original_clean = _outsortCrap( drops, flagger_original_clean = _outsortCrap(
dat_col, dat_col,
field, field,
heap[field]["original_flagger"], heap[field][Heap.FLAGGER],
# drop_suspicious=redrop_susp, # drop_suspicious=redrop_susp,
# drop_bad=True, # drop_bad=True,
drop_flags=redrop_flags, drop_flags=redrop_flags,
...@@ -177,9 +182,7 @@ def harmWrapper(harm=True, heap={}): ...@@ -177,9 +182,7 @@ def harmWrapper(harm=True, heap={}):
flags_col = flags_col.to_frame() flags_col = flags_col.to_frame()
flagger_back_full = flagger.initFlags(flags=flags_col) flagger_back_full = flagger.initFlags(flags=flags_col)
dat_col = heap[field]["original_data"].reindex( dat_col = heap[field][Heap.DATA].reindex(flags_col.index, fill_value=np.nan)
flags_col.index, fill_value=np.nan
)
dat_col.name = field dat_col.name = field
# transform the result into the form, data travels through saqc: # transform the result into the form, data travels through saqc:
data, flagger_out = _toMerged( data, flagger_out = _toMerged(
...@@ -188,7 +191,7 @@ def harmWrapper(harm=True, heap={}): ...@@ -188,7 +191,7 @@ def harmWrapper(harm=True, heap={}):
field, field,
dat_col, dat_col,
flagger_back_full, flagger_back_full,
target_index=heap["initial_ts"], target_index=heap[Heap.INDEX],
) )
# remove weight from the heap: # remove weight from the heap:
heap.pop(field) heap.pop(field)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment