From f3f011b6821eda4e85c6116efd3184e29861243f Mon Sep 17 00:00:00 2001
From: Peter Luenenschloss <peter.luenenschloss@ufz.de>
Date: Wed, 13 May 2020 12:08:18 +0200
Subject: [PATCH] adjusted oddWaters unflagging behavior
---
saqc/funcs/spikes_detection.py | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git a/saqc/funcs/spikes_detection.py b/saqc/funcs/spikes_detection.py
index 6bad83f8a..8dd15ac14 100644
--- a/saqc/funcs/spikes_detection.py
+++ b/saqc/funcs/spikes_detection.py
@@ -42,7 +42,7 @@ def _stray(val_frame, partition_freq=None, partition_min=0, scoring_method='kNNM
if partition.empty | (partition.shape[0] < partition_min):
continue
sample_size = partition.shape[0]
- nn_neighbors = min(n_neighbors, sample_size)
+ nn_neighbors = min(n_neighbors, max(sample_size, 2))
resids = kNNfunc(partition.values, n_neighbors=nn_neighbors-1, algorithm='ball_tree')
sorted_i = resids.argsort()
resids = resids[sorted_i]
@@ -145,7 +145,7 @@ def _reduceMVflags(val_frame, fields, flagger, to_flag_frame, reduction_range):
index[1] + pd.Timedelta(reduction_range))
#test_slice = val_frame[var][index_slice].drop(np.delete(to_flag_index, index[0]), errors='ignore')
- test_slice = val_frame[var][index_slice].drop(to_flag_index, errors='ignore')
+ test_slice = val_frame[var][index_slice]
if not test_slice.empty:
x = (test_slice.index.values.astype(float))
x_0 = x[0]
@@ -157,15 +157,17 @@ def _reduceMVflags(val_frame, fields, flagger, to_flag_frame, reduction_range):
med_resids = np.median(resids)
MAD = np.median(np.abs(resids - med_resids))
crit_val = 0.6745*(abs(med_resids - testval)) / MAD
+ if crit_val > 3.5:
+ to_flag_frame.loc[index[1], var] = True
- test_slice = dios.DictOfSeries(test_slice)
- test_flags = flagger.initFlags(test_slice)
- test_slice, test_flags = spikes_flagSlidingZscore(test_slice, var, test_flags, window=reduction_range,
- offset='15min', count=1,
- polydeg=1, z=3.5, method="modZ")
- if test_flags.isFlagged(field=var)[index[1]]:
- to_flag_frame.loc[index[1], var] = True
+ #test_slice = dios.DictOfSeries(test_slice)
+ #test_flags = flagger.initFlags(test_slice)
+ #test_slice, test_flags = spikes_flagSlidingZscore(test_slice, var, test_flags, window=reduction_range,
+ #offset='15min', count=1,
+ #polydeg=1, z=3.5, method="modZ")
+ #if test_flags.isFlagged(field=var)[index[1]]:
+ # to_flag_frame.loc[index[1], var] = True
return to_flag_frame
--
GitLab