Skip to content
Snippets Groups Projects
Commit 73210d6a authored by Peter Lünenschloß's avatar Peter Lünenschloß
Browse files

minor cleanup

parent 3a30e5a6
No related branches found
No related tags found
5 merge requests!193Release 1.4,!188Release 1.4,!35Oddwater od implementation,!34Oddwater od implementation,!33Oddwater od implementation
Pipeline #3440 passed with stage
in 5 minutes and 58 seconds
...@@ -24,6 +24,10 @@ def _stray(val_frame, partition_freq=None, partition_min=0, scoring_method='kNNM ...@@ -24,6 +24,10 @@ def _stray(val_frame, partition_freq=None, partition_min=0, scoring_method='kNNM
alpha=0.05): alpha=0.05):
kNNfunc = getattr(ts_ops, scoring_method) kNNfunc = getattr(ts_ops, scoring_method)
# partitioning
if not partition_freq:
partition_freq = val_frame.shape[0]
if isinstance(partition_freq, str): if isinstance(partition_freq, str):
partitions = val_frame.groupby(pd.Grouper(freq=partition_freq)) partitions = val_frame.groupby(pd.Grouper(freq=partition_freq))
else: else:
...@@ -31,6 +35,7 @@ def _stray(val_frame, partition_freq=None, partition_min=0, scoring_method='kNNM ...@@ -31,6 +35,7 @@ def _stray(val_frame, partition_freq=None, partition_min=0, scoring_method='kNNM
grouper_series = grouper_series.transform(lambda x: int(np.floor(x / partition_freq))) grouper_series = grouper_series.transform(lambda x: int(np.floor(x / partition_freq)))
partitions = val_frame.groupby(grouper_series) partitions = val_frame.groupby(grouper_series)
# calculate flags for every partition
to_flag = [] to_flag = []
for _, partition in partitions: for _, partition in partitions:
if partition.empty | (partition.shape[0] < partition_min): if partition.empty | (partition.shape[0] < partition_min):
...@@ -131,10 +136,10 @@ def _expFit(val_frame, scoring_method='kNNMaxGap', n_neighbors=10, iter_start=0. ...@@ -131,10 +136,10 @@ def _expFit(val_frame, scoring_method='kNNMaxGap', n_neighbors=10, iter_start=0.
return val_frame.index[sorted_i[iter_index:]] return val_frame.index[sorted_i[iter_index:]]
@register("spikes_oddWater") @register("spikes_multivariateKNNScoring")
def flagSpikes_oddWater(data, field, flagger, fields, trafo='normScale', alpha=0.05, binning='auto', n_neighbors=2, def flagSpikes_multivariateKNNScoring(data, field, flagger, fields, trafo='normScale', alpha=0.05, binning='auto', n_neighbors=2,
iter_start=0.5, scoring_method='kNNMaxGap', threshing='stray', stray_partition=None, iter_start=0.5, scoring_method='kNNMaxGap', threshing='stray', stray_partition=None,
**kwargs): **kwargs):
trafo = composeFunction(trafo.split(',')) trafo = composeFunction(trafo.split(','))
# data fransformation/extraction # data fransformation/extraction
......
...@@ -11,7 +11,7 @@ from saqc.funcs.spike_detection import ( ...@@ -11,7 +11,7 @@ from saqc.funcs.spike_detection import (
flagSpikes_slidingZscore, flagSpikes_slidingZscore,
flagSpikes_basic, flagSpikes_basic,
flagSpikes_limitRaise, flagSpikes_limitRaise,
flagSpikes_oddWater flagSpikes_multivariateKNNScoring
) )
from test.common import TESTFLAGGER from test.common import TESTFLAGGER
...@@ -112,7 +112,7 @@ def test_flagSpikesOddWater(dat, flagger): ...@@ -112,7 +112,7 @@ def test_flagSpikesOddWater(dat, flagger):
fields = ['data1', 'data2'] fields = ['data1', 'data2']
data = pd.DataFrame({'data1': data1.squeeze(), 'data2': data2.squeeze()}, index=data1.index) data = pd.DataFrame({'data1': data1.squeeze(), 'data2': data2.squeeze()}, index=data1.index)
flagger = flagger.initFlags(data) flagger = flagger.initFlags(data)
_, flagger_result = flagSpikes_oddWater( _, flagger_result = flagSpikes_multivariateKNNScoring(
data, field, flagger, fields=fields, binning=50, trafo='np.log', data, field, flagger, fields=fields, binning=50, trafo='np.log',
iter_start=0.95, n_neighbors=10 iter_start=0.95, n_neighbors=10
) )
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment