Skip to content
Snippets Groups Projects

Fixes to the speed improved spikes.flagBasic function

Merged David Schäfer requested to merge spikeBasicFixes into develop
2 files
+ 13
14
Compare changes
  • Side-by-side
  • Inline
Files
2
@@ -940,7 +940,7 @@ def spikes_flagBasic(data, field, flagger, thresh, tolerance, window, numba_kick
to_roll = post_jumps.reindex(dataseries.index, method="bfill", tolerance=window, fill_value=False).dropna()
# define spike testing function to roll with:
def spike_tester(chunk, thresh=thresh, tol=tolerance):
def spikeTester(chunk, thresh=thresh, tol=tolerance):
# signum change!!!
chunk_stair = (np.abs(chunk - chunk[-1]) < thresh)[::-1].cumsum()
initial = np.searchsorted(chunk_stair, 2)
@@ -957,21 +957,21 @@ def spikes_flagBasic(data, field, flagger, thresh, tolerance, window, numba_kick
engine=None
if roll_mask.sum() > numba_kickin:
engine = 'numba'
result = customRolling(to_roll, window, spike_tester, roll_mask, closed='both', engine=engine)
result = customRolling(to_roll, window, spikeTester, roll_mask, closed='both', engine=engine)
group_col = np.nancumsum(result)
group_frame = pd.DataFrame({'group_col': group_col[:-1],
'diff_col': np.diff(group_col).astype(int)},
index=result.index[:-1])
groups = group_frame.groupby('group_col')
def g_func(x):
r = np.array([False] * x.shape[0])
def gFunc(x):
r = np.zeros(shape=x.shape[0], dtype=np.bool)
r[-x[-1]:] = True
return r
to_flag = groups['diff_col'].transform(g_func)
flagger = flagger.setFlags(field, to_flag, **kwargs)
to_flag = groups['diff_col'].transform(gFunc)
flagger = flagger.setFlags(field, to_flag[to_flag == True].index, **kwargs)
return data, flagger
Loading