implemented mad

funcs/functions.py

@@ -53,7 +53,7 @@ def flagConstant(data, flags, field, flagger, eps,
         if valueRange(dates_chunk) < length:
             continue
         if valueRange(values_chunk) < eps:
-            flagcol[start_idx:end_idx] = flagger.setFlags(**kwargs)
+            flagcol[start_idx:end_idx] = flagger.setFlags(flagcol[start_idx:end_idx], **kwargs)
 
     data[field] = datacol
     flags[field] = flagcol
@@ -64,5 +64,37 @@ def flagManual(data, flags, field, flagger, **kwargs):
     return data, flags
 
 
-def flagMad(data, flags, field, flagger, **kwargs):
+def flagMad(data, flags, field, flagger, length, z, deriv, **kwargs):
+
+    def _flagMad(data: np.ndarray, z: int, deriv: int) -> np.ndarray:
+        # NOTE: numpy is at least twice as fast as numba.jit(nopython)
+        # median absolute deviation
+        for i in range(deriv):
+            data[i+1:] = np.diff(data[i:])
+            data[i] = np.nan
+        median = np.nanmedian(data)
+        mad = np.nanmedian(np.abs(data-median))
+        tresh = mad * (z/0.6745)
+        with np.errstate(invalid="ignore"):
+            return (data < (median - tresh)) | (data > (median + tresh))
+
+    datacol = data[field]
+    flagcol = flags[field]
+
+    values = (datacol
+              .mask(flagger.isFlagged(flagcol))
+              .values)
+
+    window = pd.to_timedelta(length) - pd.to_timedelta(data.index.freq)
+    mask = np.zeros_like(values, dtype=bool)
+
+    for start_idx, end_idx in slidingWindowIndices(datacol.index, window, "1D"):
+        mad_flags = _flagMad(values[start_idx:end_idx], z, deriv)
+        # reset the mask
+        mask[:] = False
+        mask[start_idx:end_idx] = mad_flags
+        flagcol[mask] = flagger.setFlag(flagcol[mask], **kwargs)
+
+    flags[field] = flagcol
+
     return data, flags

resources/meta.csv

 headerout,date start,date end,Flag_1,Flag_2,Flag_3,Flag_4
-SHFsc04cal,2013-04-04 11:00:00,,manflag,"generic, {func: ismissing(this)}",,
+SHFsc04cal,2013-04-04 11:00:00,,manflag,"generic, {func: ismissing(this)}","mad, {z: 4, length: 5h, deriv: 1}",
 SHFsc05cal,2013-04-04 11:00:00,,manflag,"generic, {func: ismissing(this)}",,
 SHFsc06cal,2013-04-04 11:00:00,2014-04-04 07:50:00,manflag,"generic, {func: ismissing(this)}",,
 dendro_185,2014-04-03 14:10:00,,manflag,"generic, {func: ismissing(this)}",,"generic, {func: T5_Ptemp < 0, flag_period: 5h}"