added plot feature

config.py

@@ -18,6 +18,7 @@ class Params:
     FLAGPERIOD = "flag_period"
     FLAGVALUES = "flag_values"
     FLAG = "flag"
+    PLOT = "plot"
 
 
 # FUNCMAP = {

core.py

@@ -3,6 +3,7 @@
 
 import numpy as np
 import pandas as pd
+import matplotlib as mpl
 
 from config import Fields, Params
 from funcs import flagDispatch
@@ -117,6 +118,11 @@ def runner(meta, flagger, data, flags=None, nodata=np.nan):
                 fchunk = fchunk.astype({
                     c: flagger.flags for c in fchunk.columns if flagger.flag_fields[0] in c})
 
+            if Params.PLOT in flag_params:
+                new = flagger.getFlags(fchunk[varname])
+                mask = old != new
+                plot(dchunk, fchunk, mask, varname, flagger, title=flag_test)
+
             data.loc[start_date:end_date] = dchunk
             flags[start_date:end_date] = fchunk.squeeze()
 
@@ -124,6 +130,85 @@ def runner(meta, flagger, data, flags=None, nodata=np.nan):
     return data, flags
 
 
+def plot(data, flags, flagmask, varname, flagger, interactive_backend=True, title="Data Plot"):
+    # the flagmask is True for flags to be shown False otherwise
+    if not interactive_backend:
+        # Import plot libs without interactivity, if not needed. This ensures that this can
+        # produce an plot.png even if tkinter is not installed. E.g. if one want to run this
+        # on machines without X-Server aka. graphic interface.
+        mpl.use('Agg')
+    else:
+        mpl.use('TkAgg')
+    from matplotlib import pyplot as plt
+    # needed for datetime conversion
+    from pandas.plotting import register_matplotlib_converters
+    register_matplotlib_converters()
+
+    def plot_vline(plt, points, color='blue'):
+        # workaround for ax.vlines() as this work unexpected
+        for point in points:
+            plt.axvline(point, color=color, linestyle=':')
+
+    def _plot(varname, ax):
+        x = data.index
+        y = data[varname]
+        flags_ = flags[varname]
+        nrofflags = len(flagger.flags.categories)
+        ax.plot(x, y, '-',markersize=1, color='silver')
+        if nrofflags == 3:
+            colors = {0:'silver', 1:'lime', 2:'red'}
+        if nrofflags == 4:
+            colors = {0:'silver', 1:'lime', 2:'yellow', 3:'red'}
+
+        # plot (all) data in silver
+        ax.plot(x, y, '-', color='silver', label='data')
+        # plot (all) missing data in silver
+        nans = y.isna()
+        ylim = plt.ylim()
+        flagged = flagger.isFlagged(flags_)
+        idx = y.index[nans & ~flagged]
+        # ax.vlines(idx, *ylim, linestyles=':', color='silver', label="missing")
+        plot_vline(ax, idx, color='silver')
+
+        # plot all flagged data in black
+        ax.plot(x[flagged], y[flagged], '.', color='black', label="flagged by other test")
+        # plot all flagged missing data (flagged before) in black
+        idx = y.index[nans & flagged & ~flagmask]
+        # ax.vlines(idx, *ylim, linestyles=':', color='black')
+        plot_vline(ax, idx, color='black')
+        ax.set_ylabel(varname)
+
+        # plot currently flagged data in color of flag
+        for i, f in enumerate(flagger.flags):
+            if i == 0:
+                continue
+            flagged = flagger.isFlagged(flags_, flag=f) & flagmask
+            label = f"flag: {f}" if i else 'data'
+            ax.plot(x[flagged], y[flagged], '.', color=colors[i], label=label)
+            idx = y.index[nans & flagged]
+            # ax.vlines(idx, *ylim, linestyles=':', color=colors[i])
+            plot_vline(ax, idx, color=colors[i])
+
+    if not isinstance(varname, (list, set)):
+        varname = set([varname])
+    plots = len(varname)
+    if plots > 1:
+        fig, axes = plt.subplots(plots, 1, sharex=True)
+        axes[0].set_title(title)
+        for i, v in enumerate(varname):
+            _plot(v, axes[i])
+    else:
+        fig, ax = plt.subplots()
+        plt.title(title)
+        _plot(varname.pop(), ax)
+
+    plt.xlabel('time')
+    # dummy plot for label `missing` see plot_vline for more info
+    plt.plot([], [], ':', color='silver', label="missing data")
+    plt.legend()
+    plt.show()
+    
+
 def prepareMeta(meta, data):
     # NOTE: an option needed to only pass tests within an file and deduce
     #       everything else from data