From 677769a70712250985628149802d965f8e0a785b Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Thu, 6 Jun 2019 12:01:11 +0200
Subject: [PATCH] added plot feature
---
config.py | 1 +
core.py | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 86 insertions(+)
diff --git a/config.py b/config.py
index 340db537d..9ff511c5b 100644
--- a/config.py
+++ b/config.py
@@ -18,6 +18,7 @@ class Params:
FLAGPERIOD = "flag_period"
FLAGVALUES = "flag_values"
FLAG = "flag"
+ PLOT = "plot"
# FUNCMAP = {
diff --git a/core.py b/core.py
index c13a2f69e..dd1df1828 100644
--- a/core.py
+++ b/core.py
@@ -3,6 +3,7 @@
import numpy as np
import pandas as pd
+import matplotlib as mpl
from config import Fields, Params
from funcs import flagDispatch
@@ -117,6 +118,11 @@ def runner(meta, flagger, data, flags=None, nodata=np.nan):
fchunk = fchunk.astype({
c: flagger.flags for c in fchunk.columns if flagger.flag_fields[0] in c})
+ if Params.PLOT in flag_params:
+ new = flagger.getFlags(fchunk[varname])
+ mask = old != new
+ plot(dchunk, fchunk, mask, varname, flagger, title=flag_test)
+
data.loc[start_date:end_date] = dchunk
flags[start_date:end_date] = fchunk.squeeze()
@@ -124,6 +130,85 @@ def runner(meta, flagger, data, flags=None, nodata=np.nan):
return data, flags
+def plot(data, flags, flagmask, varname, flagger, interactive_backend=True, title="Data Plot"):
+ # the flagmask is True for flags to be shown False otherwise
+ if not interactive_backend:
+ # Import plot libs without interactivity, if not needed. This ensures that this can
+ # produce an plot.png even if tkinter is not installed. E.g. if one want to run this
+ # on machines without X-Server aka. graphic interface.
+ mpl.use('Agg')
+ else:
+ mpl.use('TkAgg')
+ from matplotlib import pyplot as plt
+ # needed for datetime conversion
+ from pandas.plotting import register_matplotlib_converters
+ register_matplotlib_converters()
+
+ def plot_vline(plt, points, color='blue'):
+ # workaround for ax.vlines() as this work unexpected
+ for point in points:
+ plt.axvline(point, color=color, linestyle=':')
+
+ def _plot(varname, ax):
+ x = data.index
+ y = data[varname]
+ flags_ = flags[varname]
+ nrofflags = len(flagger.flags.categories)
+ ax.plot(x, y, '-',markersize=1, color='silver')
+ if nrofflags == 3:
+ colors = {0:'silver', 1:'lime', 2:'red'}
+ if nrofflags == 4:
+ colors = {0:'silver', 1:'lime', 2:'yellow', 3:'red'}
+
+ # plot (all) data in silver
+ ax.plot(x, y, '-', color='silver', label='data')
+ # plot (all) missing data in silver
+ nans = y.isna()
+ ylim = plt.ylim()
+ flagged = flagger.isFlagged(flags_)
+ idx = y.index[nans & ~flagged]
+ # ax.vlines(idx, *ylim, linestyles=':', color='silver', label="missing")
+ plot_vline(ax, idx, color='silver')
+
+ # plot all flagged data in black
+ ax.plot(x[flagged], y[flagged], '.', color='black', label="flagged by other test")
+ # plot all flagged missing data (flagged before) in black
+ idx = y.index[nans & flagged & ~flagmask]
+ # ax.vlines(idx, *ylim, linestyles=':', color='black')
+ plot_vline(ax, idx, color='black')
+ ax.set_ylabel(varname)
+
+ # plot currently flagged data in color of flag
+ for i, f in enumerate(flagger.flags):
+ if i == 0:
+ continue
+ flagged = flagger.isFlagged(flags_, flag=f) & flagmask
+ label = f"flag: {f}" if i else 'data'
+ ax.plot(x[flagged], y[flagged], '.', color=colors[i], label=label)
+ idx = y.index[nans & flagged]
+ # ax.vlines(idx, *ylim, linestyles=':', color=colors[i])
+ plot_vline(ax, idx, color=colors[i])
+
+ if not isinstance(varname, (list, set)):
+ varname = set([varname])
+ plots = len(varname)
+ if plots > 1:
+ fig, axes = plt.subplots(plots, 1, sharex=True)
+ axes[0].set_title(title)
+ for i, v in enumerate(varname):
+ _plot(v, axes[i])
+ else:
+ fig, ax = plt.subplots()
+ plt.title(title)
+ _plot(varname.pop(), ax)
+
+ plt.xlabel('time')
+ # dummy plot for label `missing` see plot_vline for more info
+ plt.plot([], [], ':', color='silver', label="missing data")
+ plt.legend()
+ plt.show()
+
+
def prepareMeta(meta, data):
# NOTE: an option needed to only pass tests within an file and deduce
# everything else from data
--
GitLab