Merge branch 'plotInHistory' into cookBux

saqc/core/modules/tools.py

@@ -38,7 +38,8 @@ class Tools(ModuleBase):
         path: Optional[str] = None,
         max_gap: Optional[FreqString] = None,
         stats: bool = False,
-        plot_kwargs: Optional[dict] = None,
+        history: Optional[Literal["valid", "complete"]] = "valid",
+        ax_kwargs: Optional[dict] = None,
         fig_kwargs: Optional[dict] = None,
         scatter_kwargs: Optional[dict] = None,
         stats_dict: Optional[dict] = None,

saqc/funcs/tools.py

@@ -252,7 +252,9 @@ def plot(
     path: Optional[str] = None,
     max_gap: Optional[FreqString] = None,
     stats: bool = False,
-    plot_kwargs: Optional[dict] = None,
+    history: Optional[Literal["valid", "complete"]] = "valid",
+    s: Optional[slice] = None,
+    ax_kwargs: Optional[dict] = None,
     fig_kwargs: Optional[dict] = None,
     scatter_kwargs: Optional[dict] = None,
     stats_dict: Optional[dict] = None,
@@ -294,20 +296,26 @@ def plot(
     stats : bool, default False
         Whether to include statistics table in plot.
 
-    plot_kwargs : dict, default None
+    history : {"valid", "complete", None}, default "valid"
+        Discriminate the plotted flags with respect to the tests they originate from.
+        * "valid" - Only plot those flags, that do not get altered or "unflagged" by subsequent tests. Only list tests
+          in the legend, that actually contributed flags to the overall resault.
+        * "complete" - plot all the flags set and list all the tests ran on a variable. Suitable for debugging/tracking.
+        * None - just plot the resulting flags for one variable, without any historical meta information.
+
+    s : slice or Offset, default None
+        Parameter, that determines a chunk of the data to be plotted /
+        processed. `s` can be anything, that is a valid argument to the ``pandas.Series.__getitem__`` method.
+
+    ax_kwargs : dict, default None
+        ax_kwargs : dict, default None
         Keyword arguments controlling plot generation. Will be passed on to the
         ``Matplotlib.axes.Axes.set()`` property batch setter for the axes showing the
-        data plot. The most relevant of those properties might be "ylabel", "title" and
-        "ylim". In Addition, following options are available:
-
-        * {'slice': s} property, that determines a chunk of the data to be plotted /
-            processed. `s` can be anything, that is a valid argument to the
-            ``pandas.Series.__getitem__`` method.
-        * {'history': str}
-            * str="all": All the flags are plotted with colored dots, refering to the
-                tests they originate from
-            * str="valid": - same as 'all' - but only plots those flags, that are not
-                removed by later tests
+        data plot. The most relevant of those properties might be "ylabel",
+        "title" and in addition: "ylim".
+        The "ylim" keyword can be passed a slice object with date offset entries to controll figure
+        scope.
+
 
     fig_kwargs : dict, default None
         Keyword arguments controlling figure generation. In interactive mode,
@@ -318,6 +326,8 @@ def plot(
         Keyword arguments controlling the appearance of the dots, marking flagged values.
         Dict just gets passed on to the matplotlib.pyplot.scatter method. Keywords of interest may be:
         ``"alpha"`` (transparancy), ``"marker"`` (marker appearance) and ``"s"`` (dot size).
+        The ``"marker"`` and ``"color"`` keywords can also be passed lists, that the plotting routine will then cycle
+        through.
 
     store_kwargs : dict, default {}
         Keywords to be passed on to the ``matplotlib.pyplot.savefig`` method, handling
@@ -376,7 +386,8 @@ def plot(
         level=kwargs.get("flag", BAD),
         max_gap=max_gap,
         stats=stats,
-        plot_kwargs=plot_kwargs,
+        history=history,
+        ax_kwargs=ax_kwargs,
         fig_kwargs=fig_kwargs,
         scatter_kwargs=scatter_kwargs,
         stats_dict=stats_dict,

saqc/lib/plotting.py

@@ -2,9 +2,13 @@
 # -*- coding: utf-8 -*-
 
 from typing import Optional
-
+from typing_extensions import Literal
+from saqc.lib.tools import toSequence
 import pandas as pd
+import numpy as np
 import matplotlib as mpl
+import itertools
+import matplotlib.pyplot as plt
 
 from saqc.constants import *
 from saqc.core import Flags
@@ -27,7 +31,8 @@ def makeFig(
     level: float,
     max_gap: Optional[FreqString] = None,
     stats: bool = False,
-    plot_kwargs: Optional[dict] = None,
+    history: Optional[Literal["valid", "complete"]] = "valid",
+    ax_kwargs: Optional[dict] = None,
     fig_kwargs: Optional[dict] = None,
     scatter_kwargs: Optional[dict] = None,
     stats_dict: Optional[dict] = None,
@@ -58,21 +63,13 @@ def makeFig(
     stats : bool, default False
         Whether to include statistics table in plot.
 
-    plot_kwargs : dict, default None
+    ax_kwargs : dict, default None
         Keyword arguments controlling plot generation. Will be passed on to the
         ``Matplotlib.axes.Axes.set()`` property batch setter for the axes showing the
         data plot. The most relevant of those properties might be "ylabel",
-        "title" and "ylim".
-        In Addition, following options are available:
-
-        * {'slice': s} property, that determines a chunk of the data to be plotted /
-            processed. `s` can be anything,
-            that is a valid argument to the ``pandas.Series.__getitem__`` method.
-        * {'history': str}
-            * str="all": All the flags are plotted with colored dots, refering to the
-                tests they originate from
-            * str="valid": - same as 'all' - but only plots those flags, that are not
-                removed by later tests
+        "title" and in addition: "ylim".
+        The "ylim" keyword can be passed a slice object with date offset entries.
+
     fig_kwargs : dict, default None
         Keyword arguments controlling figure generation. None defaults to
         {"figsize": (16, 9)}
@@ -112,8 +109,8 @@ def makeFig(
 
     >>> func = lambda x, y, z: round((x.isna().sum()) / len(x), 2)
     """
-    if plot_kwargs is None:
-        plot_kwargs = {"history": False}
+    if ax_kwargs is None:
+        ax_kwargs = {}
     if fig_kwargs is None:
         fig_kwargs = {}
     if scatter_kwargs is None:
@@ -124,7 +121,7 @@ def makeFig(
     # data retrieval
     d = data[field]
     # data slicing:
-    s = plot_kwargs.pop("slice", slice(None))
+    s = ax_kwargs.pop("ylim", slice(None))
     d = d[s]
     flags_vals = flags[field][s]
     flags_hist = flags.history[field].hist.loc[s]
@@ -157,8 +154,9 @@ def makeFig(
         flags_vals,
         flags_hist,
         flags_meta,
+        history,
         level,
-        plot_kwargs,
+        ax_kwargs,
         scatter_kwargs,
         na_mask,
     )
@@ -193,25 +191,43 @@ def _plotVarWithFlags(
     flags_vals,
     flags_hist,
     flags_meta,
+    history,
     level,
-    plot_kwargs,
+    ax_kwargs,
     scatter_kwargs,
     na_mask,
 ):
     ax.set_title(datser.name)
-    ax.plot(datser)
-    history = plot_kwargs.pop("history", False)
-    ax.set(**plot_kwargs)
+    ax.plot(datser, color="black")
+    ax.set(**ax_kwargs)
     if history:
+        shape_cycle = scatter_kwargs.pop("marker", "o")
+        shape_cycle = itertools.cycle(toSequence(shape_cycle))
+        color_cycle = scatter_kwargs.pop(
+            "color", plt.rcParams["axes.prop_cycle"].by_key()["color"]
+        )
+        color_cycle = itertools.cycle(toSequence(color_cycle))
         for i in flags_hist.columns:
             scatter_kwargs.update({"label": flags_meta[i]["func"].split(".")[-1]})
-            if history == "all":
-                _plotFlags(ax, datser, flags_hist[i], na_mask, level, scatter_kwargs)
+            flags_i = flags_hist[i].astype(float)
+            if history == "complete":
+                _plotFlags(ax, datser, flags_i, na_mask, level, scatter_kwargs)
             if history == "valid":
+                # only plot those flags, that do not get altered later on:
+                mask = flags_i.eq(flags_vals)
+                flags_i[~mask] = np.nan
+                # Skip plot, if the test did not have no effect on the all over flagging result. This avoids
+                # legend overflow
+                if ~(flags_i >= level).any():
+                    continue
+
+                scatter_kwargs.update(
+                    {"color": next(color_cycle), "marker": next(shape_cycle)}
+                )
                 _plotFlags(
                     ax,
                     datser,
-                    flags_hist[i].combine(flags_vals, min),
+                    flags_i,
                     na_mask,
                     level,
                     scatter_kwargs,

tests/funcs/test_tools.py

@@ -28,11 +28,9 @@ def test_makeFig():
 
     d_saqc = d_saqc.tools.plot(field="data", path="")
     d_saqc = d_saqc.tools.plot(
-        field="data", path=dummy_path, plot_kwargs={"history": "valid"}, stats=True
+        field="data", path=dummy_path, history="valid", stats=True
     )
+    d_saqc = d_saqc.tools.plot(field="data", path=dummy_path, history="complete")
     d_saqc = d_saqc.tools.plot(
-        field="data", path=dummy_path, plot_kwargs={"history": "all"}
-    )
-    d_saqc = d_saqc.tools.plot(
-        field="data", path=dummy_path, plot_kwargs={"slice": "2000-10"}, stats=True
+        field="data", path=dummy_path, ax_kwargs={"ylim": "2000-10"}, stats=True
     )