From 02cadb1ff745183f77c33172e29b24fb2ba5b7e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20L=C3=BCnenschlo=C3=9F?= <peter.luenenschloss@ufz.de>
Date: Wed, 16 Aug 2023 16:59:36 +0200
Subject: [PATCH] Multi var plot
---
CHANGELOG.md | 7 +-
docs/cookbooks/DriftDetection.rst | 80 +----
saqc/funcs/outliers.py | 2 +-
saqc/funcs/tools.py | 116 +++++--
saqc/lib/plotting.py | 495 ++++++++++++++++++++++--------
tests/funcs/test_tools.py | 4 +-
6 files changed, 486 insertions(+), 218 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ccbc82abe..c0c4d6d66 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,8 @@ SPDX-License-Identifier: GPL-3.0-or-later
## Unreleased
[List of commits](https://git.ufz.de/rdm-software/saqc/-/compare/v2.4.0...develop)
### Added
+- add multivariate plotting options to `plot`
+- added `plot_kwargs` keyword to `plot` function
- added checks and unified error message for common inputs.
- added command line `--version` option
- `-ll` CLI option as a shorthand for `--log-level`
@@ -20,11 +22,15 @@ SPDX-License-Identifier: GPL-3.0-or-later
`bool` instead of one of `[None, "linear"]`
### Removed
- removed deprecated `DictOfSeries.to_df`
+- removed plotting option with complete history (`history="complete"`)
### Fixed
- Bug in `SaQC.assignChangePointCluster` and `SaQC.flagChangePoints`: A tuple passed
to `min_period` was only recognised if also `window` was a tuple.
- `SaQC.propagateFlags` was overwriting existing flags
### Deprecated
+- Deprecate `plot` parameter `phaseplot` in favor of usage with `mode="biplot"`
+- Deprecate `plot` parameter `cyclestart` in favor of usage with `marker_kwargs`
+- Deprecate option `"complete"` for `plot` funcs parameter `history`
## [2.4.1](https://git.ufz.de/rdm-software/saqc/-/tags/v2.4.1) - 2023-06-22
[List of commits](https://git.ufz.de/rdm-software/saqc/-/compare/v2.4.0...develop)
@@ -61,7 +67,6 @@ SPDX-License-Identifier: GPL-3.0-or-later
- Deprecate `interpolate`, `linear` and `shift` in favor of `align`
- Deprecate `roll` in favor of `rolling`
- Deprecate `DictOfSeries.to_df` in favor of `DictOfSeries.to_pandas`
-
## [2.3.0](https://git.ufz.de/rdm-software/saqc/-/tags/v2.3.0) - 2023-01-17
[List of commits](https://git.ufz.de/rdm-software/saqc/-/compare/v2.2.1...v2.3.0)
### Added
diff --git a/docs/cookbooks/DriftDetection.rst b/docs/cookbooks/DriftDetection.rst
index 145247e74..c27852ecc 100644
--- a/docs/cookbooks/DriftDetection.rst
+++ b/docs/cookbooks/DriftDetection.rst
@@ -100,18 +100,22 @@ Example Data Import
import pandas as pd
data = pd.read_csv('../resources/data/tempSensorGroup.csv', index_col=0)
data.index = pd.DatetimeIndex(data.index)
+ variables = ['temp1 [degC]', 'temp2 [degC]', 'temp3 [degC]', 'temp4 [degC]', 'temp5 [degC]']
qc = saqc.SaQC(data)
We load the example `data set <https://git.ufz.de/rdm-software/saqc/-/blob/develop/docs/resources/data/tempsenorGroup.csv>`_
from the *saqc* repository using the `pandas <https://pandas.pydata.org/>`_ csv
file reader. Subsequently, we cast the index of the imported data to `DatetimeIndex`
-and use the dataframe's `plot` method, to inspect the imported data:
+instantiate a saqc object and plot the data:
.. doctest:: flagDriftFromNorm
+ >>> import saqc
>>> data = pd.read_csv('./resources/data/tempSensorGroup.csv', index_col=0)
>>> data.index = pd.DatetimeIndex(data.index)
- >>> data.plot() # doctest: +SKIP
+ >>> variables = ['temp1 [degC]', 'temp2 [degC]', 'temp3 [degC]', 'temp4 [degC]', 'temp5 [degC]']
+ >>> qc = saqc.SaQC(data)
+ >>> qc.plot(variables) # doctest: +SKIP
.. plot::
@@ -119,22 +123,13 @@ and use the dataframe's `plot` method, to inspect the imported data:
:include-source: False
:class: center
- data.plot()
+ qc.plot(variables)
Example Algorithm Application
-----------------------------
-Looking at our example data set more closely, we see that 2 of the 5 variables start to drift away.
-
-.. plot::
- :context: close-figs
- :include-source: False
- :class: center
- :caption: 2 variables start departing the majority group of variables (the group containing more than ``frac`` variables) around july.
-
- data['2017-05':'2017-11'].plot()
-
+Looking at the example data set more closely, we see that 2 of the 5 variables start to drift away.
.. plot::
:context: close-figs
@@ -142,17 +137,9 @@ Looking at our example data set more closely, we see that 2 of the 5 variables s
:class: center
:caption: 2 variables are departed from the majority group of variables (the group containing more than ``frac`` variables) by the end of the year.
- data['2017-09':'2018-01'].plot()
-
-Lets try to detect those drifts via saqc. There for we import the *saqc* package and instantiate a :py:class:`saqc.SaQC`
-object with the data:
-
-.. doctest:: flagDriftFromNorm
-
- >>> import saqc
- >>> qc = saqc.SaQC(data)
+ qc.plot(variables, xscope=slice('2017-05', '2017-11'))
-The changes we observe in the data seem to develop significantly only in temporal spans over a month,
+Lets try to detect those drifts via saqc. The changes we observe in the data seem to develop significantly only in temporal spans over a month,
so we go for ``"1M"`` as value for the
``window`` parameter. We identified the majority group as a group containing three variables, whereby two variables
seem to be scattered away, so that we can leave the ``frac`` value at its default ``.5`` level.
@@ -179,55 +166,12 @@ Lets check the results:
.. doctest:: flagDriftFromNorm
- >>> qc.plot('temp1 [degC]') # doctest: +SKIP
+ >>> qc.plot(variables, marker_kwargs={'alpha':.3, 's': 1, 'color': 'red', 'edgecolor': 'face'}) # doctest: +SKIP
.. plot::
:context: close-figs
:include-source: False
:class: center
- qc.plot('temp1 [degC]')
-
-.. doctest:: flagDriftFromNorm
-
- >>> qc.plot('temp2 [degC]') # doctest: +SKIP
-
-.. plot::
- :context: close-figs
- :include-source: False
- :class: center
-
- qc.plot('temp2 [degC]')
-
-.. doctest:: flagDriftFromNorm
-
- >>> qc.plot('temp3 [degC]') # doctest: +SKIP
-
-.. plot::
- :context: close-figs
- :include-source: False
- :class: center
-
- qc.plot('temp3 [degC]')
-
-.. doctest:: flagDriftFromNorm
-
- >>> qc.plot('temp4 [degC]') # doctest: +SKIP
-
-.. plot::
- :context: close-figs
- :include-source: False
- :class: center
-
- qc.plot('temp4 [degC]')
-
-.. doctest:: flagDriftFromNorm
-
- >>> qc.plot('temp5 [degC]') # doctest: +SKIP
-
-.. plot::
- :context: close-figs
- :include-source: False
- :class: center
+ qc.plot(variables, marker_kwargs={'alpha':.3, 's': 1, 'color': 'red', 'edgecolor': 'face'})
- qc.plot('temp5 [degC]')
\ No newline at end of file
diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py
index 3be87afc9..7b76ca3b5 100644
--- a/saqc/funcs/outliers.py
+++ b/saqc/funcs/outliers.py
@@ -329,7 +329,7 @@ class OutliersMixin:
:include-source: False
:class: center
- qc.plot('sac254_raw')
+ qc.plot('sac254_raw')
We apply :py:meth:`~saqc.SaqC.flagUniLOF` in with default parameter
values. Meaning, that the main calibration paramters :py:attr:`n`
diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py
index bce714086..599ac6bc9 100644
--- a/saqc/funcs/tools.py
+++ b/saqc/funcs/tools.py
@@ -8,6 +8,7 @@
from __future__ import annotations
import pickle
+import warnings
from typing import TYPE_CHECKING, Optional
import matplotlib as mpl
@@ -197,18 +198,25 @@ class ToolsMixin:
self._flags[mask, field] = UNFLAGGED
return self
- @register(mask=[], demask=[], squeeze=[])
+ @register(
+ mask=[],
+ demask=[],
+ squeeze=[],
+ multivariate=True,
+ )
def plot(
self: "SaQC",
- field: str,
+ field: str | list[str],
path: str | None = None,
max_gap: str | None = None,
+ mode: Literal["subplots", "oneplot"] | str = "oneplot",
history: Literal["valid", "complete"] | list[str] | None = "valid",
xscope: slice | None = None,
- phaseplot: str | None = None,
store_kwargs: dict | None = None,
ax: mpl.axes.Axes | None = None,
ax_kwargs: dict | None = None,
+ marker_kwargs: dict | None = None,
+ plot_kwargs: dict | None = None,
dfilter: float = FILTER_NONE,
**kwargs,
) -> "SaQC":
@@ -235,26 +243,29 @@ class ToolsMixin:
plotting. If an offset string is passed, only points that have a distance
below ``max_gap`` are connected via the plotting line.
+ mode :
+ How to process multiple variables to be plotted:
+ * `"oneplot"` : plot all variables with their flags in one axis (default)
+ * `"subplots"` : generate subplot grid where each axis contains one variable plot with associated flags
+ * `"biplot"` : plotting first and second variable in field against each other in a scatter plot (point cloud).
+
history :
Discriminate the plotted flags with respect to the tests they originate from.
* ``"valid"``: Only plot flags, that are not overwritten by subsequent tests.
Only list tests in the legend, that actually contributed flags to the overall
result.
- * ``"complete"``: Plot all flags set and list all the tests executed on a variable.
- Suitable for debugging/tracking.
* ``None``: Just plot the resulting flags for one variable, without any historical
and/or meta information.
* list of strings: List of tests. Plot flags from the given tests, only.
+ * ``complete`` (not recommended, deprecated): Plot all the flags set by any test, independently from them being removed or modified by
+ subsequent modifications. (this means: plotted flags do not necessarily match with flags ultimately
+ assigned to the data)
xscope :
- Determine a chunk of the data to be plotted processed. ``xscope`` can be anything,
+ Determine a chunk of the data to be plotted. ``xscope`` can be anything,
that is a valid argument to the ``pandas.Series.__getitem__`` method.
- phaseplot :
- If a string is passed, plot ``field`` in the phase space it forms together with the
- variable ``phaseplot``.
-
ax :
If not ``None``, plot into the given ``matplotlib.Axes`` instance, instead of a
newly created ``matplotlib.Figure``. This option offers a possibility to integrate
@@ -267,21 +278,86 @@ class ToolsMixin:
To reopen a pickled figure execute: ``pickle.load(open(savepath, "w")).show()``
ax_kwargs :
- Axis keywords. Change the axis labeling defaults. Most important keywords:
- ``"xlabel"``, ``"ylabel"``, ``"title"``, ``"fontsize"``, ``"cycleskip"``.
+ Axis keywords. Change axis specifics. Those are passed on to the
+ `matplotlib.axes.Axes.set <https://matplotlib.org/stable/api/_as_gen/matplotlib.axes.Axes.set.html>`_
+ method and can have the options listed there.
+ The following options are `saqc` specific:
+
+ * ``"xlabel"``: Either single string, that is to be attached to all x-axis´, or
+ a List of labels, matching the number of variables to plot in length, or a dictionary, directly
+ assigning labels to certain fields - defaults to ``None`` (no labels)
+ * ``"ylabel"``: Either single string, that is to be attached to all y-axis´, or
+ a List of labels, matching the number of variables to plot in length, or a dictionary, directly
+ assigning labels to certain fields - defaults to ``None`` (no labels)
+ * ``"title"``: Either a List of labels, matching the number of variables to plot in length, or a dictionary, directly
+ assigning labels to certain variables - defaults to ``None`` (every plot gets titled the plotted variables name)
+ * ``"fontsize"``: (float) Adjust labeling and titeling fontsize
+ * ``"nrows"``, ``"ncols"``: shape of the subplot matrix the plots go into: If both are assigned, a subplot
+ matrix of shape `nrows` x `ncols` is generated. If only one is assigned, the unassigned dimension is 1.
+ defaults to plotting into subplot matrix with 2 columns and the necessary number of rows to fit the
+ number of variables to plot.
+
+ marker_kwargs :
+ Keywords to modify flags marker appearance. The markers are set via the
+ `matplotlib.pyplot.scatter <https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.scatter.html>`_
+ method and can have the options listed there.
+ The following options are `saqc` specific:
+
+ * ``"cycleskip"``: (int) start the cycle of shapes that are assigned any flag-type with a certain lag - defaults to ``0`` (no skip)
+
+ plot_kwargs :
+ Keywords to modify data line appearance. The markers are set via the
+ `matplotlib.pyplot.plot <https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.scatter.html>`_
+ method and can have the options listed there.
+
+ Notes
+ -----
+
+ * Check/modify the module parameter `saqc.lib.plotting.SCATTER_KWARGS` to see/modify global marker defaults
+ * Check/modify the module parameter `saqc.lib.plotting.PLOT_KWARGS` to see/modify global plot line defaults
"""
+ if history == "complete":
+ warnings.warn(
+ "Plotting with history='complete' is deprecated and will be removed in a future release (2.5)."
+ "To get access to an saqc variables complete flagging history and analyze or plot it in detail, use flags"
+ "history acces via `qc._flags.history[variable_name].hist` and a plotting library, such as pyplot.\n"
+ "Minimal Pseudo example, having a saqc.SaQC instance `qc`, holding a variable `'data1'`, "
+ "and having matplotlib.pyplot imported as `plt`:\n\n"
+ "plt.plot(data)\n"
+ "for f in qc._flags.history['data1'].hist \n"
+ " markers = qc._flags.history['data1'].hist[f] > level \n"
+ " markers=data[markers] \n"
+ " plt.scatter(markers.index, markers.values) \n",
+ DeprecationWarning,
+ )
+
+ if "phaseplot" in kwargs:
+ warnings.warn(
+ 'Parameter "phaseplot" is deprecated and will be removed in a future release (2.5). Assign to parameter "mode" instead. (plot(field, mode=phaseplot))',
+ DeprecationWarning,
+ )
+ mode = kwargs["phaseplot"]
+
+ if "cycleskip" in (ax_kwargs or {}):
+ warnings.warn(
+ 'Passing "cycleskip" option with the "ax_kwargs" parameter is deprecated and will be removed in a future release (2.5). '
+ 'The option now has to be passed with the "marker_kwargs" parameter',
+ DeprecationWarning,
+ )
+ marker_kwargs["cycleskip"] = ax_kwargs.pop("cycleskip")
+
data, flags = self._data.copy(), self._flags.copy()
level = kwargs.get("flag", UNFLAGGED)
if dfilter < np.inf:
- data[field].loc[flags[field] >= dfilter] = np.nan
-
- if store_kwargs is None:
- store_kwargs = {}
+ for f in field:
+ data[f].loc[flags[f] >= dfilter] = np.nan
- if ax_kwargs is None:
- ax_kwargs = {}
+ store_kwargs = store_kwargs or {}
+ ax_kwargs = ax_kwargs or {}
+ marker_kwargs = marker_kwargs or {}
+ plot_kwargs = plot_kwargs or {}
if not path:
mpl.use(_MPL_DEFAULT_BACKEND)
@@ -293,12 +369,14 @@ class ToolsMixin:
field=field,
flags=flags,
level=level,
+ mode=mode,
max_gap=max_gap,
history=history,
xscope=xscope,
- phaseplot=phaseplot,
ax=ax,
ax_kwargs=ax_kwargs,
+ scatter_kwargs=marker_kwargs,
+ plot_kwargs=plot_kwargs,
)
if ax is None and not path:
diff --git a/saqc/lib/plotting.py b/saqc/lib/plotting.py
index a8f9b0540..5c5d60448 100644
--- a/saqc/lib/plotting.py
+++ b/saqc/lib/plotting.py
@@ -9,6 +9,7 @@
from __future__ import annotations
import itertools
+from collections import OrderedDict
import matplotlib as mpl
import matplotlib.pyplot as plt
@@ -19,18 +20,8 @@ from typing_extensions import Literal
from saqc.core import DictOfSeries, Flags
from saqc.lib.tools import toSequence
-STATSDICT = {
- "values total": lambda x, y, z: len(x),
- "invalid total (=NaN)": lambda x, y, z: x.isna().sum(),
- "invalid percentage": lambda x, y, z: round((x.isna().sum()) / len(x), 2),
- "flagged total": lambda x, y, z: (y >= z).sum(),
- "flagged percentage": lambda x, y, z: round(((y >= z).sum()) / len(x), 2),
-}
-
-PLOT_KWARGS = {"alpha": 0.8, "linewidth": 1}
-FIG_KWARGS = {"figsize": (16, 9)}
-
-_seaborn_color_palette = [
+# default color cycle for flags markers (seaborn color palette)
+MARKER_COL_CYCLE = [
(0.00784313725490196, 0.24313725490196078, 1.0),
(1.0, 0.48627450980392156, 0.0),
(0.10196078431372549, 0.788235294117647, 0.2196078431372549),
@@ -43,9 +34,19 @@ _seaborn_color_palette = [
(0.0, 0.8431372549019608, 1.0),
]
+# default color cycle for plot colors (many-in-one-plots)
+PLOT_COL_CYCLE = MARKER_COL_CYCLE # itertools.cycle(MARKER_COL_CYCLE)
+
+# default data plot configuration (color kwarg only effective for many-to-one-plots)
+PLOT_KWARGS = {"alpha": 0.8, "linewidth": 1, "color": PLOT_COL_CYCLE}
+
+# default figure configuration
+FIG_KWARGS = {"figsize": (16, 9)}
+
+# default flags markers configuration
SCATTER_KWARGS = {
"marker": ["s", "D", "^", "o", "v"],
- "color": _seaborn_color_palette,
+ "color": MARKER_COL_CYCLE,
"alpha": 0.7,
"zorder": 10,
"edgecolors": "black",
@@ -55,15 +56,17 @@ SCATTER_KWARGS = {
def makeFig(
data: DictOfSeries,
- field: str,
+ field: list[str],
flags: Flags,
level: float,
+ mode: Literal["subplots", "oneplot"] = "subplots",
max_gap: str | None = None,
history: Literal["valid", "complete"] | None | list[str] = "valid",
xscope: slice | None = None,
- phaseplot: str | None = None,
ax: mpl.axes.Axes | None = None,
ax_kwargs: dict | None = None,
+ scatter_kwargs: dict | None = None,
+ plot_kwargs: dict | None = None,
):
"""
Returns a figure object, containing data graph with flag marks for field.
@@ -82,6 +85,12 @@ def makeFig(
level : str, float, default None
Flaglevel above wich flagged values should be displayed.
+ mode: Literal["subplots", "oneplot"] | str = "oneplot"
+ How to process multiple variables to be plotted:
+ * `"oneplot"` : plot all variables with their flags in one axis (default)
+ * `"subplots"` : generate subplot grid where each axis contains one variable plot with associated flags
+ * `"biplot"` : plotting first and second variable in field against each other in a scatter plot (point cloud).
+
max_gap : str, default None
If None, all the points in the data will be connected, resulting in long linear
lines, where continous chunks of data is missing. Nans in the data get dropped
@@ -89,7 +98,7 @@ def makeFig(
below `max_gap` get connected via the plotting line.
- history : {"valid", "complete", None, list of strings}, default "valid"
+ history : {"valid", "complete", None, list of strings}, default "valid"
Discriminate the plotted flags with respect to the tests they originate from.
* "valid" - Only plot those flags, that do not get altered or "unflagged" by subsequent tests. Only list tests
@@ -100,12 +109,47 @@ def makeFig(
* list of strings - for any string ``s`` in the list, plot the flags set by test labeled, ``s`` - if ``s`` is
not present in the history labels, plot any flags, set by a test labeled ``s``
- xscope : slice or Offset, default None
- Parameter, that determines a chunk of the data to be plotted /
- processed. `s` can be anything, that is a valid argument to the ``pandas.Series.__getitem__`` method.
-
- phaseplot :
-
+ xscope :
+ Determine a chunk of the data to be plotted. ``xscope`` can be anything,
+ that is a valid argument to the ``pandas.Series.__getitem__`` method.
+
+ ax :
+ If not ``None``, plot into the given ``matplotlib.Axes`` instance, instead of a
+ newly created ``matplotlib.Figure``. This option offers a possibility to integrate
+ ``SaQC`` plots into custom figure layouts.
+
+ ax_kwargs :
+ Axis keywords. Change axis specifics. Those are passed on to the
+ `matplotlib.axes.Axes.set <https://matplotlib.org/stable/api/_as_gen/matplotlib.axes.Axes.set.html>`_
+ method and can have the options listed there.
+ The following options are `saqc` specific:
+
+ * ``"xlabel"``: Either single string, that is to be attached to all x-axis´, or
+ a List of labels, matching the number of variables to plot in length, or a dictionary, directly
+ assigning labels to certain fields - defaults to ``None`` (no labels)
+ * ``"ylabel"``: Either single string, that is to be attached to all y-axis´, or
+ a List of labels, matching the number of variables to plot in length, or a dictionary, directly
+ assigning labels to certain fields - defaults to ``None`` (no labels)
+ * ``"title"``: Either a List of labels, matching the number of variables to plot in length, or a dictionary, directly
+ assigning labels to certain variables - defaults to ``None`` (every plot gets titled the plotted variables name)
+ * ``"fontsize"``: (float) Adjust labeling and titeling fontsize
+ * ``"nrows"``, ``"ncols"``: shape of the subplot matrix the plots go into: If both are assigned, a subplot
+ matrix of shape `nrows` x `ncols` is generated. If only one is assigned, the unassigned dimension is 1.
+ defaults to plotting into subplot matrix with 2 columns and the necessary number of rows to fit the
+ number of variables to plot.
+
+ scatter_kwargs :
+ Keywords to modify flags marker appearance. The markers are set via the
+ `matplotlib.pyplot.scatter <https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.scatter.html>`_
+ method and can have the options listed there.
+ The following options are `saqc` specific:
+
+ * ``"cycleskip"``: (int) start the cycle of shapes that are assigned any flag-type with a certain lag - defaults to ``0`` (no skip)
+
+ plot_kwargs :
+ Keywords to modify data line appearance. The markers are set via the
+ `matplotlib.pyplot.plot <https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.scatter.html>`_
+ method and can have the options listed there.
Returns
-------
@@ -113,51 +157,81 @@ def makeFig(
figure object.
"""
-
- if ax_kwargs is None:
- ax_kwargs = {}
- # data retrieval
- d = data[field].copy(deep=False)
- d.name = field
- # data slicing:
xscope = xscope or slice(xscope)
- d = d[xscope]
- flags_vals = flags[field][xscope]
- flags_hist = flags.history[field].hist.loc[xscope]
- flags_meta = flags.history[field].meta
+ # data retrieval
+ d = dict()
+ na_mask = {}
+ for f in field:
+ chunk = data[f][xscope].rename(f)
+ mask = chunk.isna()
+ d[f] = chunk[~mask]
+ na_mask[f] = mask
+
+ flags_vals = {f: flags[f][xscope] for f in field}
+ flags_hist = {f: flags.history[f].hist.loc[xscope] for f in field}
+ flags_meta = {f: flags.history[f].meta for f in field}
# set fontsize:
- default = plt.rcParams["font.size"]
- plt.rcParams["font.size"] = ax_kwargs.pop("fontsize", None) or default
+ plt.rcParams["font.size"] = (
+ ax_kwargs.pop("fontsize", None) or plt.rcParams["font.size"]
+ )
- # set shapecycle start:
- cyclestart = ax_kwargs.pop("cycleskip", 0)
+ # set default axis sharing behavior (share x axis over rows if not explicitly opted sharex=False):
+ sharex = False
+ if len(d) > 1:
+ sharex = ax_kwargs.pop("sharex", True)
- na_mask = d.isna()
- d = d[~na_mask]
- if phaseplot:
- flags_vals = flags_vals.copy()
+ if mode not in ["subplots", "oneplot"]: # phaseplot
+ if len(d) != 1:
+ raise ValueError(
+ f"mode {mode!r} not supported. Use one of 'subplots' or 'oneplot'"
+ )
+ f0 = field[0]
flags_hist = flags_hist.copy()
- phase_index = data[phaseplot][xscope].values
- phase_index_d = phase_index[~na_mask]
- na_mask.index = phase_index
- d.index = phase_index_d
- flags_vals.index = phase_index
- flags_hist.index = phase_index
+ phase_index = data[mode][xscope].values
+ phase_index_d = phase_index[~na_mask[f0]]
+ na_mask[f0].index = phase_index
+ d[f0].index = phase_index_d
+ flags_vals[f0].index = phase_index
+ flags_hist[f0].index = phase_index
plot_kwargs = {**PLOT_KWARGS, **{"marker": "o", "linewidth": 0}}
- ax_kwargs = {**{"xlabel": phaseplot, "ylabel": d.name}, **ax_kwargs}
- else:
- plot_kwargs = PLOT_KWARGS
+ ax_kwargs = {"xlabel": mode, "ylabel": d[f0].name, **ax_kwargs}
# insert nans between values mutually spaced > max_gap
- if max_gap and not d.empty:
- d = _insertBlockingNaNs(d, max_gap)
+ if max_gap:
+ for f in field:
+ if not d[f].empty:
+ d[f] = _insertBlockingNaNs(d[f], max_gap)
# figure composition
if ax is None:
- fig = mpl.pyplot.figure(constrained_layout=True, **FIG_KWARGS)
- grid = fig.add_gridspec()
- ax = fig.add_subplot(grid[0])
+ nrows, ncols = ax_kwargs.pop("nrows", None), ax_kwargs.pop("ncols", None)
+ if nrows is None and ncols is not None:
+ nrows = int(np.ceil(len(d) / ncols))
+ elif nrows is not None and ncols is None:
+ ncols = int(np.ceil(len(d) / nrows))
+ elif ncols is None and nrows is None: # default:
+ if len(d) <= 2:
+ nrows, ncols = len(d), 1
+ else:
+ nrows, ncols = int(np.ceil(len(d) / 2)), 2
+ if nrows * ncols < len(d):
+ raise ValueError(
+ f"Too many variables (got {len(d)}), to plot into subplot matrix of passed shape {nrows}x{ncols}"
+ )
+
+ if mode == "oneplot":
+ fig, ax = plt.subplots(1, 1, sharex=sharex)
+ ax_arr = np.empty(len(field)).astype(object)
+ ax_arr[:] = ax
+ ax = ax_arr
+
+ else: # mode == 'subplots'
+ fig, ax = plt.subplots(nrows, ncols, sharex=sharex)
+ if nrows * ncols == 1:
+ ax = np.array(ax)
+ else: # custom ax passed
+ fig, ax = ax.figure, np.array(ax)
_plotVarWithFlags(
ax,
@@ -170,17 +244,144 @@ def makeFig(
na_mask,
plot_kwargs,
ax_kwargs,
- SCATTER_KWARGS,
- cyclestart,
+ scatter_kwargs,
+ mode,
+ )
+
+ return fig
+
+
+def _instantiateKwargContext(
+ plot_kwargs, scatter_kwargs, ax_kwargs, var_num, var_name, mode
+):
+ _scatter_mem = {}
+ _plot_kwargs = plot_kwargs.copy()
+ _scatter_kwargs = scatter_kwargs.copy()
+ _ax_kwargs = ax_kwargs.copy()
+ _scatter_mem = {}
+ # pop shape/color cycles:
+ cyclestart = _scatter_kwargs.pop("cycleskip", 0)
+ marker_shape_cycle = itertools.cycle(toSequence(_scatter_kwargs.pop("marker")))
+ marker_col_cycle = itertools.cycle(
+ toSequence(
+ _scatter_kwargs.pop(
+ "color", plt.rcParams["axes.prop_cycle"].by_key()["color"]
+ )
+ )
+ )
+ # skip through cycles on to the desired start
+ for k in range(0, cyclestart):
+ next(_scatter_kwargs["color"])
+ next(_scatter_kwargs["marker"])
+
+ # assign variable specific labels/titles
+ for axis_spec in ["xlabel", "ylabel", "title"]:
+ spec = _ax_kwargs.get(axis_spec, None)
+ if isinstance(spec, list):
+ _ax_kwargs[axis_spec] = spec[var_num]
+ elif isinstance(spec, dict):
+ _ax_kwargs[axis_spec] = spec.get(var_name, None)
+
+ title = _ax_kwargs.get("title", "" if mode != "subplots" else None)
+ _ax_kwargs["title"] = var_name if title is None else title
+
+ return (
+ _plot_kwargs,
+ _scatter_kwargs,
+ _ax_kwargs,
+ _scatter_mem,
+ marker_col_cycle,
+ marker_shape_cycle,
)
- plt.rcParams["font.size"] = default
- return ax.figure
+
+def _configMarkers(
+ flags_hist,
+ flags_vals,
+ flags_meta,
+ var_name,
+ var_dat,
+ history,
+ _scatter_kwargs,
+ _scatter_mem,
+ level,
+ marker_shape_cycle,
+ marker_col_cycle,
+ test_i,
+):
+ test_i_meta = flags_meta[var_name][test_i]
+ # catch empty but existing history case (flags_meta={})
+ if len(test_i_meta) == 0:
+ return None, _scatter_kwargs, _scatter_mem, marker_shape_cycle, marker_col_cycle
+ # retrieve label information:
+ label = test_i_meta["kwargs"].get("label", None) or test_i_meta["func"]
+ # are only flags from certain origins to be plotted? (than "history" is a list)
+ if isinstance(history, list):
+ # where to get label information from
+ if label not in history:
+ return (
+ None,
+ _scatter_kwargs,
+ _scatter_mem,
+ marker_shape_cycle,
+ marker_col_cycle,
+ )
+
+ # assign label to current marker kwarg dictionary
+ _scatter_kwargs.update({"label": label})
+ # retrieve flags to be plotted
+ flags_i = flags_hist[var_name][test_i].astype(float)
+
+ if history != "complete":
+ # only plot those flags, that do not get altered later on:
+ mask = flags_i.eq(flags_vals[var_name])
+ flags_i[~mask] = np.nan
+
+ # Skip plot, if the test did not have no effect on the all over flagging result. This avoids
+ # legend overflow
+ if ~(flags_i > level).any():
+ return (
+ None,
+ _scatter_kwargs,
+ _scatter_mem,
+ marker_shape_cycle,
+ marker_col_cycle,
+ )
+
+ # Also skip plot, if all flagged values are np.nans (to catch flag missing and masked results mainly)
+ temp_i = var_dat.index.join(flags_i.index, how="inner")
+ if var_dat[temp_i][flags_i[temp_i].notna()].isna().all() or (
+ "flagMissing" in test_i_meta["func"]
+ ):
+ return None, _scatter_kwargs, _scatter_mem, marker_shape_cycle, marker_col_cycle
+
+ # if encountering a label already associated with some marker shape/color, use that specific shape/color
+ if _scatter_kwargs["label"] in _scatter_mem.keys():
+ _scatter_kwargs.update(
+ {
+ "color": _scatter_mem[_scatter_kwargs["label"]][1],
+ "marker": _scatter_mem[_scatter_kwargs["label"]][0],
+ }
+ )
+
+ # if label is new, use next color/shape in the cycle
+ else:
+ _scatter_kwargs.update(
+ {
+ "color": next(marker_col_cycle),
+ "marker": next(marker_shape_cycle),
+ }
+ )
+ _scatter_mem[_scatter_kwargs["label"]] = (
+ _scatter_kwargs["marker"],
+ _scatter_kwargs["color"],
+ )
+ return flags_i, _scatter_kwargs, _scatter_mem, marker_shape_cycle, marker_col_cycle
def _plotVarWithFlags(
- ax,
- datser,
+ axes,
+ dat_dict,
flags_vals,
flags_hist,
flags_meta,
@@ -190,90 +391,128 @@ def _plotVarWithFlags(
plot_kwargs,
ax_kwargs,
scatter_kwargs,
- cyclestart,
+ mode,
):
- scatter_kwargs = scatter_kwargs.copy()
- ax.set_title(datser.name)
- ax.plot(datser, color="black", label="data", **plot_kwargs)
- ax.set(**ax_kwargs)
- shape_cycle = scatter_kwargs.get("marker", "o")
- shape_cycle = itertools.cycle(toSequence(shape_cycle))
- color_cycle = scatter_kwargs.get(
- "color", plt.rcParams["axes.prop_cycle"].by_key()["color"]
- )
- color_cycle = itertools.cycle(toSequence(color_cycle))
- for k in range(0, cyclestart):
- next(color_cycle)
- next(shape_cycle)
-
- if history:
- for i in flags_hist.columns:
- if isinstance(history, list):
- meta_field = "label" if "label" in flags_meta[i].keys() else "func"
- to_plot = (
- flags_meta[i][meta_field]
- if flags_meta[i][meta_field] in history
- else None
- )
- if not to_plot:
- continue
- else:
- hist_key = "valid"
- else:
- hist_key = history
- # catch empty but existing history case (flags_meta={})
- if len(flags_meta[i]) == 0:
- continue
- label = (
- flags_meta[i]["kwargs"].get("label", None)
- or flags_meta[i]["func"].split(".")[-1]
+ # ensure array of axes reference is flat
+ axes = axes.flatten()
+ # zip references for the variable loop:
+ loop_ref = zip(axes, dat_dict.keys(), dat_dict.values())
+ # include default settings if not modified
+ scatter_kwargs = {**SCATTER_KWARGS, **scatter_kwargs}
+ plot_kwargs = {**PLOT_KWARGS, **plot_kwargs}
+ if mode == "subplots":
+ plot_kwargs["color"] = "black"
+ # pop plot cycles options (will throw error when passed on) - plot always black data lines for one-dat-to-one-ax plots:
+ plot_col_cycle = plot_kwargs.pop("color")
+ plot_col_cycle = itertools.cycle(toSequence(plot_col_cycle))
+
+ for var_num, (ax, var_name, var_dat) in enumerate(loop_ref):
+ # every time, axis target is fresh, reinstantiate the kwarg-contexts :
+ if var_num == 0 or mode == "subplots":
+ (
+ _plot_kwargs,
+ _scatter_kwargs,
+ _ax_kwargs,
+ _scatter_mem,
+ marker_col_cycle,
+ marker_shape_cycle,
+ ) = _instantiateKwargContext(
+ plot_kwargs, scatter_kwargs, ax_kwargs, var_num, var_name, mode
)
- scatter_kwargs.update({"label": label})
- flags_i = flags_hist[i].astype(float)
- if hist_key == "complete":
- scatter_kwargs.update(
- {"color": next(color_cycle), "marker": next(shape_cycle)}
+ ax.set(**_ax_kwargs)
+
+ # get current color from plot color cycle
+ _plot_kwargs["color"] = next(plot_col_cycle)
+ if mode == "oneplot":
+ _plot_kwargs["label"] = var_name
+ # when plotting in subplots, plot black line and label it as 'data' (if not opted otherwise)
+ else:
+ _plot_kwargs["label"] = _plot_kwargs.get("label", None) or "data"
+
+ # plot the data
+ ax.plot(var_dat, **_plot_kwargs)
+
+ # start flags plotting
+ if history: # history information is processed
+ for test_i in flags_hist[var_name].columns:
+ (
+ flags_i,
+ _scatter_kwargs,
+ _scatter_mem,
+ marker_shape_cycle,
+ marker_col_cycle,
+ ) = _configMarkers(
+ flags_hist,
+ flags_vals,
+ flags_meta,
+ var_name,
+ var_dat,
+ history,
+ _scatter_kwargs,
+ _scatter_mem,
+ level,
+ marker_shape_cycle,
+ marker_col_cycle,
+ test_i,
)
- _plotFlags(ax, datser, flags_i, na_mask, level, scatter_kwargs)
- if hist_key == "valid":
- # only plot those flags, that do not get altered later on:
- mask = flags_i.eq(flags_vals)
- flags_i[~mask] = np.nan
- # Skip plot, if the test did not have no effect on the all over flagging result. This avoids
- # legend overflow
- if ~(flags_i > level).any():
- continue
- # Also skip plot, if all flagged values are np.nans (to catch flag missing and masked results mainly)
- temp_i = datser.index.join(flags_i.index, how="inner")
- if datser[temp_i][flags_i[temp_i].notna()].isna().all() or (
- "flagMissing" in flags_meta[i]["func"]
- ):
+ if flags_i is None:
continue
- scatter_kwargs.update(
- {"color": next(color_cycle), "marker": next(shape_cycle)}
- )
+ # plot the flags
_plotFlags(
ax,
- datser,
+ var_dat,
flags_i,
- na_mask,
+ na_mask[var_name],
level,
- scatter_kwargs,
+ _scatter_kwargs,
)
- ax.legend()
- else:
- scatter_kwargs.update({"color": next(color_cycle), "marker": next(shape_cycle)})
- _plotFlags(ax, datser, flags_vals, na_mask, level, scatter_kwargs)
+ else: # history is None
+ _scatter_kwargs.update(
+ {"color": next(marker_col_cycle), "marker": next(marker_shape_cycle)}
+ )
+ _plotFlags(
+ ax,
+ var_dat,
+ flags_vals[var_name],
+ na_mask[var_name],
+ level,
+ _scatter_kwargs,
+ )
+
+ _rmDupesFromLegend(ax, dat_dict)
+
+ return
+
+
+def _rmDupesFromLegend(ax, dat_dict):
+ # the legend generated might contain dublucate entries, we remove those, since dubed entries are assigned all
+ # the same marker color and shape:
+ legend_h, legend_l = ax.get_legend_handles_labels()
+ legend_v = []
+ legend_f = []
+
+ for l in enumerate(legend_l):
+ if l[1] in [k[1] for k in legend_f]:
+ continue
+ if l[1] in dat_dict.keys():
+ legend_v.append((legend_h[l[0]], l[1]))
+ else:
+ legend_f.append((legend_h[l[0]], l[1]))
+ leg_l = [l[1] for l in legend_v] + [l[1] for l in legend_f]
+ leg_h = [l[0] for l in legend_v] + [l[0] for l in legend_f]
+ ax.legend(leg_h, leg_l)
+ return
-def _plotFlags(ax, datser, flags, na_mask, level, scatter_kwargs):
+def _plotFlags(ax, datser, flags, na_mask, level, _scatter_kwargs):
+ # print(f"kwargs={_scatter_kwargs} \n variable={datser.name}")
is_flagged = flags.astype(float) > level
is_flagged = is_flagged[~na_mask]
is_flagged = datser[is_flagged[is_flagged].index]
- ax.scatter(is_flagged.index, is_flagged.values, **scatter_kwargs)
+ ax.scatter(is_flagged.index, is_flagged.values, **_scatter_kwargs)
def _insertBlockingNaNs(d, max_gap):
diff --git a/tests/funcs/test_tools.py b/tests/funcs/test_tools.py
index 7fc412e38..028bae5d8 100644
--- a/tests/funcs/test_tools.py
+++ b/tests/funcs/test_tools.py
@@ -33,7 +33,9 @@ def test_makeFig(tmp_path):
d_saqc = d_saqc.plot(field="data", path=outfile)
d_saqc = d_saqc.plot(field="data", path=outfile, history="valid", stats=True)
- d_saqc = d_saqc.plot(field="data", path=outfile, history="complete")
+ with pytest.deprecated_call():
+ d_saqc = d_saqc.plot(field="data", path=outfile, history="complete")
+
d_saqc = d_saqc.plot(
field="data", path=outfile, ax_kwargs={"ylabel": "data is data"}, stats=True
)
--
GitLab