diff --git a/CHANGELOG.md b/CHANGELOG.md index 0eca0b8c7344f6f18f82eaa1949b2a5ff48d4766..cd0e053e33d0f6a25382b3f8999a17b706e6c48c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ This changelog starts with version 2.0.0. Basically all parts of the system, inc ### Fixed - `flagOffset` bug with zero-valued threshold - `flagCrossStatistics` bug with unaligned input variables +- `plot` fixed data loss when using *dfilter* kwarg ## [2.0.1](https://git.ufz.de/rdm-software/saqc/-/tags/v2.0.1) - 2021-12-20 [List of commits](https://git.ufz.de/rdm-software/saqc/-/compare/v2.0.0...v2.0.1) diff --git a/requirements.txt b/requirements.txt index c1d935c238b0656fb7f853926b2fbb2fe301df8b..e255ff85f1a986d9e60f26204d3330f0e4e40864 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,8 +6,8 @@ Click==8.0.3 dtw==1.4.0 hypothesis==6.34.1 matplotlib==3.5.1 -numba==0.54.1 -numpy==1.20.3 +numba>=0.55.0 +numpy>=1.21.5 outlier-utils==0.0.3 pyarrow==6.0.1 pandas==1.3.5 diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py index c06a2a473f78773f470da552e447ea8a596d9a18..9fe7bdf30d1eed8207e9eddc818e9fa089ce3746 100644 --- a/saqc/funcs/tools.py +++ b/saqc/funcs/tools.py @@ -334,10 +334,10 @@ def plot( """ interactive = path is None - level = kwargs.get("flag", BAD) + level = kwargs.get("flag", UNFLAGGED) if dfilter < np.inf: - data = data.copy() + data_temp = data[field].copy() data.loc[flags[field] >= dfilter, field] = np.nan if store_kwargs is None: @@ -374,4 +374,7 @@ def plot( else: fig.savefig(path, **store_kwargs) + if dfilter < np.inf: + data[field] = data_temp + return data, flags diff --git a/saqc/lib/plotting.py b/saqc/lib/plotting.py index 78fa66af57bd7cd974dd6348096f84180d61d8a9..e7ba75fab11ac9e58e8b28c0e02448af900837d3 100644 --- a/saqc/lib/plotting.py +++ b/saqc/lib/plotting.py @@ -225,7 +225,7 @@ def _plotVarWithFlags( flags_i[~mask] = np.nan # Skip plot, if the test did not have no effect on the all over flagging result. This avoids # legend overflow - if ~(flags_i >= level).any(): + if ~(flags_i > level).any(): continue # Also skip plot, if all flagged values are np.nans (to catch flag missing and masked results mainly) @@ -254,7 +254,7 @@ def _plotVarWithFlags( def _plotFlags(ax, datser, flags, na_mask, level, scatter_kwargs): - is_flagged = flags.astype(float) >= level + is_flagged = flags.astype(float) > level is_flagged = is_flagged[~na_mask] is_flagged = datser[is_flagged[is_flagged].index] ax.scatter(is_flagged.index, is_flagged.values, **scatter_kwargs) diff --git a/setup.py b/setup.py index 183ae6c12d83315c399935270f2e68a0e1fbc665..20d3b0be6108921736e7b2d4cf63be9929ebd91e 100644 --- a/setup.py +++ b/setup.py @@ -20,20 +20,20 @@ with open("README.md", "r") as fh: setup( name="saqc", version=version, - author="Bert Palm, David Schaefer, Peter Luenenschloss, Lennard Schmidt", + author="Bert Palm, David Schaefer, Peter Luenenschloss, Lennart Schmidt", author_email="david.schaefer@ufz.de", description="Data quality checking and processing tool/framework", long_description=long_description, long_description_content_type="text/markdown", url="https://git.ufz.de/rdm-software/saqc", packages=find_packages(exclude=("tests",)), - python_requires=">=3.7, <3.10", + python_requires=">=3.7", install_requires=[ "Click==8.0.*", "dtw==1.4.*", "matplotlib>=3.4,<3.6", - "numba==0.54.*", - "numpy==1.20.*", + "numba>=0.54", + "numpy==1.21.5", "outlier-utils==0.0.3", "pyarrow==6.0.*", "pandas==1.3.*",