diff --git a/CHANGELOG.md b/CHANGELOG.md index 2625096d8ef377075e657bafade37538d5711eaf..0e90c4c52e5a8149d86fa2afa532e84219b0b686 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ SPDX-License-Identifier: GPL-3.0-or-later ### Changed ### Removed ### Fixed +- `flagConstants`: fixed bug where last `min_periods` will never get flagged ### Deprecated ## [2.6.0](https://git.ufz.de/rdm-software/saqc/-/tags/v2.6.0) - 2024-04-15 diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py index 3e4afba3a4b125f2dcf68c94a4cd0cdb3bff098b..2d57fba5f07749b0dbafe4de0f0b8041219720d9 100644 --- a/saqc/funcs/constants.py +++ b/saqc/funcs/constants.py @@ -77,9 +77,8 @@ class ConstantsMixin: starting_points_mask = removeRollingRamps(starting_points_mask, window=window) # mimic forward rolling by roll over inverse [::-1] - rolling = starting_points_mask[::-1].rolling( - window=window, min_periods=min_periods - ) + + rolling = starting_points_mask[::-1].rolling(window=window, min_periods=0) # mimic any() mask = (rolling.sum()[::-1] > 0) & d.notna() diff --git a/tests/core/test_flags.py b/tests/core/test_flags.py index 9e67651880f5981349d70444049b0164085a1e77..eb11470fbc57f57f8052eac4d11ecca79bd39155 100644 --- a/tests/core/test_flags.py +++ b/tests/core/test_flags.py @@ -206,7 +206,7 @@ def test_set_flags(data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]] @pytest.mark.parametrize("data", testdata) def test_set_flags_with_mask( - data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]] + data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]], ): flags = Flags(data) @@ -253,7 +253,7 @@ def test_set_flags_with_mask( @pytest.mark.parametrize("data", testdata) def test_set_flags_with_index( - data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]] + data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]], ): flags = Flags(data) diff --git a/tests/funcs/test_constants_detection.py b/tests/funcs/test_constants_detection.py index a6ec797929f933e413e4c7aa9c6478259914147a..74cef249d90b71cd9968240f9c2d743040cdbf82 100644 --- a/tests/funcs/test_constants_detection.py +++ b/tests/funcs/test_constants_detection.py @@ -7,6 +7,7 @@ # -*- coding: utf-8 -*- import numpy as np +import pandas as pd import pytest from saqc import BAD, UNFLAGGED, SaQC @@ -24,6 +25,15 @@ def data(): return constants_data +@pytest.fixture +def data_const_tail(): + constants_data = pd.DataFrame( + {"a": [1, 2, 3, 4, 5, 9, 9, 9, 9, 9]}, + index=pd.date_range("2000", freq="1h", periods=10), + ) + return constants_data + + def test_constants_flagBasic(data): field, *_ = data.columns flags = initFlagsLike(data) @@ -35,6 +45,16 @@ def test_constants_flagBasic(data): assert np.all(flagscol[25 + 1 :] == UNFLAGGED) +@pytest.mark.parametrize("window", [3, "3h", 5, "5h"]) +def test_constants_tail(data_const_tail, window): + field, *_ = data_const_tail.columns + qc = SaQC(data_const_tail) + qc = qc.flagConstants(field, thresh=1, window=window, flag=BAD) + flagscol = qc._flags[field] + assert np.all(flagscol[-5:] == BAD) + assert np.all(flagscol[:-5] == UNFLAGGED) + + def test_constants_flagVarianceBased(data): field, *_ = data.columns flags = initFlagsLike(data)