From 1eb54e15005320b03bff76b247bd54e6a5c9c5a5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Peter=20L=C3=BCnenschlo=C3=9F?= <peter.luenenschloss@ufz.de>
Date: Wed, 19 Mar 2025 10:16:17 +0100
Subject: [PATCH] Flag constants fix
---
CHANGELOG.md | 1 +
saqc/funcs/constants.py | 5 ++---
tests/core/test_flags.py | 4 ++--
tests/funcs/test_constants_detection.py | 20 ++++++++++++++++++++
4 files changed, 25 insertions(+), 5 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2625096d8..0e90c4c52 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ SPDX-License-Identifier: GPL-3.0-or-later
### Changed
### Removed
### Fixed
+- `flagConstants`: fixed bug where last `min_periods` will never get flagged
### Deprecated
## [2.6.0](https://git.ufz.de/rdm-software/saqc/-/tags/v2.6.0) - 2024-04-15
diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py
index 3e4afba3a..2d57fba5f 100644
--- a/saqc/funcs/constants.py
+++ b/saqc/funcs/constants.py
@@ -77,9 +77,8 @@ class ConstantsMixin:
starting_points_mask = removeRollingRamps(starting_points_mask, window=window)
# mimic forward rolling by roll over inverse [::-1]
- rolling = starting_points_mask[::-1].rolling(
- window=window, min_periods=min_periods
- )
+
+ rolling = starting_points_mask[::-1].rolling(window=window, min_periods=0)
# mimic any()
mask = (rolling.sum()[::-1] > 0) & d.notna()
diff --git a/tests/core/test_flags.py b/tests/core/test_flags.py
index 9e6765188..eb11470fb 100644
--- a/tests/core/test_flags.py
+++ b/tests/core/test_flags.py
@@ -206,7 +206,7 @@ def test_set_flags(data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]]
@pytest.mark.parametrize("data", testdata)
def test_set_flags_with_mask(
- data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]]
+ data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]],
):
flags = Flags(data)
@@ -253,7 +253,7 @@ def test_set_flags_with_mask(
@pytest.mark.parametrize("data", testdata)
def test_set_flags_with_index(
- data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]]
+ data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]],
):
flags = Flags(data)
diff --git a/tests/funcs/test_constants_detection.py b/tests/funcs/test_constants_detection.py
index a6ec79792..74cef249d 100644
--- a/tests/funcs/test_constants_detection.py
+++ b/tests/funcs/test_constants_detection.py
@@ -7,6 +7,7 @@
# -*- coding: utf-8 -*-
import numpy as np
+import pandas as pd
import pytest
from saqc import BAD, UNFLAGGED, SaQC
@@ -24,6 +25,15 @@ def data():
return constants_data
+@pytest.fixture
+def data_const_tail():
+ constants_data = pd.DataFrame(
+ {"a": [1, 2, 3, 4, 5, 9, 9, 9, 9, 9]},
+ index=pd.date_range("2000", freq="1h", periods=10),
+ )
+ return constants_data
+
+
def test_constants_flagBasic(data):
field, *_ = data.columns
flags = initFlagsLike(data)
@@ -35,6 +45,16 @@ def test_constants_flagBasic(data):
assert np.all(flagscol[25 + 1 :] == UNFLAGGED)
+@pytest.mark.parametrize("window", [3, "3h", 5, "5h"])
+def test_constants_tail(data_const_tail, window):
+ field, *_ = data_const_tail.columns
+ qc = SaQC(data_const_tail)
+ qc = qc.flagConstants(field, thresh=1, window=window, flag=BAD)
+ flagscol = qc._flags[field]
+ assert np.all(flagscol[-5:] == BAD)
+ assert np.all(flagscol[:-5] == UNFLAGGED)
+
+
def test_constants_flagVarianceBased(data):
field, *_ = data.columns
flags = initFlagsLike(data)
--
GitLab