Merge branch 'flagConstantsFix' into 'develop'

Flag constants fix

See merge request !867

CHANGELOG.md

@@ -15,6 +15,7 @@ SPDX-License-Identifier: GPL-3.0-or-later
 ### Changed
 ### Removed
 ### Fixed
+- `flagConstants`: fixed bug where last `min_periods` will never get flagged
 ### Deprecated
 
 ## [2.6.0](https://git.ufz.de/rdm-software/saqc/-/tags/v2.6.0) - 2024-04-15

saqc/funcs/constants.py

@@ -77,9 +77,8 @@ class ConstantsMixin:
         starting_points_mask = removeRollingRamps(starting_points_mask, window=window)
 
         # mimic forward rolling by roll over inverse [::-1]
-        rolling = starting_points_mask[::-1].rolling(
-            window=window, min_periods=min_periods
-        )
+
+        rolling = starting_points_mask[::-1].rolling(window=window, min_periods=0)
         # mimic any()
         mask = (rolling.sum()[::-1] > 0) & d.notna()
 

tests/core/test_flags.py

@@ -206,7 +206,7 @@ def test_set_flags(data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]]
 
 @pytest.mark.parametrize("data", testdata)
 def test_set_flags_with_mask(
-    data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]]
+    data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]],
 ):
     flags = Flags(data)
 
@@ -253,7 +253,7 @@ def test_set_flags_with_mask(
 
 @pytest.mark.parametrize("data", testdata)
 def test_set_flags_with_index(
-    data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]]
+    data: Union[pd.DataFrame, DictOfSeries, Dict[str, pd.Series]],
 ):
     flags = Flags(data)
 

tests/funcs/test_constants_detection.py

@@ -7,6 +7,7 @@
 # -*- coding: utf-8 -*-
 
 import numpy as np
+import pandas as pd
 import pytest
 
 from saqc import BAD, UNFLAGGED, SaQC
@@ -24,6 +25,15 @@ def data():
     return constants_data
 
 
+@pytest.fixture
+def data_const_tail():
+    constants_data = pd.DataFrame(
+        {"a": [1, 2, 3, 4, 5, 9, 9, 9, 9, 9]},
+        index=pd.date_range("2000", freq="1h", periods=10),
+    )
+    return constants_data
+
+
 def test_constants_flagBasic(data):
     field, *_ = data.columns
     flags = initFlagsLike(data)
@@ -35,6 +45,16 @@ def test_constants_flagBasic(data):
     assert np.all(flagscol[25 + 1 :] == UNFLAGGED)
 
 
+@pytest.mark.parametrize("window", [3, "3h", 5, "5h"])
+def test_constants_tail(data_const_tail, window):
+    field, *_ = data_const_tail.columns
+    qc = SaQC(data_const_tail)
+    qc = qc.flagConstants(field, thresh=1, window=window, flag=BAD)
+    flagscol = qc._flags[field]
+    assert np.all(flagscol[-5:] == BAD)
+    assert np.all(flagscol[:-5] == UNFLAGGED)
+
+
 def test_constants_flagVarianceBased(data):
     field, *_ = data.columns
     flags = initFlagsLike(data)