Merge branch 'generic-fix' into 'develop'

[FIX] inconsistencies with generic flag handling and meta writing

See merge request !461

CHANGELOG.md

@@ -28,6 +28,7 @@ This changelog starts with version 2.0.0. Basically all parts of the system, inc
 - renamed `maskTime` to `selectTime`
 - `.gitlab-ci.py`: always run all pytest-tests in CI/CD pipelines
 - `.gitlab-ci.py`: use reports to enable `Tests` in CI/CD pipeline results
+- `procGeneric`: changed default `flag` value to `np.nan`
 
 ### Removed
 ### Fixed
@@ -40,7 +41,10 @@ This changelog starts with version 2.0.0. Basically all parts of the system, inc
 - `interpolateInvalid`: Fix: replacement of flags for interpolated values now works
 - `resample`: resampling func now actually gets passed on to `history.appy()`
 - `tools.seasonalMask`: fixed bug that swaps the entire mask upon `include_bounds=True`
-
+- `flagGeneric`:
+  - fixed inconsistent history meta writing
+  - fixed handling of existing flags
+- `proGeneeric`: fixed inconsistent history meta writing
 
 ## [2.0.1](https://git.ufz.de/rdm-software/saqc/-/tags/v2.0.1) - 2021-12-20
 [List of commits](https://git.ufz.de/rdm-software/saqc/-/compare/v2.0.0...v2.0.1)

saqc/funcs/generic.py

@@ -72,7 +72,7 @@ def processGeneric(
     flags: Flags,
     func: GenericFunction,
     target: str | Sequence[str] = None,
-    flag: float = UNFLAGGED,
+    flag: float = np.nan,
     dfilter: float = FILTER_ALL,
     **kwargs,
 ) -> Tuple[DictOfSeries, Flags]:
@@ -106,10 +106,9 @@ def processGeneric(
         The variable(s) to write the result of ``func`` to. If not given, the variable(s)
         specified in ``field`` will be overwritten. If a ``target`` is not given, it will be
         created.
-    flag: float, default ``UNFLAGGED``
-        The quality flag to set. The default ``UNFLAGGED`` states the general idea, that
-        ``processGeneric`` generates 'new' data without direct relation to the potentially
-        already present flags.
+    flag: float, default ``np.nan``
+        The quality flag to set. The default ``np.nan`` states the general idea, that
+        ``processGeneric`` generates 'new' data without any flags.
     dfilter: float, default ``FILTER_ALL``
         Threshold flag. Flag values greater than ``dfilter`` indicate that the associated
         data value is inappropiate for further usage.
@@ -148,14 +147,12 @@ def processGeneric(
 
     meta = {
         "func": "procGeneric",
-        "args": (),
+        "args": (field, target),
         "kwargs": {
-            **kwargs,
-            "field": field,
             "func": func.__name__,
-            "target": target,
             "flag": flag,
             "dfilter": dfilter,
+            **kwargs,
         },
     }
 
@@ -222,8 +219,8 @@ def flagGeneric(
         The variable(s) to write the result of ``func`` to. If not given, the variable(s)
         specified in ``field`` will be overwritten. If a ``target`` is not given, it will be
         created.
-    flag: float, default ``UNFLAGGED``
-        The quality flag to set. The default ``UNFLAGGED`` states the general idea, that
+    flag: float, default ``BAD``
+        The quality flag to set. The default ``BAD`` states the general idea, that
         ``processGeneric`` generates 'new' data without direct relation to the potentially
         already present flags.
     dfilter: float, default ``FILTER_ALL``
@@ -284,14 +281,12 @@ def flagGeneric(
 
     meta = {
         "func": "flagGeneric",
-        "args": (),
+        "args": (field, target),
         "kwargs": {
-            **kwargs,
-            "field": field,
             "func": func.__name__,
-            "target": target,
             "flag": flag,
             "dfilter": dfilter,
+            **kwargs,
         },
     }
 
@@ -308,7 +303,6 @@ def flagGeneric(
         if col not in data:
             data[col] = pd.Series(np.nan, index=maskcol.index)
 
-        maskcol = maskcol & ~_isflagged(flags[col], dfilter)
         flagcol = maskcol.replace({False: np.nan, True: flag}).astype(float)
 
         # we need equal indices to work on

tests/funcs/test_generic_api_functions.py

@@ -48,11 +48,9 @@ def test_writeTargetFlagGeneric(data):
     for targets, func in params:
         expected_meta = {
             "func": "flagGeneric",
-            "args": (),
+            "args": (data.columns.tolist(), targets),
             "kwargs": {
-                "field": data.columns.tolist(),
                 "func": func.__name__,
-                "target": targets,
                 "flag": BAD,
                 "dfilter": FILTER_ALL,
             },
@@ -80,10 +78,8 @@ def test_overwriteFieldFlagGeneric(data):
     for fields, func in params:
         expected_meta = {
             "func": "flagGeneric",
-            "args": (),
+            "args": (fields, fields),
             "kwargs": {
-                "field": fields,
-                "target": fields,
                 "func": func.__name__,
                 "flag": flag,
                 "dfilter": FILTER_ALL,
@@ -104,11 +100,9 @@ def test_overwriteFieldFlagGeneric(data):
 
         res = saqc.flagGeneric(field=fields, func=func, flag=flag)
         for field in fields:
-            assert (data[field] == res.data[field]).all(axis=None)
-            histcol0 = res._flags.history[field].hist[0]
             histcol1 = res._flags.history[field].hist[1]
-            assert (histcol1[histcol0 == 127.0].isna()).all()
-            assert (histcol1[histcol0 != 127.0] == flag).all()
+            assert (histcol1 == flag).all()
+            assert (data[field] == res.data[field]).all(axis=None)
             assert res._flags.history[field].meta[0] == {}
             assert res._flags.history[field].meta[1] == expected_meta
 
@@ -128,10 +122,8 @@ def test_writeTargetProcGeneric(data):
 
         expected_meta = {
             "func": "procGeneric",
-            "args": (),
+            "args": (fields, targets),
             "kwargs": {
-                "field": data.columns.tolist(),
-                "target": targets,
                 "func": func.__name__,
                 "flag": BAD,
                 "dfilter": dfilter,
@@ -173,10 +165,8 @@ def test_overwriteFieldProcGeneric(data):
 
         expected_meta = {
             "func": "procGeneric",
-            "args": (),
+            "args": (fields, fields),
             "kwargs": {
-                "field": fields,
-                "target": fields,
                 "func": func.__name__,
                 "flag": flag,
                 "dfilter": dfilter,