Merge branch 'master' into assignto

dsl/evaluator.py

@@ -103,21 +103,16 @@ def evalExpression(expr: str, flagger: BaseFlagger,
                 # name is not referring to an DataFrame field
                 return field
 
-            if namespace.get("target") == "flags":
-                try:
-                    # check if variable is existent
-                    flagcol = namespace["flags"][field]
-                except KeyError:
-                    _raiseNameError(field, expr)
-                out = flagcol
-            else:
-                try:
-                    # check if variable is existent
-                    flagcol = namespace["flags"][field]
+            try:
+                flagcol = namespace["flags"][field]
+                if namespace.get("target") == "flags":
+                    out = flagcol
+                else:
                     datacol = namespace["data"][field]
-                except KeyError:
-                    _raiseNameError(field, expr)
-                out = np.ma.masked_array(datacol, mask=flagger.isFlagged(flagcol)).filled(True)
+                    out = np.ma.masked_array(datacol,
+                                             mask=flagger.isFlagged(flagcol))
+            except KeyError:
+                _raiseNameError(field, expr)
 
             return out
 

flagger/dmpflagger.py

 #! /usr/bin/env python
 # -*- coding: utf-8 -*-
-
-import numpy as np
 import pandas as pd
 
 from .baseflagger import BaseFlagger
 
+
 class FlagFields:
     FLAG = "quality_flag"
     CAUSE = "quality_cause"
     COMMENT = "quality_comment"
 
 
+class ColumnLevels:
+    VARIABLES = "variables"
+    FLAGS = "flags"
+
+
 class Flags:
     OK = "OK"
     DOUBTFUL = "DOUBTFUL"
@@ -29,19 +33,27 @@ class DmpFlagger(BaseFlagger):
 
     def emptyFlags(self, data, value="NIL", **kwargs):
         columns = data.columns if isinstance(data, pd.DataFrame) else [data.name]
-        columns = pd.MultiIndex.from_product([columns, self.flag_fields])
+        columns = pd.MultiIndex.from_product(
+            [columns, self.flag_fields],
+            names=[ColumnLevels.VARIABLES, ColumnLevels.FLAGS])
         return pd.DataFrame(data=value, columns=columns, index=data.index)
 
     def setFlag(self, flags, flag=Flags.BAD, cause="NIL", comment="NIL", **kwargs):
         self._isFlag(flag)
+        flags = self._reduceColumns(flags)
         for field, f in zip(self.flag_fields, [flag, cause, comment]):
             flags.loc[:, field] = f
         return flags
 
     def isFlagged(self, flags, flag=None):
+        flags = self._reduceColumns(flags)
         flagcol = flags.loc[:, FlagFields.FLAG].squeeze()
         return super().isFlagged(flagcol, flag)
 
+    def _reduceColumns(self, flags):
+        if isinstance(flags.columns, pd.MultiIndex):
+            flags.columns = flags.columns.get_level_values(ColumnLevels.FLAGS)
+        return flags
+
     def _isFlag(self, flag):
         assert Flags.isValid(flag)
-

funcs/functions.py

@@ -21,7 +21,6 @@ def flagDispatch(func_name, *args, **kwargs):
     raise NameError(f"function name {func_name} is not definied")
 
 
-
 def flagGeneric(data, flags, field, flagger, nodata=np.nan, **flag_params):
 
     expression = flag_params[Params.FUNC]

test/test_core.py

@@ -71,9 +71,13 @@ def test_flagNextMulticolumn():
     flagger = DmpFlagger()
     data = initData().iloc[:, 0]
     flags = flagger.emptyFlags(data)
+    var, *_ = flags.columns.get_level_values(0)
 
-    flags.iloc[0::3] = flagger.setFlag(flags.iloc[0::3], "DOUBTFUL")
-    flags.iloc[2::3] = flagger.setFlag(flags.iloc[2::3], "BAD")
+    flags.loc[data.index[0::3], var] = flagger.setFlag(
+        flags.loc[data.index[0::3], var], "DOUBTFUL").values
+
+    flags.loc[data.index[2::3], var] = flagger.setFlag(
+        flags.loc[data.index[2::3], var], "BAD").values
 
     fflags = flagNext(flagger, flags.copy(), 4)
     assert ((fflags.values[pd.isnull(flags)] == 1).all(axis=None))

test/test_evaluator.py

@@ -54,7 +54,10 @@ def test_flagPropagation():
     var2_flags = flagger.isFlagged(flags[var2])
     var2_data = data[var2].mask(var2_flags)
 
-    result = evalExpression("var2 < mean(var2)", flagger, data, flags, data.columns[0])
+    result = evalExpression("var2 < mean(var2)",
+                            flagger,
+                            data, flags,
+                            data.columns[0])
 
     expected = (var2_flags | (var2_data < var2_data.mean()))
     assert (result.filled(True) == expected).all()