removed the need for explicit assignments

saqc/core/config.py

@@ -6,7 +6,6 @@ class Fields:
     VARNAME = "varname"
     START = "start_date"
     END = "end_date"
-    ASSIGN = "assign"
     TESTS = "test*"
     PLOT = "plot"
     LINENUMBER = "line"

saqc/core/core.py

@@ -20,11 +20,11 @@ def _collectVariables(meta, data):
     variables = []
     for idx, configrow in meta.iterrows():
         varname = configrow[Fields.VARNAME]
-        assign = configrow[Fields.ASSIGN]
+        # assign = configrow[Fields.ASSIGN]
         if varname in variables:
             continue
-        if (varname in data) or (varname not in variables and assign is True):
-            variables.append(varname)
+        # if (varname in data):  # or (varname not in variables and assign is True):
+        variables.append(varname)
     return variables
 
 
@@ -84,6 +84,7 @@ def runner(config_file, flagger, data, flags=None, nodata=np.nan, error_policy="
     # user-test needs fully prepared flags
     checkConfig(config, data, flagger, nodata)
 
+
     # NOTE:
     # the outer loop runs over the flag tests, the inner one over the
     # variables. Switching the loop order would complicate the
@@ -116,7 +117,7 @@ def runner(config_file, flagger, data, flags=None, nodata=np.nan, error_policy="
 
             try:
                 # actually run the tests
-                dchunk, flagger_chunk_result = evalExpression(
+                dchunk_result, flagger_chunk_result = evalExpression(
                     flag_test,
                     data=dchunk,
                     field=varname,
@@ -131,7 +132,7 @@ def runner(config_file, flagger, data, flags=None, nodata=np.nan, error_policy="
             flagger = flagger.setFlagger(flagger_chunk_result)
 
             plotHook(
-                dchunk,
+                dchunk_result,
                 flagger_chunk,
                 flagger_chunk_result,
                 varname,

saqc/core/reader.py

@@ -34,10 +34,12 @@ def checkConfig(config_df, data, flagger, nodata):
             )
 
         var_name = config_row[F.VARNAME]
-        if var_name not in data.columns and not config_row[F.ASSIGN]:
-            _raise(config_row, NameError, f"unknown variable '{var_name}'")
+        if not var_name:
+            _raise(config_row, SyntaxError, f"field '{F.VARNAME}' may not be empty")
 
         for col, expr in test_fields.iteritems():
+            if not expr:
+                _raise(config_row, SyntaxError, f"field '{col}' may not be empty")
             try:
                 compileExpression(expr, data, flagger, nodata)
             except (TypeError, NameError, SyntaxError) as exc:
@@ -66,7 +68,7 @@ def prepareConfig(config_df, data):
         raise SyntaxWarning("config file is empty or all lines are #commented")
 
     # fill missing header fields
-    for field in [F.VARNAME, F.START, F.END, F.ASSIGN, F.PLOT]:
+    for field in [F.VARNAME, F.START, F.END, F.PLOT]:
         if field not in config_df:
             config_df = config_df.assign(**{field: np.nan})
 
@@ -76,7 +78,6 @@ def prepareConfig(config_df, data):
             F.VARNAME: np.nan,
             F.START: data.index.min(),
             F.END: data.index.max(),
-            F.ASSIGN: False,
             F.PLOT: False,
         }
     )

test/core/test_core.py

@@ -113,23 +113,6 @@ def test_missingConfig(data, flagger, flags):
     assert var1 in pdata and var2 not in pflagger.getFlags()
 
 
-@pytest.mark.parametrize("flagger", TESTFLAGGER)
-def test_missingVariable(data, flagger):
-    """
-    Test if variables available in the config but not dataset
-    are handled correctly, i.e. are ignored
-    """
-    var, *_ = data.columns
-
-    metadict = [
-        {F.VARNAME: var, F.TESTS: "flagAll()"},
-        {F.VARNAME: "empty", F.TESTS: "flagAll()"},
-    ]
-    metafobj, meta = initMetaDict(metadict, data)
-    with pytest.raises(NameError):
-        runner(metafobj, flagger, data)
-
-
 @pytest.mark.parametrize("flagger", TESTFLAGGER)
 def test_errorHandling(data, flagger):
 
@@ -160,8 +143,8 @@ def test_duplicatedVariable(flagger):
     var1, *_ = data.columns
 
     metadict = [
-        {F.VARNAME: var1, F.ASSIGN: False, F.TESTS: "flagAll()"},
-        {F.VARNAME: var1, F.ASSIGN: True, F.TESTS: "flagAll()"},
+        {F.VARNAME: var1, F.TESTS: "flagAll()"},
+        {F.VARNAME: var1, F.TESTS: "flagAll()"},
     ]
     metafobj, meta = initMetaDict(metadict, data)
 
@@ -178,16 +161,15 @@ def test_duplicatedVariable(flagger):
 @pytest.mark.parametrize("flagger", TESTFLAGGER)
 def test_assignVariable(flagger):
     """
-    Test the assign keyword, a variable present in the configuration, but not
-    dataset will be added to output flags
+    test implicit assignments
     """
     data = initData(1)
     var1, *_ = data.columns
     var2 = "empty"
 
     metadict = [
-        {F.VARNAME: var1, F.ASSIGN: False, F.TESTS: "flagAll()"},
-        {F.VARNAME: var2, F.ASSIGN: True, F.TESTS: "flagAll()"},
+        {F.VARNAME: var1, F.TESTS: "flagAll()"},
+        {F.VARNAME: var2, F.TESTS: "flagAll()"},
     ]
     metafobj, meta = initMetaDict(metadict, data)
 

test/core/test_reader.py

@@ -20,14 +20,13 @@ def test_configPreparation(data):
     tests = [
         {F.VARNAME: var1, F.START: date, F.TESTS: "flagAll()", F.PLOT: True},
         {F.VARNAME: var2, F.TESTS: "flagAll()", F.PLOT: False},
-        {F.VARNAME: var3, F.END: date, F.TESTS: "flagAll()", F.ASSIGN: True},
+        {F.VARNAME: var3, F.END: date, F.TESTS: "flagAll()"},
         {F.VARNAME: var3, F.TESTS: "flagAll()",},
     ]
 
     defaults = {
         F.START: data.index.min(),
         F.END: data.index.max(),
-        F.ASSIGN: False,
         F.PLOT: False,
         F.LINENUMBER: 2,
     }
@@ -65,10 +64,9 @@ def test_configChecks(data, flagger, nodata, caplog):
 
     tests = [
         ({F.VARNAME: var1, F.TESTS: "range(mn=0)"}, TypeError),
-        ({F.VARNAME: "temp2", F.TESTS: "range(min=3)"}, NameError),
         ({F.VARNAME: var3, F.TESTS: "flagNothing()"}, NameError),
-        ({F.VARNAME: "", F.TESTS: "range(min=3)"}, NameError),
-        ({F.VARNAME: "", F.TESTS: ""}, NameError),
+        ({F.VARNAME: "", F.TESTS: "range(min=3)"}, SyntaxError),
+        ({F.VARNAME: var1, F.TESTS: ""}, SyntaxError),
         ({F.VARNAME: ""}, SyntaxError),
         ({F.TESTS: "range(min=3)"}, SyntaxError),
     ]