From bcf1a6b36cc89a10d23cd626711f3a321891429d Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Tue, 24 Aug 2021 16:36:50 +0200
Subject: [PATCH] [FIX] bring translation to config workflow
---
saqc/core/core.py | 14 ++++++++++++--
saqc/core/reader.py | 15 ++++++++++-----
saqc/core/translator/positionaltranslator.py | 2 ++
3 files changed, 24 insertions(+), 7 deletions(-)
diff --git a/saqc/core/core.py b/saqc/core/core.py
index 7bd235e5c..4abe87065 100644
--- a/saqc/core/core.py
+++ b/saqc/core/core.py
@@ -219,7 +219,9 @@ class SaQC(FuncModules):
from saqc.core.reader import readConfig
out = stdcopy.deepcopy(self)
- out._planned.extend(readConfig(fname, self._data, self._nodata))
+ out._planned.extend(
+ readConfig(fname, self._translator, self._data, self._nodata)
+ )
if self._lazy:
return out
return out.evaluate()
@@ -297,9 +299,17 @@ class SaQC(FuncModules):
out = self if inplace else self.copy(deep=True)
control = APIController()
+ # NOTE:
+ # changes here are likely to be necessary in
+ # `saqc.core.reader._parseConfig` as well
partial = func.bind(
*fargs,
- **{"nodata": self._nodata, "flag": self._translator(flag), **fkwargs},
+ **{
+ "nodata": self._nodata,
+ "flag": self._translator(flag),
+ "to_mask": self._translator.TO_MASK,
+ **fkwargs,
+ },
)
# expand regular expressions
diff --git a/saqc/core/reader.py b/saqc/core/reader.py
index 344102e5b..21bfc1a5e 100644
--- a/saqc/core/reader.py
+++ b/saqc/core/reader.py
@@ -56,7 +56,7 @@ def _injectOptionalColumns(df):
return df
-def _parseConfig(df, data, nodata):
+def _parseConfig(df, translator, data, nodata):
funcs = []
for lineno, (_, target, expr) in enumerate(df.itertuples()):
if target == "None" or pd.isnull(target) or pd.isnull(expr):
@@ -72,7 +72,12 @@ def _parseConfig(df, data, nodata):
control = ConfigController(lineno=lineno + 2, expression=expr)
- f = func.bind(**{"nodata": nodata, **kwargs})
+ if "flag" in kwargs:
+ kwargs["flag"] = translator(kwargs["flag"])
+
+ partial = func.bind(
+ **{"nodata": nodata, "to_mask": translator.TO_MASK, **kwargs}
+ )
targets = toSequence(target)
@@ -81,12 +86,12 @@ def _parseConfig(df, data, nodata):
field=kwargs.get("field", target),
target=target,
)
- funcs.append((selector, control, f))
+ funcs.append((selector, control, partial))
return funcs
-def readConfig(fname, data, nodata):
+def readConfig(fname, translator, data, nodata):
df = pd.read_csv(
fname,
sep=r"\s*;\s*",
@@ -102,4 +107,4 @@ def readConfig(fname, data, nodata):
df[F.VARNAME] = df[F.VARNAME].replace(r"^\s*$", np.nan, regex=True)
df[F.TEST] = df[F.TEST].replace(r"^\s*$", np.nan, regex=True)
- return _parseConfig(df, data, nodata)
+ return _parseConfig(df, translator, data, nodata)
diff --git a/saqc/core/translator/positionaltranslator.py b/saqc/core/translator/positionaltranslator.py
index 3de906b2c..eb3c90c1c 100644
--- a/saqc/core/translator/positionaltranslator.py
+++ b/saqc/core/translator/positionaltranslator.py
@@ -26,6 +26,8 @@ class PositionalTranslator(Translator):
Implements the translation from and to the flagging scheme implemented by CHS
"""
+ TO_MASK = DOUBTFUL
+
_FORWARD: ForwardMap = {0: UNFLAGGED, 1: DOUBTFUL, 2: BAD}
_BACKWARD: BackwardMap = {
UNTOUCHED: 0,
--
GitLab