From 7de055138766971c33b736b322cbf8a883a61734 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Sch=C3=A4fer?= <david.schaefer@ufz.de>
Date: Thu, 30 Mar 2023 21:28:21 +0200
Subject: [PATCH] Allow function arguments in config syntax
---
CHANGELOG.md | 3 ++-
saqc/parsing/visitor.py | 43 +++++++++++++++++++++++++++------------
tests/core/test_reader.py | 22 +++++++++++++++++++-
3 files changed, 53 insertions(+), 15 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c36faca39..871d5797b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,7 +10,7 @@ SPDX-License-Identifier: GPL-3.0-or-later
[List of commits](https://git.ufz.de/rdm-software/saqc/-/compare/v2.3.0...develop)
### Added
- Methods `logicalAnd` and `logicalOr`
-- `Flags` supports slicing and column selection with `list` or a `pd.Index`.
+- `Flags` supports slicing and column selection with `list` or a `pd.Index`
### Changed
- Deprecate `interpolate`, `linear` and `shift` in favor of `align`
- Rename `interplateInvalid` to `interpolate`
@@ -18,6 +18,7 @@ SPDX-License-Identifier: GPL-3.0-or-later
### Removed
- Parameter `limit` from `align`
### Fixed
+- `func` arguments in text configurations were not parsed correctly
- fail on duplicated arguments to test methods
## [2.3.0](https://git.ufz.de/rdm-software/saqc/-/tags/v2.3.0) - 2023-01-17
diff --git a/saqc/parsing/visitor.py b/saqc/parsing/visitor.py
index 91f086e2f..7224e85a0 100644
--- a/saqc/parsing/visitor.py
+++ b/saqc/parsing/visitor.py
@@ -7,6 +7,9 @@
# -*- coding: utf-8 -*-
import ast
+import importlib
+
+import numpy as np
from saqc.core.register import FUNC_MAP
from saqc.parsing.environ import ENVIRONMENT
@@ -127,18 +130,33 @@ class ConfigFunctionParser(ast.NodeVisitor):
key, value = node.arg, node.value
check_tree = True
+ imports = {}
if key == "func":
- visitor = ConfigExpressionParser(value)
- args = ast.arguments(
- posonlyargs=[],
- kwonlyargs=[],
- kw_defaults=[],
- defaults=[],
- args=[ast.arg(arg=a, annotation=None) for a in visitor.args],
- kwarg=None,
- vararg=None,
- )
- value = ast.Lambda(args=args, body=value)
+ if (isinstance(value, ast.Name) and value.id in ENVIRONMENT) or (
+ isinstance(value, ast.Constant) and value.value in ENVIRONMENT
+ ):
+ func = ENVIRONMENT[
+ value.id if isinstance(value, ast.Name) else value.value
+ ]
+ # handle the missing attribute for numpy.ufunc
+ module = getattr(func, "__module__", "numpy")
+ if module.startswith("saqc"):
+ # if it's an saqc function, we need to import the top level package first
+ imports["saqc"] = importlib.import_module("saqc")
+ imports[module] = importlib.import_module(module)
+ value = ast.parse(f"{module}.{func.__name__}").body[0].value
+ else:
+ visitor = ConfigExpressionParser(value)
+ args = ast.arguments(
+ posonlyargs=[],
+ kwonlyargs=[],
+ kw_defaults=[],
+ defaults=[],
+ args=[ast.arg(arg=a, annotation=None) for a in visitor.args],
+ kwarg=None,
+ vararg=None,
+ )
+ value = ast.Lambda(args=args, body=value)
# NOTE:
# don't pass the generated functions down
# to the checks implemented in this class...
@@ -159,8 +177,7 @@ class ConfigFunctionParser(ast.NodeVisitor):
mode="single",
)
# NOTE: only pass a copy to not clutter the ENVIRONMENT
- # try:
- exec(co, {**ENVIRONMENT}, self.kwargs)
+ exec(co, {**ENVIRONMENT, **imports}, self.kwargs)
# let's do some more validity checks
if check_tree:
diff --git a/tests/core/test_reader.py b/tests/core/test_reader.py
index c3de90c16..a7d0f2d66 100644
--- a/tests/core/test_reader.py
+++ b/tests/core/test_reader.py
@@ -9,7 +9,9 @@
import numpy as np
import pytest
-from saqc.core import DictOfSeries, Flags, flagging
+import saqc.lib.ts_operators as ts_ops
+from saqc.core import DictOfSeries, Flags, SaQC, flagging
+from saqc.parsing.environ import ENVIRONMENT
from saqc.parsing.reader import fromConfig, readFile
from tests.common import initData, writeIO
@@ -155,3 +157,21 @@ def test_supportedArguments(data):
for test in tests:
fobj = writeIO(header + "\n" + test)
fromConfig(fobj, data)
+
+
+@pytest.mark.parametrize(
+ "func_string", [k for k, v in ENVIRONMENT.items() if callable(v)]
+)
+def test_funtionArguments(data, func_string):
+ @flagging()
+ def testFunction(saqc, field, func, **kwargs):
+ assert func is ENVIRONMENT[func_string]
+ return saqc
+
+ config = f"""
+ varname ; test
+ {data.columns[0]} ; testFunction(func={func_string})
+ {data.columns[0]} ; testFunction(func="{func_string}")
+ """
+
+ fromConfig(writeIO(config), data)
--
GitLab