From f92fba5dd05f79ba849a832fc908c6ab667ca961 Mon Sep 17 00:00:00 2001
From: David Schaefer <david.schaefer@ufz.de>
Date: Fri, 14 Aug 2020 20:31:17 +0200
Subject: [PATCH] data is reduced to the fields needed by a test
---
saqc/core/register.py | 13 +++++++------
test/core/test_core.py | 3 +--
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/saqc/core/register.py b/saqc/core/register.py
index 83c845e0a..89bf09bb9 100644
--- a/saqc/core/register.py
+++ b/saqc/core/register.py
@@ -162,7 +162,8 @@ class SaQCFunc(Func):
if field not in flagger.getFlags():
flagger = flagger.merge(flagger.initFlags(data=pd.Series(name=field)))
- data_in = self._maskData(data, flagger)
+ columns_in = data.columns.intersection([field])
+ data_in = self._maskData(data.loc[:, columns_in], flagger)
data_result, flagger_result = self.func(data_in, field, flagger, *self.args, **self.kwargs)
@@ -189,20 +190,20 @@ class SaQCFunc(Func):
mask_old = flagger_old.isFlagged(flag=to_mask, comparator="==")
mask_new = flagger_new.isFlagged(flag=to_mask, comparator="==")
- for col, left in data_new.indexes.iteritems():
+ for col, right in data_new.indexes.iteritems():
if col not in mask_old:
continue
- right = mask_old[col].index
+ left = mask_old[col].index
+ col_data = data_new[col].values
# NOTE: ignore columns with changed indices (assumption: harmonization)
if left.equals(right):
# NOTE: Don't overwrite data, that was masked, but is not considered
# flagged anymore and also respect newly set data on masked locations.
mask = mask_old[col].values & mask_new[col].values & data_new[col].isna().values
if np.any(mask):
- col_data = data_new[col].values
col_data[mask] = data_old[col].values[mask]
- data_new[col] = col_data
- return data_new
+ data_old[col] = col_data
+ return data_old
# NOTE:
diff --git a/test/core/test_core.py b/test/core/test_core.py
index 21a9624a3..4b1a166c6 100644
--- a/test/core/test_core.py
+++ b/test/core/test_core.py
@@ -79,7 +79,7 @@ def test_assignVariable(flagger):
pdata, pflagger = SaQC(flagger, data).flagAll(var1).flagAll(var2).getResult()
pflags = pflagger.getFlags()
- assert (pflags.columns == [var1, var2]).all()
+ assert (set(pflags.columns) == {var1, var2})
assert pflagger.isFlagged(var2).empty
@@ -105,7 +105,6 @@ def test_masking(data, flagger):
test if flagged values are exluded during the preceding tests
"""
flagger = flagger.initFlags(data)
- flags = flagger.getFlags()
var1 = 'var1'
mn = min(data[var1])
mx = max(data[var1]) / 2
--
GitLab