bugfix: cli failed after merge of dios

.gitlab-ci.yml

@@ -20,7 +20,7 @@ test:python36:
     - pip install --upgrade pip
     - pip install -r requirements.txt
     - python -m pytest test
-    - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv
+    - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
 
 
 test:python37:
@@ -30,7 +30,7 @@ test:python37:
     - pip install --upgrade pip
     - pip install -r requirements.txt
     - python -m pytest test
-    - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv
+    - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv
 
 
 test:python38:
@@ -40,4 +40,4 @@ test:python38:
     - pip install --upgrade pip
     - pip install -r requirements.txt
     - python -m pytest test
-    - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv
+    - python -m saqc --config ressources/data/config_ci.csv --data ressources/data/data.csv --outfile /tmp/test.csv

saqc/__main__.py

@@ -50,22 +50,22 @@ def main(config, data, flagger, outfile, nodata, log_level, fail):
     )
 
     if outfile:
-        flags = flagger_result.getFlags()
-        flags_out = flags.where((flags.isnull() | flagger_result.isFlagged()), flagger_result.GOOD)
+        data_result = data_result.to_df()
+        flags = flagger_result.getFlags().to_df()
+        flags_flagged = flagger_result.isFlagged().to_df()
 
-        if isinstance(flagger_result, DmpFlagger):
-            flags = flagger_result._flags
-            flags.loc[flags_out.index, (slice(None), FlagFields.FLAG)] = flags_out.values
-            flags_out = flags
-
-        if not isinstance(flags_out.columns, pd.MultiIndex):
-            flags_out.columns = pd.MultiIndex.from_product([flags.columns, ["flag"]])
+        flags_out = flags.where((flags.isnull() | flags_flagged), flagger_result.GOOD)
+        fields = {"data": data_result, "flags": flags_out}
 
-        data_result.columns = pd.MultiIndex.from_product([data_result.columns, ["data"]])
-
-        # flags_out.columns = flags_out.columns.map("_".join)
-        data_out = data_result.join(flags_out)
-        data_out.sort_index(axis="columns").to_csv(outfile, header=True, index=True, na_rep=nodata)
+        if isinstance(flagger_result, DmpFlagger):
+            fields["comments"] = flagger_result.comments.to_df()
+            fields["causes"] = flagger_result.causes.to_df()
+
+        out = (pd.concat(fields.values(), axis=1, keys=fields.keys())
+               .reorder_levels(order=[1, 0], axis=1)
+               .sort_index(axis=1, level=0, sort_remaining=False))
+        out.columns = out.columns.rename(["", ""])
+        out.to_csv(outfile, header=True, index=True, na_rep=nodata)
 
 
 if __name__ == "__main__":

saqc/flagger/dmpflagger.py

@@ -3,15 +3,11 @@
 import subprocess
 import json
 from copy import deepcopy
-from collections import OrderedDict
-from typing import Union, Sequence
 
-import pandas as pd
 import dios.dios as dios
 
 from saqc.flagger.categoricalflagger import CategoricalFlagger
-from saqc.flagger.baseflagger import diosT
-from saqc.lib.tools import assertDictOfSeries, toSequence, assertScalar
+from saqc.lib.tools import assertScalar
 
 
 class Keywords: