flagPattern fails
I guess the problem arises from here https://git.ufz.de/rdm-software/saqc/-/blob/develop/saqc/funcs/functions.py#L329
test = data[field].copy()
ref = data[reference_field].copy()
pattern_start_date = ref.index[0].time()
pattern_end_date = ref.index[-1].time()
### Extract partition frequency from pattern if needed
if not isinstance(partition_freq, str):
raise ValueError('Partition frequency has to be given in string format.')
elif partition_freq == "days" or partition_freq == "months":
# Get partition frequency from reference field
partition_count = (pattern_end_date - pattern_start_date).days
partitions = test.groupby(pd.Grouper(freq="%d D" % (partition_count + 1)))
else:
partitions = test.groupby(pd.Grouper(freq=partition_freq))
this ref.index[1].time() - ref.index[0].time()
fails, but this ref.index[1] - ref.index[0]
would work and will result in a pd.Timedelta
.
Traceback of Error
Traceback (most recent call last):
File "/home/palmb/projects/gcefQA/pipeline_testing.py", line 272, in <module>
test_pipeline(usr, pwd)
File "/home/palmb/projects/gcefQA/pipeline_testing.py", line 125, in test_pipeline
qc1 = pipeline_stages.qc_stage_I(qc, meta, db)
File "/home/palmb/projects/gcefQA/pipeline_stages.py", line 62, in qc_stage_I
qc = cls.preProcess(qc, field, meta).evaluate()
File "/home/palmb/projects/gcefQA/saqc/saqc/core/core.py", line 173, in evaluate
_handleErrors(e, func_dump, self._error_policy)
File "/home/palmb/projects/gcefQA/saqc/saqc/core/core.py", line 168, in evaluate
data_result, flagger_result = _saqcCallFunc(func_dump, data, flagger)
File "/home/palmb/projects/gcefQA/saqc/saqc/core/core.py", line 294, in _saqcCallFunc
data_result, flagger_result = func(data_in, field, flagger, *func_args, func_name=func_name, **func_kws)
File "/home/palmb/projects/gcefQA/saqc/saqc/funcs/functions.py", line 150, in flagPattern
partition_count = (pattern_end_date - pattern_start_date).days
TypeError: unsupported operand type(s) for -: 'datetime.time' and 'datetime.time'
Traceback (most recent call last):
File "/home/palmb/projects/gcefQA/pipeline_testing.py", line 272, in <module>
test_pipeline(usr, pwd)
File "/home/palmb/projects/gcefQA/pipeline_testing.py", line 125, in test_pipeline
qc1 = pipeline_stages.qc_stage_I(qc, meta, db)
File "/home/palmb/projects/gcefQA/pipeline_stages.py", line 62, in qc_stage_I
qc = cls.preProcess(qc, field, meta).evaluate()
File "/home/palmb/projects/gcefQA/saqc/saqc/core/core.py", line 173, in evaluate
_handleErrors(e, func_dump, self._error_policy)
File "/home/palmb/projects/gcefQA/saqc/saqc/core/core.py", line 168, in evaluate
data_result, flagger_result = _saqcCallFunc(func_dump, data, flagger)
File "/home/palmb/projects/gcefQA/saqc/saqc/core/core.py", line 294, in _saqcCallFunc
data_result, flagger_result = func(data_in, field, flagger, *func_args, func_name=func_name, **func_kws)
File "/home/palmb/projects/gcefQA/saqc/saqc/funcs/functions.py", line 150, in flagPattern
partition_count = (pattern_end_date - pattern_start_date).days
TypeError: unsupported operand type(s) for -: 'datetime.time' and 'datetime.time'