Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
SaQC
Manage
Activity
Members
Labels
Plan
Issues
36
Issue boards
Milestones
Wiki
Code
Merge requests
8
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container Registry
Model registry
Operate
Environments
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
rdm-software
SaQC
Commits
a1087b70
Commit
a1087b70
authored
2 years ago
by
Peter Lünenschloß
Browse files
Options
Downloads
Patches
Plain Diff
removed left over/adopted tests from MR 599
parent
7670fcb9
No related branches found
No related tags found
1 merge request
!600
Inter limit fix
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
saqc/lib/ts_operators.py
+6
-4
6 additions, 4 deletions
saqc/lib/ts_operators.py
tests/lib/test_ts_operators.py
+10
-39
10 additions, 39 deletions
tests/lib/test_ts_operators.py
with
16 additions
and
43 deletions
saqc/lib/ts_operators.py
+
6
−
4
View file @
a1087b70
...
...
@@ -314,11 +314,11 @@ def interpolateNANs(
:param method: String. Method keyword designating interpolation method to use.
:param order: Integer. If your desired interpolation method needs an order to be passed -
here you pass it.
:param inter_limit: Integer. Default = 2.
Limit
up to which consecutive nan - values in the data get
:param inter_limit: Integer. Default = 2.
Number
up to which consecutive nan - values in the data get
replaced by interpolation.
Its default value suits an interpolation that only will apply to points of an
inserted frequency grid. (regularization by interpolation)
Gaps
wider than
"
limit
"
will NOT be interpolated at all.
Gaps
of size
"
limit
"
or greater
will NOT be interpolated at all.
:param downgrade_interpolation: Boolean. Default False. If True:
If a data chunk not contains enough values for interpolation of the order
"
order
"
,
the highest order possible will be selected for that chunks interpolation.
...
...
@@ -330,11 +330,13 @@ def interpolateNANs(
if
inter_limit
is
None
:
# if there is actually no limit set to the gaps to-be interpolated, generate a dummy mask for the gaps
gap_mask
=
pd
.
Series
(
True
,
index
=
data
.
index
,
name
=
data
.
name
)
elif
inter_limit
<
2
:
return
data
else
:
# if there is a limit to the gaps to be interpolated, generate a mask that evaluates to False at the right side
# of each too-large gap with a rolling.sum combo
gap_mask
=
data
.
isna
().
rolling
(
inter_limit
,
min_periods
=
0
).
sum
()
!=
inter_limit
if
inter_limit
<
2
0
:
if
inter_limit
==
2
:
# for the common case of inter_limit=2 (default "harmonisation"), we efficiently bag propagate the False
# value to fill the whole too-large gap by a shift and a conjunction.
gap_mask
&=
gap_mask
&
gap_mask
.
shift
(
-
1
,
fill_value
=
True
)
...
...
@@ -363,7 +365,7 @@ def interpolateNANs(
)
else
:
# if the method that is inerpolated with depends on not only the left and right border points of any gap,
# if the method that is in
t
erpolated with depends on not only the left and right border points of any gap,
# but includes more points, it has to be applied on any data chunk seperated by the too-big gaps individually.
# So we use the gap_mask to group the data into chunks and perform the interpolation on every chunk seperatly
# with the .transform method of the grouper.
...
...
This diff is collapsed.
Click to expand it.
tests/lib/test_ts_operators.py
+
10
−
39
View file @
a1087b70
...
...
@@ -198,69 +198,40 @@ def test_rateOfChange(data, expected):
@pytest.mark.parametrize
(
"
limit,
area,direction,
data,expected
"
,
"
limit,data,expected
"
,
[
(
1
,
"
inside
"
,
None
,
[
np
.
nan
,
0
,
np
.
nan
,
np
.
nan
,
np
.
nan
,
4
,
np
.
nan
],
[
np
.
nan
,
0
,
np
.
nan
,
np
.
nan
,
np
.
nan
,
4
,
np
.
nan
],
),
(
2
,
"
inside
"
,
None
,
[
np
.
nan
,
0
,
np
.
nan
,
np
.
nan
,
np
.
nan
,
4
,
np
.
nan
],
[
np
.
nan
,
0
,
np
.
nan
,
np
.
nan
,
np
.
nan
,
4
,
np
.
nan
],
),
(
3
,
"
inside
"
,
None
,
[
np
.
nan
,
0
,
np
.
nan
,
np
.
nan
,
np
.
nan
,
4
,
np
.
nan
],
[
np
.
nan
,
0
,
1
,
2
,
3
,
4
,
np
.
nan
],
),
(
None
,
"
inside
"
,
None
,
[
np
.
nan
,
0
,
np
.
nan
,
np
.
nan
,
np
.
nan
,
4
,
np
.
nan
],
[
np
.
nan
,
0
,
1
,
2
,
3
,
4
,
np
.
nan
],
),
(
None
,
"
outside
"
,
"
forward
"
,
[
np
.
nan
,
0
,
np
.
nan
,
np
.
nan
,
np
.
nan
,
4
,
np
.
nan
],
[
np
.
nan
,
0
,
np
.
nan
,
np
.
nan
,
np
.
nan
,
4
,
4
],
),
(
None
,
"
outside
"
,
"
backward
"
,
[
np
.
nan
,
0
,
np
.
nan
,
np
.
nan
,
np
.
nan
,
4
,
np
.
nan
],
[
0
,
0
,
np
.
nan
,
np
.
nan
,
np
.
nan
,
4
,
np
.
nan
],
),
(
None
,
"
outside
"
,
"
both
"
,
4
,
[
np
.
nan
,
0
,
np
.
nan
,
np
.
nan
,
np
.
nan
,
4
,
np
.
nan
],
[
0
,
0
,
np
.
nan
,
np
.
nan
,
np
.
nan
,
4
,
4
],
[
np
.
nan
,
0
,
1
,
2
,
3
,
4
,
np
.
nan
],
),
(
None
,
None
,
"
both
"
,
[
np
.
nan
,
0
,
np
.
nan
,
np
.
nan
,
np
.
nan
,
4
,
np
.
nan
],
[
0
,
0
,
1
,
2
,
3
,
4
,
4
],
[
np
.
nan
,
0
,
1
,
2
,
3
,
4
,
np
.
nan
],
),
],
)
def
test_interpolatNANs
(
limit
,
area
,
direction
,
data
,
expected
):
def
test_interpolatNANs
(
limit
,
data
,
expected
):
got
=
interpolateNANs
(
pd
.
Series
(
data
),
inter_limit
=
limit
pd
.
Series
(
data
),
inter_limit
=
limit
,
method
=
'
linear
'
)
assert
got
.
equals
(
pd
.
Series
(
expected
,
dtype
=
float
))
try
:
assert
got
.
equals
(
pd
.
Series
(
expected
,
dtype
=
float
))
except
:
print
(
'
stop
'
)
\ No newline at end of file
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment