diff --git a/saqc/__init__.py b/saqc/__init__.py index d7155f47acb228ad2215d8a4dd283351860b1299..6262ae74d17d86a6eb67b7bdc617173134189bca 100644 --- a/saqc/__init__.py +++ b/saqc/__init__.py @@ -5,6 +5,4 @@ __version__ = "1.4" # import order: from small to big from saqc.constants import * -from saqc.flagger import * -from saqc.core.register import register -from saqc.core.core import SaQC +from saqc.core import register, initFlagsLike, Flags, SaQC diff --git a/saqc/core/__init__.py b/saqc/core/__init__.py index 097236acb6ecd8511ca716cb2b5206a8f00cdb80..0b4aacf337c1efe9a7d1c7e725a546b6a86903bb 100644 --- a/saqc/core/__init__.py +++ b/saqc/core/__init__.py @@ -2,4 +2,5 @@ # -*- coding: utf-8 -*- from saqc.core.register import register +from saqc.core.flags import Flags, initFlagsLike from saqc.core.core import SaQC, logger diff --git a/saqc/core/core.py b/saqc/core/core.py index 6d5ecbf16fcde0f327a012365f025c6b474d2cb1..2fe5c6e1106cbe3c0fc1a267a43c47fa83fb7cbc 100644 --- a/saqc/core/core.py +++ b/saqc/core/core.py @@ -18,7 +18,7 @@ import timeit import inspect from saqc.constants import * -from saqc.flagger import initFlagsLike, Flagger +from saqc.core.flags import initFlagsLike, Flags as Flagger from saqc.core.lib import APIController, ColumnSelector from saqc.core.register import FUNC_MAP, SaQCFunction from saqc.core.modules import FuncModules diff --git a/saqc/flagger/flags.py b/saqc/core/flags.py similarity index 99% rename from saqc/flagger/flags.py rename to saqc/core/flags.py index 63d3a505f3322ce4aef94f2fd6b89067d1a5b376..1d95adf310ebac9e5ecaf89030578471061e64f4 100644 --- a/saqc/flagger/flags.py +++ b/saqc/core/flags.py @@ -2,12 +2,14 @@ from __future__ import annotations -import dios -from saqc.constants import * -from saqc.flagger.history import History import pandas as pd +import dios from typing import Union, Dict, DefaultDict, Optional, Type, Tuple, Iterable +from saqc.constants import * +from saqc.core.history import History + + _VAL = Union[pd.Series, History] DictLike = Union[ pd.DataFrame, @@ -343,7 +345,3 @@ def initFlagsLike( result[k] = History(item) return Flags(result) - - -# for now we keep this name -Flagger = Flags diff --git a/saqc/flagger/history.py b/saqc/core/history.py similarity index 100% rename from saqc/flagger/history.py rename to saqc/core/history.py diff --git a/saqc/core/modules/breaks.py b/saqc/core/modules/breaks.py index b07edba09759687f21746222bea0cf10c2a70562..6fab21ff34a07674f430322f377f410019a5acce 100644 --- a/saqc/core/modules/breaks.py +++ b/saqc/core/modules/breaks.py @@ -6,8 +6,8 @@ import numpy as np from dios import DictOfSeries from saqc.constants import * -from saqc import Flagger from saqc.core.modules.base import ModuleBase +from saqc.core import Flags as Flagger from saqc.lib.types import FreqString, IntegerWindow, ColumnName diff --git a/saqc/core/modules/changepoints.py b/saqc/core/modules/changepoints.py index 19ed26d296d28baeb6a7182e1069ed39a2082ad8..7e5946cc6669ec65c5247fb5e6f6ded4fc601f22 100644 --- a/saqc/core/modules/changepoints.py +++ b/saqc/core/modules/changepoints.py @@ -8,7 +8,7 @@ from dios import DictOfSeries from typing_extensions import Literal from saqc.constants import * -from saqc import Flagger +from saqc.core import Flags as Flagger from saqc.core.modules.base import ModuleBase from saqc.lib.types import FreqString, IntegerWindow diff --git a/saqc/core/modules/constants.py b/saqc/core/modules/constants.py index 22239aa09e09ba4299ae7fe410c70f56731ff894..6787b08ed7a342b17ab99c0e188fb1b252b4edeb 100644 --- a/saqc/core/modules/constants.py +++ b/saqc/core/modules/constants.py @@ -5,7 +5,7 @@ from typing import Tuple from dios import DictOfSeries from saqc.constants import * -from saqc import Flagger +from saqc.core import Flags as Flagger from saqc.core.modules.base import ModuleBase from saqc.lib.types import FreqString, ColumnName diff --git a/saqc/core/modules/curvefit.py b/saqc/core/modules/curvefit.py index c24ce08b046fbb490f26550eeca2aeccd728db69..de43a906b6658bd8903af73d79bc820f37fbb064 100644 --- a/saqc/core/modules/curvefit.py +++ b/saqc/core/modules/curvefit.py @@ -6,7 +6,7 @@ from dios import DictOfSeries from typing_extensions import Literal from saqc.constants import * -from saqc import Flagger +from saqc.core import Flags as Flagger from saqc.core.modules.base import ModuleBase diff --git a/saqc/core/modules/flagtools.py b/saqc/core/modules/flagtools.py index 7cc2b16337eff358929e103b4968f93ee94b7dda..94b4748f8a7aaf3f3d907cef9fb39fe2d5c53fb2 100644 --- a/saqc/core/modules/flagtools.py +++ b/saqc/core/modules/flagtools.py @@ -7,7 +7,7 @@ import pandas as pd from dios.dios import DictOfSeries from typing_extensions import Literal -from saqc import Flagger +from saqc.core import Flags as Flagger from saqc.constants import * from saqc.core.modules.base import ModuleBase from saqc.lib.types import ColumnName diff --git a/saqc/core/modules/generic.py b/saqc/core/modules/generic.py index da80700c325b233793a217bd73dd6ad0f7755f39..649f5aafbc13bdc6b529186c3eea3b7006b6634a 100644 --- a/saqc/core/modules/generic.py +++ b/saqc/core/modules/generic.py @@ -7,7 +7,8 @@ import numpy as np import pandas as pd from dios import DictOfSeries -from saqc import Flagger, BAD +from saqc.constants import * +from saqc.core import Flags as Flagger from saqc.core.modules.base import ModuleBase diff --git a/saqc/core/modules/interpolation.py b/saqc/core/modules/interpolation.py index c9aed6105c51e9356cd78035b0b04a3d99235de4..8df763b902c4cd9eaa2dfd0ab8a686e1a46c57fa 100644 --- a/saqc/core/modules/interpolation.py +++ b/saqc/core/modules/interpolation.py @@ -8,7 +8,7 @@ import pandas as pd from dios import DictOfSeries from typing_extensions import Literal -from saqc import Flagger +from saqc.core import Flags as Flagger from saqc.constants import * from saqc.core.modules.base import ModuleBase from saqc.funcs.interpolation import _SUPPORTED_METHODS diff --git a/saqc/core/modules/outliers.py b/saqc/core/modules/outliers.py index bf81520390f82b5dcb2f2a17e7a5bf918944da1d..d202af9b904c7b71c4918581387b0834aac067fc 100644 --- a/saqc/core/modules/outliers.py +++ b/saqc/core/modules/outliers.py @@ -9,7 +9,7 @@ from dios import DictOfSeries from typing_extensions import Literal from saqc.constants import * -from saqc import Flagger +from saqc.core import Flags as Flagger from saqc.core.modules.base import ModuleBase from saqc.lib.types import IntegerWindow, FreqString, ColumnName diff --git a/saqc/core/modules/pattern.py b/saqc/core/modules/pattern.py index 38d0839454ee27340ff830794b0e331ea8abfbef..56db5f8523d9d4d7c30b16e3b197f653786edd97 100644 --- a/saqc/core/modules/pattern.py +++ b/saqc/core/modules/pattern.py @@ -6,7 +6,7 @@ from typing import Sequence, Tuple from dios import DictOfSeries from saqc.constants import * -from saqc import Flagger +from saqc.core import Flags as Flagger from saqc.core.modules.base import ModuleBase diff --git a/saqc/core/modules/resampling.py b/saqc/core/modules/resampling.py index 9822bbd8b146af03088548b6db3e7ca8f1436e8e..e5996987a49c8a6ee026b40589e52fc8bb16b32b 100644 --- a/saqc/core/modules/resampling.py +++ b/saqc/core/modules/resampling.py @@ -9,7 +9,7 @@ from dios import DictOfSeries from typing_extensions import Literal from saqc.constants import * -from saqc import Flagger +from saqc.core import Flags as Flagger from saqc.core.modules.base import ModuleBase from saqc.funcs.interpolation import _SUPPORTED_METHODS diff --git a/saqc/core/modules/residues.py b/saqc/core/modules/residues.py index 877323546bbf4dc89c6d622cb45c4e45f261b415..d0a03bac90f52703d0d4d8e44be6b413e25cd80c 100644 --- a/saqc/core/modules/residues.py +++ b/saqc/core/modules/residues.py @@ -8,7 +8,7 @@ from dios import DictOfSeries from typing_extensions import Literal from saqc.constants import * -from saqc import Flagger +from saqc.core import Flags as Flagger from saqc.core.modules.base import ModuleBase diff --git a/saqc/core/modules/scores.py b/saqc/core/modules/scores.py index a36e73d887575f85141c30e47899bb4395796a50..eafc44d096c3dff5442087c0d7a0e0f47c3a9ab9 100644 --- a/saqc/core/modules/scores.py +++ b/saqc/core/modules/scores.py @@ -8,7 +8,7 @@ import pandas as pd from dios import DictOfSeries from typing_extensions import Literal -from saqc import Flagger +from saqc.core import Flags as Flagger from saqc.core.modules.base import ModuleBase diff --git a/saqc/core/modules/tools.py b/saqc/core/modules/tools.py index 7d3c398595d06fe96300a96f7ef954a2815d322b..16a19bc0e298a683b6d945cb49ed18b71bb14e9a 100644 --- a/saqc/core/modules/tools.py +++ b/saqc/core/modules/tools.py @@ -6,7 +6,7 @@ from typing import Optional, Tuple from dios import DictOfSeries from typing_extensions import Literal -from saqc import Flagger +from saqc.core import Flags as Flagger from saqc.core.modules.base import ModuleBase diff --git a/saqc/core/modules/transformation.py b/saqc/core/modules/transformation.py index f85053fc55b9caee738eb3edf75b4cf6dcb1ad74..ff94e0bbd2827ba6dcffd5aa2d3c007783842724 100644 --- a/saqc/core/modules/transformation.py +++ b/saqc/core/modules/transformation.py @@ -6,7 +6,7 @@ from typing import Callable, Optional, Union, Tuple import pandas as pd from dios import DictOfSeries -from saqc import Flagger +from saqc.core import Flags as Flagger from saqc.core.modules.base import ModuleBase diff --git a/saqc/core/reader.py b/saqc/core/reader.py index 32d5b6985ac686f385301c5bb0eadb8ab0ae449e..e5aa0bce925bf798e3ac048ad05aea667c4c58d2 100644 --- a/saqc/core/reader.py +++ b/saqc/core/reader.py @@ -2,19 +2,17 @@ # -*- coding: utf-8 -*- import ast -from saqc.core.core import ColumnSelector - import numpy as np - import pandas as pd +from saqc.core.core import ColumnSelector from saqc.core.config import Fields as F from saqc.core.visitor import ConfigFunctionParser from saqc.core.lib import ConfigController from saqc.core.register import FUNC_MAP - from saqc.lib.tools import isQuoted + COMMENT = "#" EMPTY = "None" diff --git a/saqc/core/register.py b/saqc/core/register.py index 8cd859530713e679d5b0fdd43015ef2ce9bbbe69..c3b3945fa01e63557ebbcfafe41740aabc2531bf 100644 --- a/saqc/core/register.py +++ b/saqc/core/register.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import logging -from typing import Dict, Optional, Union, Tuple, List +from typing import Dict, Optional, Union, Tuple from typing_extensions import Literal from functools import wraps import dataclasses @@ -11,8 +10,8 @@ import warnings from saqc.constants import * from saqc.core.lib import SaQCFunction +from saqc.core.flags import initFlagsLike, Flags as Flagger from saqc.lib.types import FuncReturnT -from saqc.flagger.flags import Flagger, initFlagsLike # NOTE: # the global SaQC function store, diff --git a/saqc/core/visitor.py b/saqc/core/visitor.py index c517261e0318a0a1bee0bef6423bdb139070fb68..0fadf4878bce9575450a92035bc0156284db317b 100644 --- a/saqc/core/visitor.py +++ b/saqc/core/visitor.py @@ -2,7 +2,6 @@ # -*- coding: utf-8 -*- import ast - import numpy as np import pandas as pd diff --git a/saqc/flagger/__init__.py b/saqc/flagger/__init__.py deleted file mode 100644 index bbf08253176273442cab40eaf62b886816403329..0000000000000000000000000000000000000000 --- a/saqc/flagger/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -from .history import History -from .flags import Flagger, initFlagsLike diff --git a/saqc/funcs/breaks.py b/saqc/funcs/breaks.py index 8f10e9b7208b7a3d11d9a7911b1f6336e8a696ca..aede9d6310ba6e124f3b485c9c10c24c27f3b206 100644 --- a/saqc/funcs/breaks.py +++ b/saqc/funcs/breaks.py @@ -20,8 +20,7 @@ from saqc.constants import * from saqc.lib.tools import groupConsecutives from saqc.lib.types import FreqString, ColumnName, IntegerWindow from saqc.funcs.changepoints import assignChangePointCluster -from saqc.core.register import register -from saqc.flagger import Flagger +from saqc.core import register, Flags as Flagger @register(masking='field', module="breaks") diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py index 4ef620f54515a8362fc5a4a19bd47f9085b64223..7c37b9ca1804a14e975c5fac197afed20bdcece1 100644 --- a/saqc/funcs/changepoints.py +++ b/saqc/funcs/changepoints.py @@ -12,9 +12,8 @@ from typing_extensions import Literal from dios import DictOfSeries from saqc.constants import * -from saqc.core.register import register from saqc.lib.tools import customRoller -from saqc.flagger import Flagger +from saqc.core import register, Flags as Flagger from saqc.lib.types import ColumnName, FreqString, IntegerWindow logger = logging.getLogger("SaQC") diff --git a/saqc/funcs/constants.py b/saqc/funcs/constants.py index 5d0b30804f36b9081cf51bd0273ef70a933daf74..a6b99a07c7f354a5c7394e06ab21cf5d36f4cf4b 100644 --- a/saqc/funcs/constants.py +++ b/saqc/funcs/constants.py @@ -10,8 +10,7 @@ import pandas as pd from dios import DictOfSeries from saqc.constants import * -from saqc.core.register import register -from saqc.flagger import Flagger +from saqc.core import register, Flags as Flagger from saqc.lib.ts_operators import varQC from saqc.lib.tools import customRoller, getFreqDelta from saqc.lib.types import FreqString, ColumnName diff --git a/saqc/funcs/curvefit.py b/saqc/funcs/curvefit.py index f77b75346d66afda2457146b8705874e090a5ebf..d48d7ae4e0ed59b8e8e5b773620ff88d9cb81f3d 100644 --- a/saqc/funcs/curvefit.py +++ b/saqc/funcs/curvefit.py @@ -9,9 +9,8 @@ import pandas as pd from dios import DictOfSeries from saqc.constants import * -from saqc.core.register import register +from saqc.core import register, Flags as Flagger from saqc.lib.tools import getFreqDelta -from saqc.flagger import Flagger from saqc.lib.ts_operators import ( polyRollerIrregular, polyRollerNumba, diff --git a/saqc/funcs/drift.py b/saqc/funcs/drift.py index 2dd2fe7c411e026b85e43b3d694ba5a4ab08407f..0892673a80a4e042511d3bc4a19c022e340f637b 100644 --- a/saqc/funcs/drift.py +++ b/saqc/funcs/drift.py @@ -15,7 +15,7 @@ from scipy.spatial.distance import pdist from saqc.constants import * from saqc.core.register import register -from saqc.flagger import Flagger +from saqc.core import Flags as Flagger from saqc.funcs.resampling import shift from saqc.funcs.changepoints import assignChangePointCluster from saqc.funcs.tools import drop, copy diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py index 56c6a689c0a489f0d36a1833a10ef801b82e5d5c..db0ce930df9b47c9a43e82d963cabd007907c2fa 100644 --- a/saqc/funcs/flagtools.py +++ b/saqc/funcs/flagtools.py @@ -2,15 +2,12 @@ # -*- coding: utf-8 -*- from typing import Any, Tuple, Optional, Union from typing_extensions import Literal - import pandas as pd - from dios import DictOfSeries from saqc.constants import * from saqc.lib.types import * -from saqc.core.register import register -from saqc.flagger import Flagger +from saqc.core import register, Flags as Flagger import warnings diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py index b8677f199e3893634137f9580fd41f1354e3f98c..095ae1eea9cc9c750ee5c7044b9b820d143c6358 100644 --- a/saqc/funcs/generic.py +++ b/saqc/funcs/generic.py @@ -11,9 +11,8 @@ import pandas as pd from dios import DictOfSeries from saqc.constants import * -from saqc.core.register import register +from saqc.core import register, initFlagsLike, Flags as Flagger from saqc.core.visitor import ENVIRONMENT -from saqc.flagger import Flagger, initFlagsLike import operator as op diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py index a7880f4b0a067ca98b5e3d18c4975040fbdea0e8..9f3b985d7b7b7fb11242aa2e002383f02090b369 100644 --- a/saqc/funcs/interpolation.py +++ b/saqc/funcs/interpolation.py @@ -1,18 +1,15 @@ #! /usr/bin/env python # -*- coding: utf-8 -*- -import warnings -from typing import Tuple, Union, Optional, Any, Callable, Sequence +from typing import Tuple, Union, Callable from typing_extensions import Literal - import numpy as np import pandas as pd - from dios import DictOfSeries from saqc.constants import * -from saqc.core.register import register, _isflagged -from saqc.flagger import Flagger -from saqc.flagger.history import applyFunctionOnHistory +from saqc.core import register, Flags as Flagger +from saqc.core.register import _isflagged +from saqc.core.history import applyFunctionOnHistory from saqc.lib.ts_operators import interpolateNANs _SUPPORTED_METHODS = Literal[ diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py index d8b2fcd26c80a33d3f98db12a636de232620bef0..e6f804be895311dc4ca7a0850fac6d53a417e083 100644 --- a/saqc/funcs/outliers.py +++ b/saqc/funcs/outliers.py @@ -3,26 +3,18 @@ from typing import Optional, Union, Tuple, Sequence, Callable from typing_extensions import Literal - +import numba import numpy as np import numpy.polynomial.polynomial as poly -from scipy.optimize import curve_fit import pandas as pd -import numba - -from outliers import smirnov_grubbs - from dios import DictOfSeries +from outliers import smirnov_grubbs +from scipy.optimize import curve_fit from saqc.constants import * -from saqc.core.register import register -from saqc.flagger import Flagger -from saqc.lib.tools import ( - customRoller, - findIndex, - getFreqDelta -) +from saqc.core import register, Flags as Flagger from saqc.lib.types import ColumnName, FreqString, IntegerWindow +from saqc.lib.tools import customRoller, findIndex, getFreqDelta from saqc.funcs.scores import assignKNNScore import saqc.lib.ts_operators as ts_ops diff --git a/saqc/funcs/pattern.py b/saqc/funcs/pattern.py index 6562fd0d3415867726178c84015c426c3f2cfb1b..a51a157a4e8483c90beac2858b5593bda8cba537 100644 --- a/saqc/funcs/pattern.py +++ b/saqc/funcs/pattern.py @@ -9,8 +9,7 @@ from mlxtend.evaluate import permutation_test from dios.dios import DictOfSeries from saqc.constants import * -from saqc.core.register import register -from saqc.flagger import Flagger +from saqc.core import register, Flags as Flagger from saqc.lib.tools import customRoller diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py index 0a796d726e440a544359a0f7ec084a0edf77c6f5..466bcf382837f2687468e7c29592fca283d466ca 100644 --- a/saqc/funcs/resampling.py +++ b/saqc/funcs/resampling.py @@ -1,25 +1,22 @@ #! /usr/bin/env python # -*- coding: utf-8 -*- -from typing import Callable, Tuple, Optional, Union, Any, Sequence +from typing import Callable, Tuple, Optional, Union from typing_extensions import Literal - -import numpy as np import logging - +import numpy as np import pandas as pd - from dios import DictOfSeries from saqc.constants import * -from saqc.core.register import register, _isflagged -from saqc.flagger.history import applyFunctionOnHistory -from saqc.flagger.flags import Flagger -from saqc.funcs.tools import copy, drop, rename -from saqc.funcs.interpolation import interpolateIndex, _SUPPORTED_METHODS +from saqc.core import register, Flags as Flagger +from saqc.core.register import _isflagged +from saqc.core.history import applyFunctionOnHistory from saqc.lib.tools import evalFreqStr, getFreqDelta from saqc.lib.ts_operators import shift2Freq, aggregate2Freq -from saqc.lib.rolling import customRoller +from saqc.funcs.tools import copy, drop, rename +from saqc.funcs.interpolation import interpolateIndex, _SUPPORTED_METHODS + logger = logging.getLogger("SaQC") diff --git a/saqc/funcs/residues.py b/saqc/funcs/residues.py index 0b0046bea5e1baf6437272441b3c9d34b0fb3e68..b58c0cdf3cb322da5aa9fc464395cb0e3c006ce9 100644 --- a/saqc/funcs/residues.py +++ b/saqc/funcs/residues.py @@ -3,14 +3,11 @@ from typing import Tuple, Union, Optional, Callable from typing_extensions import Literal - import numpy as np - from dios import DictOfSeries from saqc.constants import * -from saqc.core.register import register -from saqc.flagger import Flagger +from saqc.core import register, Flags as Flagger from saqc.funcs.rolling import roll from saqc.funcs.curvefit import fitPolynomial diff --git a/saqc/funcs/rolling.py b/saqc/funcs/rolling.py index 6d58dfbc69a52aa607a5ee774a7093fbbab69235..6990bb72fcd7ed35c9e9c9515e0ae1d7c6e78762 100644 --- a/saqc/funcs/rolling.py +++ b/saqc/funcs/rolling.py @@ -2,14 +2,12 @@ # -*- coding: utf-8 -*- from typing import Union, Callable - import numpy as np import pandas as pd from dios import DictOfSeries from saqc.constants import * -from saqc.core.register import register -from saqc.flagger import Flagger +from saqc.core import register, Flags as Flagger from saqc.lib.tools import getFreqDelta diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py index 1f5d0456ebdf284097b7739eb719f6656115a201..f1690e0fa4ad605bcabe5274c025f7a4048af86d 100644 --- a/saqc/funcs/scores.py +++ b/saqc/funcs/scores.py @@ -2,17 +2,14 @@ # -*- coding: utf-8 -*- from typing import Union, Tuple, Callable, Sequence, Optional from typing_extensions import Literal - import numpy as np import pandas as pd - from dios import DictOfSeries from saqc.constants import * -from saqc.core.register import register -from saqc.flagger import Flagger -from saqc.lib import ts_operators as ts_ops +from saqc.core import register, Flags as Flagger from saqc.lib.tools import toSequence +import saqc.lib.ts_operators as ts_ops @register(masking='all', module="scores") diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py index c2a451b95b8eef1105b0a2af1697927002e10ebc..4ac072016a67afac6df94b90e653961df529c1de 100644 --- a/saqc/funcs/tools.py +++ b/saqc/funcs/tools.py @@ -3,14 +3,11 @@ from typing import Optional, Tuple from typing_extensions import Literal - import numpy as np - from dios import DictOfSeries from saqc.constants import * -from saqc.core.register import register -from saqc.flagger import Flagger +from saqc.core import register, Flags as Flagger from saqc.lib.tools import periodicMask diff --git a/saqc/funcs/transformation.py b/saqc/funcs/transformation.py index 6a176b4a905a8fc2c811f7aa90e816ca4692f727..fbda3ea6ff283dd53fb556022e36bfbb3085af2e 100644 --- a/saqc/funcs/transformation.py +++ b/saqc/funcs/transformation.py @@ -2,14 +2,11 @@ # -*- coding: utf-8 -*- from typing import Optional, Callable, Tuple, Union - import numpy as np import pandas as pd - from dios import DictOfSeries -from saqc.core.register import register -from saqc.flagger import Flagger +from saqc.core import register, Flags as Flagger @register(masking='field', module="transformation") diff --git a/saqc/lib/plotting.py b/saqc/lib/plotting.py index 5f79f28bb5932ee67044e706c0d3cc39d09c6db9..0543110c145892234fc6a0874ab792674f0f6c94 100644 --- a/saqc/lib/plotting.py +++ b/saqc/lib/plotting.py @@ -9,7 +9,7 @@ import dios import matplotlib.pyplot as plt from typing import List, Dict, Optional from saqc.constants import * -from saqc.flagger import Flagger +from saqc.core import Flags as Flagger def __importHelper(): diff --git a/saqc/lib/tools.py b/saqc/lib/tools.py index 6edfb047146bcd8240dbbb686773e748f272b3af..1f15df6d0710e54610ce4a55b26b62c5c834c45e 100644 --- a/saqc/lib/tools.py +++ b/saqc/lib/tools.py @@ -12,9 +12,9 @@ import pandas as pd from scipy import fft import logging import dios - import collections from scipy.cluster.hierarchy import linkage, fcluster + from saqc.lib.types import T # keep this for external imports diff --git a/saqc/lib/ts_operators.py b/saqc/lib/ts_operators.py index 37d8253ab55dc575669ae7c656e065599220c6b1..b22973074a59e5234c595517d2f16de4a33745f6 100644 --- a/saqc/lib/ts_operators.py +++ b/saqc/lib/ts_operators.py @@ -5,14 +5,11 @@ The module gathers all kinds of timeseries tranformations. """ import logging - import re from typing import Union - import pandas as pd import numpy as np import numba as nb - from sklearn.neighbors import NearestNeighbors from scipy.stats import iqr, median_abs_deviation import numpy.polynomial.polynomial as poly diff --git a/saqc/lib/types.py b/saqc/lib/types.py index 9a437a2a91f4574031b084f69d3e9ab6a6099d98..29830a86936bd3f79cefb428c6c325fc29cae3d6 100644 --- a/saqc/lib/types.py +++ b/saqc/lib/types.py @@ -15,18 +15,18 @@ __all__ = [ from typing import TypeVar, Union, NewType from typing_extensions import Protocol, Literal - import numpy as np import pandas as pd from dios import DictOfSeries -from saqc import Flagger +from saqc.core.flags import Flags + T = TypeVar("T") ArrayLike = TypeVar("ArrayLike", np.ndarray, pd.Series, pd.DataFrame) PandasLike = TypeVar("PandasLike", pd.Series, pd.DataFrame, DictOfSeries) DiosLikeT = Union[DictOfSeries, pd.DataFrame] -FuncReturnT = [DictOfSeries, Flagger] +FuncReturnT = [DictOfSeries, Flags] # we only support fixed length offsets FreqString = NewType("FreqString", Literal["D", "H", "T", "min", "S", "L", "ms", "U", "us", "N"]) @@ -36,6 +36,7 @@ ColumnName = NewType("ColumnName", str) IntegerWindow = NewType("IntegerWindow", int) TimestampColumnName = TypeVar("TimestampColumnName", bound=str) + # needed for deeper typy hinting magic class CurveFitter(Protocol): def __call__(self, data: np.ndarray, *params: float) -> np.ndarray: diff --git a/tests/common.py b/tests/common.py index e225b64102b7848ba128315322bb2e6f3d756499..eddda827d1df0852134a426f051eaeffa9e2c7d2 100644 --- a/tests/common.py +++ b/tests/common.py @@ -7,7 +7,7 @@ import pandas as pd import dios from saqc.constants import * -from saqc.flagger import Flagger, initFlagsLike +from saqc.core import initFlagsLike, Flags as Flagger TESTNODATA = (np.nan, -9999) diff --git a/tests/core/test_core.py b/tests/core/test_core.py index b9773a4731aab4abb5373ca0569ce35237c9bd32..c13f8a5b85eeaac88d498f7a24be092940812c51 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -7,7 +7,7 @@ import numpy as np import pandas as pd from saqc.constants import * -from saqc.flagger import initFlagsLike +from saqc.core import initFlagsLike from saqc.funcs import flagRange from saqc.lib import plotting as splot from saqc import SaQC, register diff --git a/tests/core/test_creation.py b/tests/core/test_creation.py index b16714c8e535a9c1c4e2ad2da156ec9fe2869da7..295d2adfca3efd4c3c6ada4e51ca60cf2c76c6ca 100644 --- a/tests/core/test_creation.py +++ b/tests/core/test_creation.py @@ -6,7 +6,7 @@ import dios def test_init(): - from saqc import SaQC, Flagger + from saqc import SaQC, Flags as Flagger arr = np.array([ [0, 1, 2], diff --git a/tests/flagger/test_flags.py b/tests/flagger/test_flags.py index db40b7e6c013ea13d8fe88f8b8050cda795432a5..d0d1585bcec5d49c425ef90b3cbdc70d5e878441 100644 --- a/tests/flagger/test_flags.py +++ b/tests/flagger/test_flags.py @@ -5,7 +5,7 @@ import numpy as np import pandas as pd from saqc.constants import * -from saqc.flagger.flags import Flags +from saqc.core.flags import Flags from tests.flagger.test_history import ( History, diff --git a/tests/flagger/test_history.py b/tests/flagger/test_history.py index 5a95585c0ecac7d70eea941148d7bfef6003918a..c85eceddd24c68d0cfa8090814ab6db12473d21b 100644 --- a/tests/flagger/test_history.py +++ b/tests/flagger/test_history.py @@ -4,7 +4,7 @@ import pytest import numpy as np import pandas as pd -from saqc.flagger.history import History +from saqc.core.history import History # see #GH143 combined backtrack # (adjusted to current implementation) diff --git a/tests/funcs/test_constants_detection.py b/tests/funcs/test_constants_detection.py index da7b83836f9f38581a8480bde07a97cc0bff8757..6fcde58d1c96d5874c63f9516768ec41d32f5085 100644 --- a/tests/funcs/test_constants_detection.py +++ b/tests/funcs/test_constants_detection.py @@ -6,7 +6,7 @@ import numpy as np from saqc.constants import * from saqc.funcs.constants import flagConstants, flagByVariance -from saqc.flagger import initFlagsLike +from saqc.core import initFlagsLike, Flags as Flagger from tests.common import initData diff --git a/tests/funcs/test_functions.py b/tests/funcs/test_functions.py index 47c8ae9d22f70bb4ecb5f489bc70f69f4ade723e..06eef82da8b962c12c1142a0629f71871e3c4e98 100644 --- a/tests/funcs/test_functions.py +++ b/tests/funcs/test_functions.py @@ -4,7 +4,7 @@ import dios from saqc.constants import * -from saqc.flagger import initFlagsLike +from saqc.core import initFlagsLike from saqc.funcs.drift import flagDriftFromNorm, flagDriftFromReference, flagDriftFromScaledNorm from saqc.funcs.outliers import flagCrossStatistic, flagRange from saqc.funcs.flagtools import flagManual, forceFlags, clearFlags diff --git a/tests/funcs/test_generic_config_functions.py b/tests/funcs/test_generic_config_functions.py index fe8242cd2662708c428a40cefc376867ab2621a8..5b3a28cb52a75dd4281ea884648f0152622dfbd2 100644 --- a/tests/funcs/test_generic_config_functions.py +++ b/tests/funcs/test_generic_config_functions.py @@ -8,7 +8,7 @@ import pandas as pd import dios from saqc.constants import * -from saqc.flagger import Flagger, initFlagsLike +from saqc.core import initFlagsLike, Flags as Flagger from saqc.core.visitor import ConfigFunctionParser from saqc.core.config import Fields as F from saqc.core.register import register diff --git a/tests/funcs/test_harm_funcs.py b/tests/funcs/test_harm_funcs.py index 052d5f8daf7c818d66bbcf9c3b6870e6ef616b5b..3f0140bd6dffb4f890c964bcde60988683c28ff5 100644 --- a/tests/funcs/test_harm_funcs.py +++ b/tests/funcs/test_harm_funcs.py @@ -6,7 +6,7 @@ import numpy as np import pandas as pd import dios -from saqc.flagger import Flagger, initFlagsLike +from saqc.core import initFlagsLike, Flags as Flagger from saqc.constants import BAD, UNFLAGGED from saqc.funcs.resampling import ( linear, diff --git a/tests/funcs/test_modelling.py b/tests/funcs/test_modelling.py index 248c122460a653a24829577b2f81b1bdbb01740b..95574936aaaf56b15a6317af26e83ddd9de3c6c3 100644 --- a/tests/funcs/test_modelling.py +++ b/tests/funcs/test_modelling.py @@ -7,7 +7,7 @@ import dios from saqc import BAD, UNFLAGGED -from saqc.flagger import initFlagsLike +from saqc.core import initFlagsLike from saqc.funcs.tools import mask from saqc.funcs.residues import calculatePolynomialResidues, calculateRollingResidues diff --git a/tests/funcs/test_pattern_rec.py b/tests/funcs/test_pattern_rec.py index b434e3c240e34c2ed71ab1bf6520ca7dfec38300..db3c50249a109c67d2d3fed51e97bc3cd1a1e0e5 100644 --- a/tests/funcs/test_pattern_rec.py +++ b/tests/funcs/test_pattern_rec.py @@ -6,7 +6,7 @@ import pandas as pd import dios from saqc.constants import * -from saqc.flagger import initFlagsLike +from saqc.core import initFlagsLike from saqc.funcs.pattern import * from tests.common import initData diff --git a/tests/funcs/test_proc_functions.py b/tests/funcs/test_proc_functions.py index d9d137359ead0d1b286a330d72523c4b90fa6bac..04739b40b6663703533a9a4cb30630761c49b112 100644 --- a/tests/funcs/test_proc_functions.py +++ b/tests/funcs/test_proc_functions.py @@ -7,7 +7,7 @@ import dios from saqc.constants import * -from saqc.flagger import initFlagsLike +from saqc.core import initFlagsLike from saqc.funcs.transformation import transform from saqc.funcs.drift import correctOffset from saqc.funcs.interpolation import interpolateByRolling, interpolateInvalid, interpolateIndex diff --git a/tests/funcs/test_spikes_detection.py b/tests/funcs/test_spikes_detection.py index faa25606811d8acb7a74a73b9bf5c9a811e0982e..727ef415775108d8e88367a7274e2fdc4eebc878 100644 --- a/tests/funcs/test_spikes_detection.py +++ b/tests/funcs/test_spikes_detection.py @@ -13,7 +13,7 @@ from saqc.funcs.outliers import ( flagByGrubbs, ) from saqc.constants import * -from saqc.flagger import initFlagsLike +from saqc.core import initFlagsLike @pytest.fixture(scope="module") diff --git a/tests/fuzzy/init.py b/tests/fuzzy/init.py index ad93f02c6ceecb966ce578656b6c63d8e8044642..4096823b575097ded6621f5a241b35942e6ab755 100644 --- a/tests/fuzzy/init.py +++ b/tests/fuzzy/init.py @@ -25,7 +25,7 @@ from saqc.constants import * from saqc.core.register import FUNC_MAP from saqc.core.lib import SaQCFunction from saqc.lib.types import FreqString, ColumnName, IntegerWindow -from saqc.flagger import Flagger, initFlagsLike +from saqc.core import initFlagsLike, Flags as Flagger MAX_EXAMPLES = 50 # MAX_EXAMPLES = 100000 diff --git a/tests/fuzzy/test_masking.py b/tests/fuzzy/test_masking.py index cc763709912bb5c0777edd2e2823280e2e019ba3..9567ea7f84e58ec5f221b4d69dfcb6651dc33cb5 100644 --- a/tests/fuzzy/test_masking.py +++ b/tests/fuzzy/test_masking.py @@ -8,7 +8,7 @@ import pandas as pd from hypothesis import given, settings from saqc.constants import * -from saqc.flagger import Flagger +from saqc.core import Flags as Flagger from saqc.core.register import _maskData, _unmaskData from tests.fuzzy.init import dataFieldFlagger, MAX_EXAMPLES