diff --git a/saqc/__init__.py b/saqc/__init__.py index a801471cfe57053267271de325717b35e8a0233f..254c456229e09fd27a3458284413f44db45aebd5 100644 --- a/saqc/__init__.py +++ b/saqc/__init__.py @@ -7,8 +7,25 @@ """The System for automated Quality Control package.""" +__all__ = [ + "BAD", + "DOUBTFUL", + "GOOD", + "UNFLAGGED", + "FILTER_ALL", + "FILTER_NONE", + "Flags", + "DictOfSeries", + "SaQC", + "DmpScheme", + "FloatScheme", + "PositionalScheme", + "SimpleScheme", + "AnnotatedFloatScheme", + "fromConfig", +] + from saqc.constants import BAD, DOUBTFUL, FILTER_ALL, FILTER_NONE, GOOD, UNFLAGGED -from saqc.exceptions import ParsingError from saqc.core import Flags, DictOfSeries, SaQC from saqc.core.translation import ( DmpScheme, @@ -18,8 +35,6 @@ from saqc.core.translation import ( AnnotatedFloatScheme, ) from saqc.parsing.reader import fromConfig -from saqc.version import __version__ - from . import _version __version__ = _version.get_versions()["version"] diff --git a/saqc/__main__.py b/saqc/__main__.py index d5da0036369e3b1561e06b2cd38df59e4e5f9071..be2263500d6952caeeed48a3627b0fb3887ed28a 100644 --- a/saqc/__main__.py +++ b/saqc/__main__.py @@ -136,10 +136,10 @@ def main( config = str(config) cr = _ConfigReader(data=data, scheme=scheme) if config.endswith("json"): - f = None - if json_field is not None: - f = lambda j: j[str(json_field)] - cr = cr.readJson(config, unpack=f) + cr = cr.readJson( + config, + unpack=lambda j: j[str(json_field)] if json_field is not None else None, + ) else: cr = cr.readCsv(config) diff --git a/saqc/core/__init__.py b/saqc/core/__init__.py index 1fbe7e27cf16682bce68df38479da20179647b19..bd5ac9dc696ce34eb69f3fe4427eb2caa5ef42aa 100644 --- a/saqc/core/__init__.py +++ b/saqc/core/__init__.py @@ -4,8 +4,19 @@ # -*- coding: utf-8 -*- # isort: skip_file + +__all__ = [ + "DictOfSeries", + "History", + "Flags", + "SaQC", + "flagging", + "processing", + "register", +] + from saqc.core.frame import DictOfSeries from saqc.core.history import History -from saqc.core.flags import Flags, initFlagsLike +from saqc.core.flags import Flags from saqc.core.register import flagging, processing, register from saqc.core.core import SaQC diff --git a/saqc/core/core.py b/saqc/core/core.py index c669da5672c4a2f8e4db344605bcb5fd5f24193c..952b2ed8cf9d6df9043b317bbb44c0f1c7adada5 100644 --- a/saqc/core/core.py +++ b/saqc/core/core.py @@ -12,7 +12,7 @@ import warnings from copy import copy as shallowcopy from copy import deepcopy from functools import partial -from typing import Any, Hashable, Iterable, MutableMapping +from typing import Any, Hashable, Iterable import numpy as np import pandas as pd @@ -332,7 +332,7 @@ class SaQC(FunctionsMixin): def _castData(self, data) -> DictOfSeries: if isinstance(data, pd.Series): if not isinstance(data.name, str): - raise ValueError(f"Cannot init from unnamed pd.Series") + raise ValueError("Cannot init from unnamed pd.Series") data = data.to_frame() if isinstance(data, pd.DataFrame): for idx in [data.index, data.columns]: diff --git a/saqc/core/flags.py b/saqc/core/flags.py index e597e0ee184483abf593c815cd375897a29e0fe4..fc1684193a29c4c9add225392b0f1eb0fcdfae70 100644 --- a/saqc/core/flags.py +++ b/saqc/core/flags.py @@ -7,7 +7,6 @@ from __future__ import annotations import typing -import warnings from typing import DefaultDict, Dict, Iterable, Mapping, Tuple, Type, Union, overload import numpy as np @@ -243,7 +242,7 @@ class Flags: if history.empty: return history - errm = f"History " + errm = "History " if colname: errm += f"of column {colname} " diff --git a/saqc/core/translation/__init__.py b/saqc/core/translation/__init__.py index 7549e43c599a7f838f710b03be734d8229707dda..1aaca2982f61bb95076a4c69caef890f9b889ecf 100644 --- a/saqc/core/translation/__init__.py +++ b/saqc/core/translation/__init__.py @@ -5,6 +5,16 @@ # SPDX-License-Identifier: GPL-3.0-or-later # -*- coding: utf-8 -*- +__all__ = [ + "MappingScheme", + "TranslationScheme", + "DmpScheme", + "AnnotatedFloatScheme", + "FloatScheme", + "PositionalScheme", + "SimpleScheme", +] + from saqc.core.translation.basescheme import MappingScheme, TranslationScheme from saqc.core.translation.dmpscheme import DmpScheme from saqc.core.translation.floatscheme import AnnotatedFloatScheme, FloatScheme diff --git a/saqc/core/translation/dmpscheme.py b/saqc/core/translation/dmpscheme.py index fb68713d525520bda5369a497139fe8bb13d2a39..f80267c93ee2314e94edbb153666ad14dea1c973 100644 --- a/saqc/core/translation/dmpscheme.py +++ b/saqc/core/translation/dmpscheme.py @@ -9,7 +9,6 @@ from __future__ import annotations import json -from functools import reduce import numpy as np import pandas as pd @@ -18,7 +17,6 @@ from saqc import BAD, DOUBTFUL, GOOD, UNFLAGGED from saqc.core import Flags, History from saqc.core.frame import DictOfSeries from saqc.core.translation.basescheme import BackwardMap, ForwardMap, MappingScheme -from saqc.lib.tools import getUnionIndex _QUALITY_CAUSES = [ "", diff --git a/saqc/funcs/changepoints.py b/saqc/funcs/changepoints.py index ff706f69167ec2e308057cf9eacfc3659d6141fd..b0ce725b3c0b1570c55c1f0ee8e7e8cb9a276f11 100644 --- a/saqc/funcs/changepoints.py +++ b/saqc/funcs/changepoints.py @@ -13,13 +13,11 @@ import numpy as np import pandas as pd from saqc import BAD, UNFLAGGED -from saqc.core import DictOfSeries, Flags, flagging, register +from saqc.core import flagging, register from saqc.lib.checking import ( - isInBounds, validateCallable, validateChoice, validateMinPeriods, - validateValueBounds, validateWindow, ) diff --git a/saqc/funcs/flagtools.py b/saqc/funcs/flagtools.py index d7ea5015a993c589355ae2c520a69ba4af2943fb..f57822e35fbc6e8a2ba44654677a22c3ff5202df 100644 --- a/saqc/funcs/flagtools.py +++ b/saqc/funcs/flagtools.py @@ -17,7 +17,6 @@ from typing_extensions import Literal from saqc import BAD, FILTER_ALL, UNFLAGGED from saqc.core import DictOfSeries, flagging, register -from saqc.core.flags import Flags from saqc.core.history import History from saqc.lib.checking import validateChoice, validateWindow from saqc.lib.tools import ( @@ -446,7 +445,7 @@ class FlagtoolsMixin: fields, targets, broadcasting = multivariateParameters(field, target) meta = { - "func": f"transferFlags", + "func": "transferFlags", "args": (), "kwargs": { "field": field, diff --git a/saqc/funcs/generic.py b/saqc/funcs/generic.py index ad8b200677ed7bd4b1f7a8b0bca9f7db1dc458ad..4973883dbd7e6b133b5bc32720d73357a852a5d0 100644 --- a/saqc/funcs/generic.py +++ b/saqc/funcs/generic.py @@ -267,7 +267,7 @@ class GenericMixin: ) if not result.empty and not isAllBoolean(result): - raise TypeError(f"generic expression does not return a boolean array") + raise TypeError("generic expression does not return a boolean array") # update flags & data for i, col in enumerate(targets): diff --git a/saqc/funcs/interpolation.py b/saqc/funcs/interpolation.py index 481c87d725d55acad95ca63b91413453953fc31a..65501df34c92c5305bb8dbb8bd7e93bcb6f436ad 100644 --- a/saqc/funcs/interpolation.py +++ b/saqc/funcs/interpolation.py @@ -7,7 +7,6 @@ # -*- coding: utf-8 -*- from __future__ import annotations -import warnings from typing import TYPE_CHECKING, Callable, Tuple, Union import numpy as np @@ -18,8 +17,6 @@ from saqc import UNFLAGGED from saqc.core import register from saqc.core.history import History from saqc.lib.checking import ( - isValidChoice, - validateCallable, validateChoice, validateFuncSelection, validateMinPeriods, diff --git a/saqc/funcs/noise.py b/saqc/funcs/noise.py index 7e18add8f6477fb6e57f4f3614331a402ead144d..db60bf214414db739b4180896606d2dbe8ddad60 100644 --- a/saqc/funcs/noise.py +++ b/saqc/funcs/noise.py @@ -13,16 +13,10 @@ from typing import TYPE_CHECKING, Callable, Literal import numpy as np import pandas as pd -from scipy.stats import median_abs_deviation from saqc.constants import BAD from saqc.core.register import flagging -from saqc.lib.checking import ( - isCallable, - validateFuncSelection, - validateMinPeriods, - validateWindow, -) +from saqc.lib.checking import validateFuncSelection, validateMinPeriods, validateWindow from saqc.lib.tools import isunflagged, statPass from saqc.parsing.environ import ENV_OPERATORS diff --git a/saqc/funcs/outliers.py b/saqc/funcs/outliers.py index 5ea2e2ddaebd91074d58b427a05b762538825ef8..ab878240208b6880260dd8baa8227d18e022b1a0 100644 --- a/saqc/funcs/outliers.py +++ b/saqc/funcs/outliers.py @@ -10,7 +10,7 @@ from __future__ import annotations import uuid import warnings -from typing import TYPE_CHECKING, Callable, List, Optional, Sequence, Tuple +from typing import TYPE_CHECKING, Callable, Sequence, Tuple import numpy as np import numpy.polynomial.polynomial as poly @@ -27,7 +27,6 @@ from saqc.lib.checking import ( validateChoice, validateFraction, validateFrequency, - validateFuncSelection, validateMinPeriods, validateValueBounds, validateWindow, diff --git a/saqc/funcs/resampling.py b/saqc/funcs/resampling.py index 168d16fc0f41b803c281974e2b9d26b38185cb2d..fb803e26c9875a703e443da4e3b0d267061e12a4 100644 --- a/saqc/funcs/resampling.py +++ b/saqc/funcs/resampling.py @@ -7,11 +7,10 @@ # -*- coding: utf-8 -*- from __future__ import annotations -import re import typing import uuid import warnings -from typing import TYPE_CHECKING, Callable, Union +from typing import TYPE_CHECKING, Callable import numpy as np import pandas as pd @@ -21,10 +20,9 @@ import saqc.constants from saqc.constants import UNFLAGGED from saqc.core import register from saqc.core.history import History -from saqc.funcs.interpolation import DATA_REINDEXER from saqc.lib.checking import validateChoice, validateFuncSelection from saqc.lib.docs import DOC_TEMPLATES -from saqc.lib.tools import filterKwargs, getFreqDelta +from saqc.lib.tools import getFreqDelta from saqc.lib.ts_operators import isValid if TYPE_CHECKING: @@ -288,10 +286,10 @@ class ResamplingMixin: done = True if len(stack) == 0: - raise ValueError(f"Could not find no last reindexing to invert") + raise ValueError("Could not find no last reindexing to invert") reindex_method = METHODINVERTS.get(stack[-1], False) - if reindex_method == False: + if reindex_method is False: raise ValueError(f"cant invert {stack[-1]}") return reindex_method diff --git a/saqc/funcs/residuals.py b/saqc/funcs/residuals.py index 5ef4a6a409168bb8b05cee3075921de0f1c48ee5..0ba870798135b93a385e7e04510956532c9bd732 100644 --- a/saqc/funcs/residuals.py +++ b/saqc/funcs/residuals.py @@ -7,7 +7,7 @@ # -*- coding: utf-8 -*- from __future__ import annotations -from typing import TYPE_CHECKING, Callable, Optional, Union +from typing import TYPE_CHECKING, Callable import numpy as np import pandas as pd diff --git a/saqc/funcs/rolling.py b/saqc/funcs/rolling.py index dfcdfd23cd747f59b6ccece0a89ffd2e847d2fd3..77a9d9985c458bdae31109be4a978e241b2bb193 100644 --- a/saqc/funcs/rolling.py +++ b/saqc/funcs/rolling.py @@ -14,12 +14,7 @@ import pandas as pd import saqc from saqc.core import DictOfSeries, Flags, register -from saqc.lib.checking import ( - validateCallable, - validateFuncSelection, - validateMinPeriods, - validateWindow, -) +from saqc.lib.checking import validateFuncSelection, validateMinPeriods, validateWindow from saqc.lib.tools import getFreqDelta if TYPE_CHECKING: diff --git a/saqc/funcs/scores.py b/saqc/funcs/scores.py index f72e2362f822d4d65d49f6d7332e5d83dd2cb483..aacc76eb33399decb0bee9869893ee94ca8c893e 100644 --- a/saqc/funcs/scores.py +++ b/saqc/funcs/scores.py @@ -7,14 +7,13 @@ # -*- coding: utf-8 -*- from __future__ import annotations -from typing import TYPE_CHECKING, Callable, Optional, Sequence, Tuple, Union +from typing import TYPE_CHECKING, Callable, Optional, Sequence, Tuple import numpy as np import pandas as pd from sklearn.neighbors import LocalOutlierFactor from typing_extensions import Literal -from saqc import UNFLAGGED from saqc.core import register from saqc.lib.checking import ( validateChoice, @@ -25,7 +24,7 @@ from saqc.lib.checking import ( from saqc.lib.docs import DOC_TEMPLATES from saqc.lib.tools import getApply, toSequence from saqc.lib.ts_operators import kNN -from saqc.parsing.environ import ENV_OPERATORS, ENV_TRAFOS +from saqc.parsing.environ import ENV_OPERATORS if TYPE_CHECKING: from saqc import SaQC diff --git a/saqc/funcs/tools.py b/saqc/funcs/tools.py index 453d28d882350363bbae52dece9085d981f82f08..5038df6294c09868959cb61b18c0a9a3061b4c0b 100644 --- a/saqc/funcs/tools.py +++ b/saqc/funcs/tools.py @@ -9,7 +9,6 @@ from __future__ import annotations import pickle import tkinter as tk -import warnings from typing import TYPE_CHECKING import matplotlib as mpl diff --git a/saqc/funcs/transformation.py b/saqc/funcs/transformation.py index 2b2d6db440895a949939068acb7395abd6d1786a..cfc4d226aaf6d5d95181ef2a0dc4149a076e087d 100644 --- a/saqc/funcs/transformation.py +++ b/saqc/funcs/transformation.py @@ -7,7 +7,7 @@ # -*- coding: utf-8 -*- from __future__ import annotations -from typing import TYPE_CHECKING, Callable, Optional, Union +from typing import TYPE_CHECKING, Callable import numpy as np import pandas as pd diff --git a/saqc/lib/plotting.py b/saqc/lib/plotting.py index 5dd0e944b93d93e1e0b15ddd723bb88d0b4fd5e3..41a5b5a53f0a1df379c396bd8fa11309c30a5c7b 100644 --- a/saqc/lib/plotting.py +++ b/saqc/lib/plotting.py @@ -9,7 +9,6 @@ from __future__ import annotations import itertools -from collections import OrderedDict import matplotlib as mpl import matplotlib.pyplot as plt diff --git a/saqc/lib/tools.py b/saqc/lib/tools.py index 72c201c8f36a6afa1393edfe0082472eccd1b6cd..7fd671036660d222aa59dba05b6197a212f9285d 100644 --- a/saqc/lib/tools.py +++ b/saqc/lib/tools.py @@ -442,7 +442,7 @@ def getApply(in_obj, apply_obj, attr_access="__name__", attr_or="apply") -> pd.S try: # let's try to run it somewhat optimized out = getattr(in_obj, attr_or)(apply_obj, raw=True) - except: + except Exception: # did't work out, fallback out = getattr(in_obj, attr_or)(apply_obj) diff --git a/saqc/lib/ts_operators.py b/saqc/lib/ts_operators.py index c983a314f87bd2bf373fdc8dc7b821b2ea1bf6b9..4b0c4ad44f4b77d34a4f10d9c812bd99a3eb5e9e 100644 --- a/saqc/lib/ts_operators.py +++ b/saqc/lib/ts_operators.py @@ -9,10 +9,7 @@ """ The module gathers all kinds of timeseries tranformations. """ -import re import sys -import warnings -from typing import Literal, Union import numpy as np import numpy.polynomial.polynomial as poly diff --git a/saqc/lib/types.py b/saqc/lib/types.py index 8b5b6e5fa77b1e1c0f443f9bc50abdb694362808..c5e784527dc43769731cec124753110be62d7f51 100644 --- a/saqc/lib/types.py +++ b/saqc/lib/types.py @@ -8,20 +8,12 @@ from __future__ import annotations import abc -from typing import Any, Dict, TypeVar, Union +from typing import TypeVar, Union import numpy as np import pandas as pd from typing_extensions import Protocol -__all__ = [ - "T", - "ArrayLike", - "CurveFitter", - "ExternalFlag", - "OptionalNone", -] - T = TypeVar("T") ArrayLike = TypeVar("ArrayLike", np.ndarray, pd.Series, pd.DataFrame) diff --git a/saqc/parsing/environ.py b/saqc/parsing/environ.py index a3095b36385703cea57e300a16b69be3e03c0dc1..d494ed666108df125ee93be3a04255c6c589e0ce 100644 --- a/saqc/parsing/environ.py +++ b/saqc/parsing/environ.py @@ -7,8 +7,6 @@ # -*- coding: utf-8 -*- import numpy as np -import pandas as pd -import scipy.stats as st import saqc.lib.ts_operators as ts_ops from saqc import BAD, DOUBTFUL, FILTER_ALL, FILTER_NONE, GOOD, UNFLAGGED diff --git a/tests/core/test_core.py b/tests/core/test_core.py index 4109eb6e4a633786752c0b5272088b69751f9d3d..b5062f33f86e7466c4877d5d98f2ed7d18aafcb2 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -13,7 +13,8 @@ import pandas as pd import pytest from saqc import BAD, FILTER_ALL, FILTER_NONE, UNFLAGGED, SaQC -from saqc.core import DictOfSeries, Flags, flagging, initFlagsLike, processing, register +from saqc.core import DictOfSeries, Flags, flagging, processing, register +from saqc.core.flags import initFlagsLike from saqc.lib.types import OptionalNone from tests.common import initData diff --git a/tests/funcs/test_constants_detection.py b/tests/funcs/test_constants_detection.py index 8cbf3290d58b2b76f80c8255aa874533b669a3cd..a6ec797929f933e413e4c7aa9c6478259914147a 100644 --- a/tests/funcs/test_constants_detection.py +++ b/tests/funcs/test_constants_detection.py @@ -9,8 +9,8 @@ import numpy as np import pytest -from saqc import BAD, UNFLAGGED -from saqc.core import SaQC, initFlagsLike +from saqc import BAD, UNFLAGGED, SaQC +from saqc.core.flags import initFlagsLike from tests.common import initData diff --git a/tests/funcs/test_functions.py b/tests/funcs/test_functions.py index d956e39b1a1fb6532b328bd93672966c8cc41c55..8a3e14a53d4e57a04672a6c3fc93f7358d04e01b 100644 --- a/tests/funcs/test_functions.py +++ b/tests/funcs/test_functions.py @@ -10,9 +10,8 @@ import numpy as np import pandas as pd import pytest -import saqc -from saqc import BAD, DOUBTFUL, UNFLAGGED, SaQC -from saqc.core import DictOfSeries, initFlagsLike +from saqc import BAD, DOUBTFUL, UNFLAGGED, DictOfSeries, SaQC +from saqc.core.flags import initFlagsLike from tests.common import initData from tests.fixtures import char_dict, course_1 # noqa, todo: fix fixtures diff --git a/tests/funcs/test_generic_config_functions.py b/tests/funcs/test_generic_config_functions.py index f19342a39a2d71421564d076c938d4197644cba0..d9af808da9f91e2387c5332e136aa1f1b1b0caca 100644 --- a/tests/funcs/test_generic_config_functions.py +++ b/tests/funcs/test_generic_config_functions.py @@ -12,8 +12,9 @@ import numpy as np import pandas as pd import pytest -from saqc import BAD, UNFLAGGED, SaQC -from saqc.core import DictOfSeries, Flags, initFlagsLike, register +from saqc import BAD, UNFLAGGED, DictOfSeries, Flags, SaQC +from saqc.core import register +from saqc.core.flags import initFlagsLike from saqc.funcs.generic import _execGeneric from saqc.parsing.reader import _ConfigReader from saqc.parsing.visitor import ConfigFunctionParser diff --git a/tests/funcs/test_outlier_detection.py b/tests/funcs/test_outlier_detection.py index 81bd0ec287834f7d28a0bd85834064bc165c3c5b..61953bcd09e12826ecbce8db56257905c05938a5 100644 --- a/tests/funcs/test_outlier_detection.py +++ b/tests/funcs/test_outlier_detection.py @@ -13,8 +13,8 @@ import pandas as pd import pytest import saqc -from saqc import BAD, UNFLAGGED -from saqc.core import DictOfSeries, SaQC, initFlagsLike +from saqc import BAD, UNFLAGGED, DictOfSeries, SaQC +from saqc.core.flags import initFlagsLike from tests.fixtures import char_dict, course_1, course_2, course_3, course_4 diff --git a/tests/funcs/test_pattern_rec.py b/tests/funcs/test_pattern_rec.py index 6b96ee0ca5b6a44a962322eb98d7ffb54fdbf20f..b2be3a6348c017ab3a789c59d0d53b914bd77867 100644 --- a/tests/funcs/test_pattern_rec.py +++ b/tests/funcs/test_pattern_rec.py @@ -9,8 +9,8 @@ import pandas as pd import pytest -from saqc import BAD, UNFLAGGED, SaQC -from saqc.core import DictOfSeries, initFlagsLike +from saqc import BAD, UNFLAGGED, DictOfSeries, SaQC +from saqc.core.flags import initFlagsLike from tests.common import initData diff --git a/tests/funcs/test_proc_functions.py b/tests/funcs/test_proc_functions.py index df960969b062e7a76e2a53e938f0200843d62ebf..b2f097060d982c31e516af49925d21467f9745fa 100644 --- a/tests/funcs/test_proc_functions.py +++ b/tests/funcs/test_proc_functions.py @@ -14,8 +14,8 @@ import pandas as pd import pytest import saqc -from saqc import UNFLAGGED, SaQC -from saqc.core import DictOfSeries, initFlagsLike +from saqc import UNFLAGGED, DictOfSeries, SaQC +from saqc.core.flags import initFlagsLike from saqc.lib.ts_operators import linearInterpolation, polynomialInterpolation from tests.fixtures import char_dict, course_3, course_5 # noqa, todo: fix fixtures diff --git a/tests/funcs/test_resampling.py b/tests/funcs/test_resampling.py index b765c074f64f44dfe2ee48ed85021a1b791d59be..f077c283b7314b89826dcc37bf79fe0ede3d18c8 100644 --- a/tests/funcs/test_resampling.py +++ b/tests/funcs/test_resampling.py @@ -14,7 +14,7 @@ import pandas as pd import pytest from saqc import SaQC -from saqc.core import initFlagsLike +from saqc.core.flags import initFlagsLike from saqc.funcs.resampling import ( _aggregationGrouper, _constructAggregationReindexer, diff --git a/tests/fuzzy/lib.py b/tests/fuzzy/lib.py index a70a159f3ad1fede24e2c295ce74d6a672a8ffb4..e0b30cac99acce5eb77dd068d4bb956511290f67 100644 --- a/tests/fuzzy/lib.py +++ b/tests/fuzzy/lib.py @@ -23,8 +23,8 @@ from hypothesis.strategies import ( ) from hypothesis.strategies._internal.types import _global_type_lookup -from saqc import BAD -from saqc.core import DictOfSeries, initFlagsLike +from saqc import BAD, DictOfSeries +from saqc.core.flags import initFlagsLike from saqc.core.register import FUNC_MAP MAX_EXAMPLES = 50