Merge branch 'docDecorator' into 'develop'

Doc decorator

See merge request !420

CHANGELOG.md

@@ -13,7 +13,9 @@ This changelog starts with version 2.0.0. Basically all parts of the system, inc
 [List of commits](https://git.ufz.de/rdm-software/saqc/-/compare/v2.0.1...develop)
 
 ### Added
+- generic documentation module `docurator.py` added to `lib`
 ### Changed
+- documentation pipeline changed to base on methods decorators
 - `flagOffsets` parameters `thresh` and `thresh_relative` now both are optional
 ### Removed
 ### Fixed

saqc/core/modules/breaks.py

@@ -9,14 +9,18 @@ from __future__ import annotations
 
 from saqc.constants import BAD, FILTER_ALL
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Breaks:
+    @doc(saqc.funcs.breaks.flagMissing.__doc__)
     def flagMissing(
         self, field: str, flag: float = BAD, dfilter: float = FILTER_ALL, **kwargs
     ) -> saqc.SaQC:
         return self._defer("flagMissing", locals())
 
+    @doc(saqc.funcs.breaks.flagIsolated.__doc__)
     def flagIsolated(
         self,
         field: str,
@@ -27,6 +31,7 @@ class Breaks:
     ) -> saqc.SaQC:
         return self._defer("flagIsolated", locals())
 
+    @doc(saqc.funcs.breaks.flagJumps.__doc__)
     def flagJumps(
         self,
         field: str,

saqc/core/modules/changepoints.py

@@ -14,9 +14,12 @@ from typing_extensions import Literal
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class ChangePoints:
+    @doc(saqc.funcs.changepoints.flagChangePoints.__doc__)
     def flagChangePoints(
         self,
         field: str,
@@ -32,6 +35,7 @@ class ChangePoints:
     ) -> saqc.SaQC:
         return self._defer("flagChangePoints", locals())
 
+    @doc(saqc.funcs.changepoints.assignChangePointCluster.__doc__)
     def assignChangePointCluster(
         self,
         field: str,

saqc/core/modules/constants.py

@@ -9,9 +9,12 @@ from __future__ import annotations
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Constants:
+    @doc(saqc.funcs.constants.flagByVariance.__doc__)
     def flagByVariance(
         self,
         field: str,
@@ -24,6 +27,7 @@ class Constants:
     ) -> saqc.SaQC:
         return self._defer("flagByVariance", locals())
 
+    @doc(saqc.funcs.constants.flagConstants.__doc__)
     def flagConstants(
         self, field: str, thresh: float, window: str, flag: float = BAD, **kwargs
     ) -> saqc.SaQC:

saqc/core/modules/curvefit.py

@@ -13,9 +13,12 @@ from typing_extensions import Literal
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Curvefit:
+    @doc(saqc.funcs.curvefit.fitPolynomial.__doc__)
     def fitPolynomial(
         self,
         field: str,

saqc/core/modules/drift.py

@@ -17,9 +17,12 @@ from saqc.constants import BAD
 import saqc
 from saqc.funcs import LinkageString
 from saqc.lib.types import CurveFitter
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Drift:
+    @doc(saqc.funcs.drift.flagDriftFromNorm.__doc__)
     def flagDriftFromNorm(
         self,
         field: Sequence[str],
@@ -36,6 +39,7 @@ class Drift:
     ) -> saqc.SaQC:
         return self._defer("flagDriftFromNorm", locals())
 
+    @doc(saqc.funcs.drift.flagDriftFromReference.__doc__)
     def flagDriftFromReference(
         self,
         field: Sequence[str],
@@ -51,6 +55,7 @@ class Drift:
     ) -> saqc.SaQC:
         return self._defer("flagDriftFromReference", locals())
 
+    @doc(saqc.funcs.drift.correctDrift.__doc__)
     def correctDrift(
         self,
         field: str,
@@ -61,6 +66,7 @@ class Drift:
     ) -> saqc.SaQC:
         return self._defer("correctDrift", locals())
 
+    @doc(saqc.funcs.drift.correctRegimeAnomaly.__doc__)
     def correctRegimeAnomaly(
         self,
         field: str,
@@ -72,6 +78,7 @@ class Drift:
     ) -> saqc.SaQC:
         return self._defer("correctRegimeAnomaly", locals())
 
+    @doc(saqc.funcs.drift.correctOffset.__doc__)
     def correctOffset(
         self,
         field: str,

saqc/core/modules/flagtools.py

@@ -16,21 +16,28 @@ from typing_extensions import Literal
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class FlagTools:
+    @doc(saqc.funcs.flagtools.clearFlags.__doc__)
     def clearFlags(self, field: str, **kwargs) -> saqc.SaQC:
         return self._defer("clearFlags", locals())
 
+    @doc(saqc.funcs.flagtools.forceFlags.__doc__)
     def forceFlags(self, field: str, flag: float = BAD, **kwargs) -> saqc.SaQC:
         return self._defer("forceFlags", locals())
 
+    @doc(saqc.funcs.flagtools.forceFlags.__doc__)
     def flagDummy(self, field: str, **kwargs) -> saqc.SaQC:
         return self._defer("flagDummy", locals())
 
+    @doc(saqc.funcs.flagtools.flagUnflagged.__doc__)
     def flagUnflagged(self, field: str, flag: float = BAD, **kwargs) -> saqc.SaQC:
         return self._defer("flagUnflagged", locals())
 
+    @doc(saqc.funcs.flagtools.flagManual.__doc__)
     def flagManual(
         self,
         field: str,
@@ -45,6 +52,7 @@ class FlagTools:
     ) -> saqc.SaQC:
         return self._defer("flagManual", locals())
 
+    @doc(saqc.funcs.flagtools.transferFlags.__doc__)
     def transferFlags(
         self,
         field: str | Sequence[str],

saqc/core/modules/generic.py

@@ -12,9 +12,12 @@ from typing import Sequence, Union
 import saqc
 from saqc.constants import UNFLAGGED, BAD, FILTER_ALL
 from saqc.lib.types import GenericFunction
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Generic:
+    @doc(saqc.funcs.generic.processGeneric.__doc__)
     def processGeneric(
         self,
         field: str | Sequence[str],
@@ -26,6 +29,7 @@ class Generic:
     ) -> saqc.SaQC:
         return self._defer("processGeneric", locals())
 
+    @doc(saqc.funcs.generic.flagGeneric.__doc__)
     def flagGeneric(
         self,
         field: Union[str, Sequence[str]],

saqc/core/modules/interpolation.py

@@ -15,9 +15,12 @@ import pandas as pd
 from saqc.constants import UNFLAGGED
 import saqc
 from saqc.funcs.interpolation import _SUPPORTED_METHODS
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Interpolation:
+    @doc(saqc.funcs.interpolation.interpolateByRolling.__doc__)
     def interpolateByRolling(
         self,
         field: str,
@@ -30,6 +33,7 @@ class Interpolation:
     ) -> saqc.SaQC:
         return self._defer("interpolateByRolling", locals())
 
+    @doc(saqc.funcs.interpolation.interpolateInvalid.__doc__)
     def interpolateInvalid(
         self,
         field: str,
@@ -42,6 +46,7 @@ class Interpolation:
     ) -> saqc.SaQC:
         return self._defer("interpolateInvalid", locals())
 
+    @doc(saqc.funcs.interpolation.interpolateIndex.__doc__)
     def interpolateIndex(
         self,
         field: str,

saqc/core/modules/noise.py

@@ -13,9 +13,12 @@ from typing import Callable
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Noise:
+    @doc(saqc.funcs.noise.flagByStatLowPass.__doc__)
     def flagByStatLowPass(
         self,
         field: str,

saqc/core/modules/outliers.py

@@ -15,9 +15,12 @@ from typing_extensions import Literal
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Outliers:
+    @doc(saqc.funcs.outliers.flagByStray.__doc__)
     def flagByStray(
         self,
         field: str,
@@ -30,6 +33,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagByStray", locals())
 
+    @doc(saqc.funcs.outliers.flagMVScores.__doc__)
     def flagMVScores(
         self,
         field: Sequence[str],
@@ -49,6 +53,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagMVScores", locals())
 
+    @doc(saqc.funcs.outliers.flagRaise.__doc__)
     def flagRaise(
         self,
         field: str,
@@ -64,6 +69,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagRaise", locals())
 
+    @doc(saqc.funcs.outliers.flagMAD.__doc__)
     def flagMAD(
         self,
         field: str,
@@ -74,6 +80,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagMAD", locals())
 
+    @doc(saqc.funcs.outliers.flagOffset.__doc__)
     def flagOffset(
         self,
         field: str,
@@ -86,6 +93,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagOffset", locals())
 
+    @doc(saqc.funcs.outliers.flagByGrubbs.__doc__)
     def flagByGrubbs(
         self,
         field: str,
@@ -98,6 +106,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagByGrubbs", locals())
 
+    @doc(saqc.funcs.outliers.flagRange.__doc__)
     def flagRange(
         self,
         field: str,
@@ -108,6 +117,7 @@ class Outliers:
     ) -> saqc.SaQC:
         return self._defer("flagRange", locals())
 
+    @doc(saqc.funcs.outliers.flagCrossStatistics.__doc__)
     def flagCrossStatistics(
         self,
         field: Sequence[str],

saqc/core/modules/pattern.py

@@ -9,9 +9,12 @@ from __future__ import annotations
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Pattern:
+    @doc(saqc.funcs.pattern.flagPatternByDTW.__doc__)
     def flagPatternByDTW(
         self,
         field,

saqc/core/modules/resampling.py

@@ -16,9 +16,12 @@ from typing_extensions import Literal
 from saqc.constants import BAD
 import saqc
 from saqc.funcs.interpolation import _SUPPORTED_METHODS
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Resampling:
+    @doc(saqc.funcs.resampling.linear.__doc__)
     def linear(
         self,
         field: str,
@@ -27,6 +30,7 @@ class Resampling:
     ) -> saqc.SaQC:
         return self._defer("linear", locals())
 
+    @doc(saqc.funcs.resampling.interpolate.__doc__)
     def interpolate(
         self,
         field: str,
@@ -37,6 +41,7 @@ class Resampling:
     ) -> saqc.SaQC:
         return self._defer("interpolate", locals())
 
+    @doc(saqc.funcs.resampling.shift.__doc__)
     def shift(
         self,
         field: str,
@@ -47,6 +52,7 @@ class Resampling:
     ) -> saqc.SaQC:
         return self._defer("shift", locals())
 
+    @doc(saqc.funcs.resampling.resample.__doc__)
     def resample(
         self,
         field: str,
@@ -63,6 +69,7 @@ class Resampling:
     ) -> saqc.SaQC:
         return self._defer("resample", locals())
 
+    @doc(saqc.funcs.resampling.concatFlags.__doc__)
     def concatFlags(
         self,
         field: str,

saqc/core/modules/residues.py

@@ -15,9 +15,12 @@ from typing_extensions import Literal
 
 from saqc.constants import BAD
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Residues:
+    @doc(saqc.funcs.residues.calculatePolynomialResidues.__doc__)
     def calculatePolynomialResidues(
         self,
         field: str,
@@ -28,6 +31,7 @@ class Residues:
     ) -> saqc.SaQC:
         return self._defer("calculatePolynomialResidues", locals())
 
+    @doc(saqc.funcs.residues.calculateRollingResidues.__doc__)
     def calculateRollingResidues(
         self,
         field: str,

saqc/core/modules/rolling.py

@@ -12,9 +12,12 @@ import numpy as np
 import pandas as pd
 
 from saqc.constants import BAD
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Rolling:
+    @doc(saqc.funcs.rolling.roll.__doc__)
     def roll(
         self,
         field: str,

saqc/core/modules/scores.py

@@ -14,9 +14,12 @@ import pandas as pd
 from typing_extensions import Literal
 
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Scores:
+    @doc(saqc.funcs.scores.assignKNNScore.__doc__)
     def assignKNNScore(
         self,
         field: Sequence[str],

saqc/core/modules/tools.py

@@ -14,20 +14,26 @@ import saqc
 import numpy as np
 
 from saqc.constants import FILTER_NONE
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Tools:
+    @doc(saqc.funcs.tools.copyField.__doc__)
     def copyField(
         self, field: str, target: str, overwrite: bool = False, **kwargs
     ) -> saqc.SaQC:
         return self._defer("copyField", locals())
 
+    @doc(saqc.funcs.tools.dropField.__doc__)
     def dropField(self, field: str, **kwargs) -> saqc.SaQC:
         return self._defer("dropField", locals())
 
+    @doc(saqc.funcs.tools.renameField.__doc__)
     def renameField(self, field: str, new_name: str, **kwargs) -> saqc.SaQC:
         return self._defer("renameField", locals())
 
+    @doc(saqc.funcs.tools.maskTime.__doc__)
     def maskTime(
         self,
         field: str,
@@ -40,6 +46,7 @@ class Tools:
     ) -> saqc.SaQC:
         return self._defer("maskTime", locals())
 
+    @doc(saqc.funcs.tools.plot.__doc__)
     def plot(
         self,
         field: str,

saqc/core/modules/transformation.py

@@ -12,9 +12,12 @@ from typing import Callable, Optional, Union
 import pandas as pd
 
 import saqc
+from saqc.lib.docurator import doc
+import saqc.funcs
 
 
 class Transformation:
+    @doc(saqc.funcs.transformation.transform.__doc__)
     def transform(
         self,
         field: str,

saqc/lib/docurator.py

+# SPDX-FileCopyrightText: 2021 Helmholtz-Zentrum für Umweltforschung GmbH - UFZ
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+import re
+
+FUNC_NAPOLEAN_STYLE_ORDER = [
+    "Head",
+    "Parameters",
+    "Returns",
+    "Notes",
+    "See also",
+    "Examples",
+    "References",
+]
+
+
+def doc(doc_string: str, template="saqc_methods", source="function_string"):
+    def docFunc(meth):
+        if template == "saqc_methods":
+            meth.__doc__ = saqcMethodsTemplate(doc_string, source)
+        return meth
+
+    return docFunc
+
+
+def getDocstringIndent(doc_string: list) -> str:
+    """returns a whitespace string matching the indent size of the passed docstring_list"""
+    regular_line = False
+    current_line = 0
+    while not regular_line:
+        # check if line is empty
+        if len(doc_string[current_line]) == 0 or re.match(
+            " *$", doc_string[current_line]
+        ):
+            current_line += 1
+        else:
+            regular_line = True
+    # get indent-string (smth. like "   ")
+    indent_str = re.match(" *", doc_string[current_line])[0]
+    return indent_str
+
+
+def getSections(doc_string: list, indent_str: str) -> dict:
+    """Returns a dictionary of sections, with section names as keys"""
+    section_lines = [0]
+    section_headings = ["Head"]
+    for k in range(len(doc_string) - 1):
+        # check if next line is an underscore line (section signator):
+        if re.match(indent_str + "-+$", doc_string[k + 1]):
+            # check if underscore length matches heading length
+            if len(doc_string[k + 1]) == len(doc_string[k]):
+                section_lines.append(k)
+                # skip leading whitespaces
+                skip = re.match("^ *", doc_string[k]).span()[-1]
+                section_headings.append(doc_string[k][skip:])
+    section_lines.append(len(doc_string))
+    section_content = [
+        doc_string[section_lines[k] : section_lines[k + 1]]
+        for k in range(len(section_lines) - 1)
+    ]
+    section_content = [clearTrailingWhitespace(p) for p in section_content]
+    sections = dict(zip(section_headings, section_content))
+    return sections
+
+
+def getParameters(section: list, indent_str: str) -> dict:
+    """Returns a dictionary of Parameter documentations, with parameter names as keys"""
+    parameter_lines = []
+    parameter_names = []
+    for k in range(len(section)):
+        # try catch a parameter definition start (implicitly assuming parameter names have no
+        # whitespaces):
+        param = re.match(indent_str + r"(\S+) *:", section[k])
+        if param:
+            parameter_lines.append(k)
+            parameter_names.append(param.group(1))
+
+    parameter_lines.append(len(section))
+    parameter_content = [
+        section[parameter_lines[k] : parameter_lines[k + 1]]
+        for k in range(len(parameter_lines) - 1)
+    ]
+    parameter_content = [clearTrailingWhitespace(p) for p in parameter_content]
+    parameter_dict = dict(zip(parameter_names, parameter_content))
+    return parameter_dict
+
+
+def mkParameter(
+    parameter_name: str, parameter_type: str, parameter_doc: str, indent_str: str
+) -> dict:
+    parameter_doc = parameter_doc.splitlines()
+    parameter_doc = [indent_str + " " * 4 + p for p in parameter_doc]
+    content = [indent_str + f"{parameter_name} : {parameter_type}"]
+    content += parameter_doc
+    return {parameter_name: content}
+
+
+def makeSection(section_name: str, indent_str: str, doc_content: str = None) -> dict:
+    content = [indent_str + section_name]
+    content += [indent_str + "_" * len(section_name)]
+    content += [" "]
+    if doc_content:
+        content += doc_content.splitlines()
+
+    return {section_name: content}
+
+
+def composeDocstring(
+    section_dict: dict, order: list = FUNC_NAPOLEAN_STYLE_ORDER
+) -> str:
+    """Compose final docstring from a sections dictionary"""
+    doc_string = []
+    section_dict = section_dict.copy()
+    for sec in order:
+        dc = section_dict.pop(sec, [])
+        doc_string += dc
+        # blank line at section end
+        if len(dc) > 0:
+            doc_string += [""]
+
+    return "\n".join(doc_string)
+
+
+def clearTrailingWhitespace(doc: list) -> list:
+    """Clears trailing whitespace lines"""
+    for k in range(len(doc), 0, -1):
+        if not re.match(r"^\s*$", doc[k - 1]):
+            break
+    return doc[:k]
+
+
+def saqcMethodsTemplate(doc_string: str, source="function_string"):
+    if source == "function_string":
+        doc_string = doc_string.splitlines()
+        indent_string = getDocstringIndent(doc_string)
+        sections = getSections(doc_string, indent_str=indent_string)
+        sections.pop("Returns", None)
+        returns_section = makeSection(section_name="Returns", indent_str=indent_string)
+        out_para = mkParameter(
+            parameter_name="out",
+            parameter_type="saqc.SaQC",
+            parameter_doc="An :py:meth:`saqc.SaQC` object, holding the (possibly) modified data",
+            indent_str=indent_string,
+        )
+        returns_section["Returns"] += out_para["out"]
+        sections.update(returns_section)
+        doc_string = composeDocstring(
+            section_dict=sections, order=FUNC_NAPOLEAN_STYLE_ORDER
+        )
+    return doc_string

sphinxdoc/Makefile

@@ -34,35 +34,25 @@ clean:
 
 # make doctest, make documentation, make clean
 doc:
-	# generate parent fake module for the functions to be documented
-	python scripts/make_doc_module.py -p "saqc/funcs" -sr ".." -su "funcSummaries"
 	# generate environment table from dictionary
 	python scripts/make_env_tab.py
 	@$(SPHINXBUILD) -M doctest "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 	@$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-	python scripts/modify_html_API.py
 	rm -f *.automodsumm
 	rm -f *.automodapi
 	rm -f moduleAPIs/*.automodsumm
 	rm -f moduleAPIs/*.automodapi
 	rm -f */*.automodsumm
-	rm -f -r coredoc
 
 # make documentation
 doconly:
-	# generate parent fake module for the functions to be documented
-	python scripts/make_doc_module.py -p "saqc/funcs" -sr ".." -su "funcSummaries"
 	# generate environment table from dictionary
 	python scripts/make_env_tab.py
 	@$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-	python scripts/modify_html_API.py
-
 
 # make test, clean up
 testonly:
     # generate parent fake module for the functions to be documented
-	python scripts/make_doc_module.py -p "saqc/funcs" -sr ".." -su "funcSummaries"
-    # generate environment table from dictionary
 	python scripts/make_env_tab.py
 	@$(SPHINXBUILD) -M doctest "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 	rm -f *.automodsumm