Skip to content
Snippets Groups Projects
Commit f3efc1c0 authored by Bert Palm's avatar Bert Palm 🎇
Browse files

last fixes, brought pages back online

parent f9378ad2
No related branches found
No related tags found
1 merge request!231new Flagger and Stuff
Hide whitespace changes
Inline Side-by-side
Showing
with 71 additions and 93 deletions
README.md
+ 4
− 5
View file @ f3efc1c0
......@@ -81,17 +81,16 @@ The following snippet implements the same configuration given above through
the Python-API:
```python
import saqc.funcs.outliers
from saqc import SaQC, SimpleFlagger
import numpy as np
from saqc import SaQC
saqc = saqc = (SaQC(SimpleFlagger(), data)
saqc = (SaQC(data)
.shiftToFreq("SM2", freq="15Min")
.flagMissing("SM2", nodata=np.nan)
.flagRange("SM(1|2)+", regex=True, min=10, max=60)
.flagMad("SM2", window="30d", z=3.5))
data, flagger = saqc.getResult()
data, flags = saqc.getResult()
```
## Installation
......
compatFlagger_only.patch deleted 100644 → 0
+ 0
− 26
View file @ f9378ad2
diff --git a/test/common.py b/test/common.py
index d5867e94..172d8edd 100644
--- a/test/common.py
+++ b/test/common.py
@@ -12,6 +12,7 @@ from saqc.flagger import (
CategoricalFlagger,
SimpleFlagger,
DmpFlagger,
+ CompatFlagger,
)
@@ -19,9 +20,10 @@ TESTNODATA = (np.nan, -9999)
TESTFLAGGER = (
- CategoricalFlagger(["NIL", "GOOD", "BAD"]),
- SimpleFlagger(),
- DmpFlagger(),
+ # CategoricalFlagger(["NIL", "GOOD", "BAD"]),
+ # SimpleFlagger(),
+ # DmpFlagger(),
+ CompatFlagger(),
)
def flagAll(data, field, flagger, **kwargs):
sphinx-doc/Makefile
+ 1
− 1
View file @ f3efc1c0
......@@ -20,6 +20,7 @@ clean:
rm -f *.automodsumm
rm -f func_modules/*.automodsumm
rm -f intro_modules/*.automodsumm
rm -rf ../docs
mkdir _static
# trigger (saqc) customized documentation pipeline
......@@ -39,7 +40,6 @@ doc:
python make_html_headings_proppa.py -b "sphinx-doc/_build/html/_api" -p "docs/func_modules" -sr ".."
python make_html_headings_proppa.py -b "sphinx-doc/_build/html/_api" -p "docs/intro_modules" -sr ".."
# clear fake modules/intermediate rest files
# rm -r ../docs
rm -r getting_started_md_m2r
rm -r how_to_doc_md_m2r
......
sphinx-doc/flagger.rst → sphinx-doc/flags.rst
+ 2
− 2
View file @ f3efc1c0
Flagger
Flags
=======
.. automodapi:: saqc.flagger
.. automodapi:: saqc.core.flags
:include-all-objects:
:no-heading:
......
sphinx-doc/getting_started_md/Customizations.md
+ 15
− 11
View file @ f3efc1c0
......@@ -24,13 +24,17 @@ SaQC provides two ways to integrate custom routines into the system:
In order to make a function usable within the evaluation framework of SaQC the following interface is needed:
```python
import pandas
import dios
import saqc
def yourTestFunction(
data: pandas.DataFrame,
field: str,
flagger: saqc.flagger.BaseFlagger,
*args: Any,
**kwargs: Any
) -> (dios.DictOfSeries, saqc.flagger.BaseFlagger)
flags: saqc.Flags,
*args,
**kwargs
) -> (dios.DictOfSeries, saqc.Flags)
```
#### Argument Descriptions
......@@ -39,21 +43,21 @@ def yourTestFunction(
|-----------|--------------------------------------------------------------------------------------------------|
| `data` | The actual dataset. |
| `field` | The field/column within `data`, that function is processing. |
| `flagger` | An instance of a flagger, responsible for the translation of test results into quality attributes. |
| `flags` | An instance of Flags, responsible for the translation of test results into quality attributes. |
| `args` | Any other arguments needed to parameterize the function. |
| `kwargs` | Any other keyword arguments needed to parameterize the function. |
### Integrate into SaQC
In order make your function available to the system it needs to be registered. We provide the decorator
[`register`](saqc/functions/register.py) in the module `saqc.functions.register` to integrate your
In order make your function available to the system it needs to be registered. We provide a decorator
[`register`](saqc/functions/register.py) with saqc, to integrate your
test functions into SaQC. Here is a complete dummy example:
```python
from saqc.functions.register import register
from saqc import register
@register
def yourTestFunction(data, field, flagger, *args, **kwargs):
return data, flagger
@register()
def yourTestFunction(data, field, flags, *args, **kwargs):
return data, flags
```
### Example
......
sphinx-doc/how_to_doc_md/HowToDoc.md
+ 4
− 4
View file @ f3efc1c0
......@@ -109,11 +109,11 @@ But mostly the following sections are sufficient:
^
```
## Flagger, data, field, etc.
## Flags, data, field, etc.
use this:
```py
def foo(data, field, flagger):
def foo(data, field, flags):
"""
data : dios.DictOfSeries
A saqc-data object.
......@@ -121,8 +121,8 @@ def foo(data, field, flagger):
field : str
A field denoting a column in data.
flagger : saqc.flagger.BaseFlagger
A saqc-flagger object.
flags : saqc.Flags
A Flags object.
"""
```
......
sphinx-doc/make_doc_module.py
+ 19
− 19
View file @ f3efc1c0
......@@ -9,23 +9,25 @@ import pickle
new_line_re = "(\r\n|[\r\n])"
doc_mod_structure = {'BasicFlagging': ['outliers.flagRange',
'breaks.flagMissing'],
'BasicFlagging_dcstring': '',
'AdvancedFlagging': ['pattern.flagPatternByDTW',
'outliers.flagOffset'],
'AdvancedFlagging_dcstring': ''}
doc_mod_structure = {
'BasicFlagging': ['outliers.flagRange', 'breaks.flagMissing'],
'BasicFlagging_dcstring': '',
'AdvancedFlagging': ['pattern.flagPatternByDTW', 'outliers.flagOffset'],
'AdvancedFlagging_dcstring': ''
}
def rm_section(dcstring, section, _return_section=False):
"""
Detects a section in a docstring and (default) removes it, or (_return_section=True) returns it
"""
section_re = (f'{new_line_re}(?P<s_name>[^\n\r]{{2,}}){new_line_re}(?P<s_dash>-{{2,}}){new_line_re}')
section_re = f'{new_line_re}(?P<s_name>[^\n\r]{{2,}}){new_line_re}(?P<s_dash>-{{2,}}){new_line_re}'
triggers = re.finditer(section_re, dcstring)
matches = [(trigger.groupdict()['s_name'], trigger.span()) for trigger in triggers if
len(trigger.groupdict()['s_name']) == len(trigger.groupdict()['s_dash'])] + \
[(None, (len(dcstring), None))]
matches = [
(trigger.groupdict()['s_name'], trigger.span())
for trigger in triggers
if len(trigger.groupdict()['s_name']) == len(trigger.groupdict()['s_dash'])
] + [(None, (len(dcstring), None))]
sections = [m[0] for m in matches]
starts = ends = 0
if section in sections:
......@@ -50,7 +52,7 @@ def rm_parameter(dcstring, parameter):
start = re.search(p[0], dcstring).span()[0]
try:
end = dcstring.find(next(paramatches)[0])
except(StopIteration):
except StopIteration:
end = len(re.sub(new_line_re + '$', '', dcstring))
return dcstring[0:start] + dcstring[end:]
......@@ -100,7 +102,6 @@ def parse_func_dcstrings(m_paths):
return func_dict
def parse_module_dcstrings(m_paths):
mod_dict = {}
for m in m_paths:
......@@ -137,20 +138,19 @@ def make_doc_module(targetpath, func_dict, doc_mod_structure):
@click.command()
@click.option(
"-p", "--pckpath", type=str, required=True, default="saqc/funcs",
"-p", "--pckpath", type=str, required=True, default="saqc/funcs",
help="Relative path to the package to be documented (relative to sphinx root)."
)
@click.option(
"-t", "--targetpath", type=str, required=True, default="docs/intro_modules",
"-t", "--targetpath", type=str, required=True, default="docs/intro_modules",
help="Output folder path (relative to sphinx root). Will be overridden if already existent."
)
@click.option(
"-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
"-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
)
@click.option(
"-mo", "--mode", type=str, required=True, default='intro_doc', help="either 'intro_doc' or 'module_doc'."
"-mo", "--mode", type=str, required=True, default='intro_doc', help="either 'intro_doc' or 'module_doc'."
)
def main(pckpath, targetpath, sphinxroot, mode):
root_path = os.path.abspath(sphinxroot)
pkg_path = os.path.join(root_path, pckpath)
......@@ -183,7 +183,7 @@ def main(pckpath, targetpath, sphinxroot, mode):
doc_struct[module + '_dcstring'] = mod_dict[module]
make_doc_module(targetpath, func_dict, doc_struct)
if mode == 'module_doc':
doc_struct = {m:[] for m in modules}
doc_struct = {m: [] for m in modules}
for dm in func_dict.keys():
module = re.search('([^ .]*)\.[^ ]*$', dm).group(1)
doc_struct[module].append(dm)
......@@ -191,4 +191,4 @@ def main(pckpath, targetpath, sphinxroot, mode):
if __name__ == "__main__":
main()
\ No newline at end of file
main()
sphinx-doc/make_doc_rst.py
+ 7
− 6
View file @ f3efc1c0
......@@ -4,6 +4,7 @@ import pkgutil
import ast
import shutil
def parse_imports(path):
modules = []
file = open(path)
......@@ -14,19 +15,19 @@ def parse_imports(path):
file.close()
return modules
@click.command()
@click.option(
"-p", "--pckpath", type=str, required=True, default="saqc/funcs",
"-p", "--pckpath", type=str, required=True, default="saqc/funcs",
help="Relative path to the package to be documented (relative to sphinx root)."
)
@click.option(
"-t", "--targetpath", type=str, required=True, default="sphinx-doc/internal_doc_rst",
"-t", "--targetpath", type=str, required=True, default="sphinx-doc/internal_doc_rst",
help="Output folder path (relative to sphinx root). Will be overridden if already existent."
)
@click.option(
"-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
"-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
)
def main(pckpath, targetpath, sphinxroot):
root_path = os.path.abspath(sphinxroot)
targetpath = os.path.join(root_path, targetpath)
......@@ -45,7 +46,7 @@ def main(pckpath, targetpath, sphinxroot):
for module in modules:
imports = parse_imports(os.path.join(pkg_path, f'{module}.py'))
skiplist = [f'\t:skip: {k}' for k in imports]
section = [module] + ["="*len(module)]
section = [module] + ["=" * len(module)]
automodapi_directive = [".. automodapi:: " + pckpath.replace('/', '.') + '.' + module]
no_heading = [f'\t:no-heading:']
to_write = emptyline + section + emptyline + automodapi_directive + skiplist + no_heading
......@@ -55,4 +56,4 @@ def main(pckpath, targetpath, sphinxroot):
if __name__ == "__main__":
main()
\ No newline at end of file
main()
sphinx-doc/make_html_headings_proppa.py
+ 7
− 10
View file @ f3efc1c0
import os
import click
import re
......@@ -7,31 +6,29 @@ import pickle
@click.command()
@click.option(
"-b", "--buildpath", type=str, required=True, default="sphinx-doc/_build/html/_api",
"-b", "--buildpath", type=str, required=True, default="sphinx-doc/_build/html/_api",
help="Relative path to the html api files to be manipulated (relative to sphinx root)."
)
@click.option(
"-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
"-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
)
@click.option(
"-p", "--pckpath", type=str, required=True, default="docs/doc_modules/func_modules",
"-p", "--pckpath", type=str, required=True, default="docs/doc_modules/func_modules",
help="Relative path to the documented package (relative to sphinx root)."
)
def main(buildpath, sphinxroot, pckpath):
root_path = os.path.abspath(sphinxroot)
buildpath = os.path.join(root_path, buildpath)
pckpath = os.path.join(root_path, pckpath)
files = os.listdir(buildpath)
# gather all files from the doc module
files = [f for f in files if re.search('^docs\.',f)]
files = [f for f in files if re.search('^docs\.', f)]
with open(os.path.join(pckpath, 'module_dict.pkl'), 'rb') as file_:
doc_mod_structure = pickle.load(file_)
for key in doc_mod_structure.keys():
# search for all function files assigned to the module
mod_f = [f for f in files if re.search(f'(^|[.]){key}\.[^.]*\.html',f)]
mod_f = [f for f in files if re.search(f'(^|[.]){key}\.[^.]*\.html', f)]
for file_ in mod_f:
parts = file_.split('.')
func = parts[-2]
......@@ -46,7 +43,7 @@ def main(buildpath, sphinxroot, pckpath):
code = code.replace(old_domain_str, new_domain_str)
with open(os.path.join(buildpath, file_), 'w+') as wf:
wf.write(code)
if __name__ == "__main__":
main()
\ No newline at end of file
main()
sphinx-doc/make_md_to_rst.py
+ 12
− 9
View file @ f3efc1c0
"""
The script generates a folder of rest files from a folder of markdown files.
Markdown Hyperlinks between the files in the folder get converted to rest links so that they function properly in a
......@@ -13,6 +12,7 @@ import re
new_line_re = "(\r\n|[\r\n])"
def rebaseAbsRoot(path, target, root):
"""
If path and target intersect at root, return relative path from path to target
......@@ -34,16 +34,18 @@ def rebaseAbsRoot(path, target, root):
del path[0]
del target[0]
up_steps = (len(path) - 1)*f'..{os.sep}'
up_steps = (len(path) - 1) * f'..{os.sep}'
down_steps = os.sep.join(target)
new_path = os.path.join(up_steps, down_steps)
return new_path
def fixTables(f_rst):
body_re = f'((.+){new_line_re})*{new_line_re}((.+){new_line_re})*'
tables = list(re.finditer(f'\.\. list-table::{new_line_re}' + body_re, f_rst))
for t in tables:
tab = t[0]
def pic_repl(match):
leading = match.groupdict()['list_level']
pic_dir = match.groupdict()['pic_directive']
......@@ -54,6 +56,7 @@ def fixTables(f_rst):
if end_space:
pic_dir = re.sub(f'{new_line_re}[ ]*$', end_space[0], pic_dir)
return pic_dir
messy_re = f'(?P<list_level>.*){new_line_re}(?P<pic_directive>[ ]*.. image::[^*-]*)'
# using while loop cause messed pic patterns overlap
tab, repnum = re.subn(messy_re, pic_repl, tab, 1)
......@@ -69,14 +72,14 @@ def fixTables(f_rst):
has_content = len([content for content in last_items if re.search('[^\s-]', content)]) > 0
if has_content:
# append empty cells
tab = tab + (' - \n'*(item_num - last_item_num))
tab += ' - \n' * (item_num - last_item_num)
else:
# delete last row (using replace to avoid false meta char interpretation
tab = tab.replace(bullets[-1][0], '')
bullet_num = len(list(re.finditer(f' \*(?P<items>([ ]+-.*{new_line_re})*)', tab)))
if bullet_num == 1:
#fix empty body table error:
# fix empty body table error:
tab = re.sub(':header-rows: [0-9]', ':header-rows: 0', tab)
if tab != t[0]:
......@@ -85,7 +88,7 @@ def fixTables(f_rst):
return f_rst
def fixLinks(f_rst, f ,targetpath):
def fixLinks(f_rst, f, targetpath):
md_links = list(
re.finditer('(?P<numbered>\. )?`(?P<link_name>[^<`]*) <(?P<md_link>\S*.md)?(#)?(?P<section>[^>]*)?>`_?', f_rst))
for link in md_links:
......@@ -94,7 +97,7 @@ def fixLinks(f_rst, f ,targetpath):
if not link_path:
link_path = f
# change directory to point at temporal rest dir (if link isnt relative):
if os.path.dirname(link_path) is not '':
if os.path.dirname(link_path) != '':
link_path = os.path.join(os.path.dirname(link_path) + '_m2r', os.path.basename(link_path))
# rebase the link to relative link if its not
link_path = rebaseAbsRoot(os.path.join(targetpath, f), link_path, 'sphinx-doc')
......@@ -123,11 +126,11 @@ def fixLinks(f_rst, f ,targetpath):
@click.command()
@click.option(
"-p", "--mdpath", type=str, required=True, default="sphinx-doc/getting_started_md",
"-p", "--mdpath", type=str, required=True, default="sphinx-doc/getting_started_md",
help="Relative path to the folder containing the .md files to be converted (relative to sphinx root)."
)
@click.option(
"-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
"-sr", "--sphinxroot", type=str, required=True, default='..', help="Relative path to the sphinx root."
)
def main(mdpath, sphinxroot):
root_path = os.path.abspath(sphinxroot)
......@@ -150,4 +153,4 @@ def main(mdpath, sphinxroot):
if __name__ == "__main__":
main()
\ No newline at end of file
main()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment