Dendro

@schaefed

level1/do_dendro_calculation.py

+import pandas as pd
+import re
+import math
+
+from pathlib import Path
+from itertools import zip_longest
+
+from config.data import NODATA
+from lib.faccess import getDevices
+from lib.daccess import splitTable, writeTable, iterDataFreq
+from lib.tools import firstOfYear, lastOfYear
+from lib.logger import initLogger, exceptionLogged
+
+
+def filter(data, flags):
+    mask = flags == 9
+    mask.columns = data.columns
+    data = data[mask]
+    return data[data.columns[data.columns.str.match(".*dendro(?!.*r)")]]
+
+
+def grouper(n, iterable, fillvalue=None):
+    args = [iter(iterable)] * n
+    return zip_longest(fillvalue=fillvalue, *args)
+
+
+def fillNa(df):
+    for bhd_value, bhd_manu, flag in grouper(3, df):
+        # setting 91 flag on all values, where calculated value is nan
+        df.loc[df[bhd_value].isna(), flag] = 91
+        df[bhd_value] = df[bhd_value].fillna(df[bhd_manu])
+        # setting 92 flag on all values, where interpolated vales is nan
+        df.loc[df[bhd_manu].isna(), flag] = 92
+        df[bhd_value] = df[bhd_value].fillna(NODATA)
+        df[bhd_manu] = df[bhd_manu].fillna(NODATA)
+    return df
+
+
+def dendroCalculation(df, dendro, first_value):
+    try:
+        first_usable_dendro_value = df[dendro][df[dendro].first_valid_index()]
+    except KeyError:
+        first_usable_dendro_value = first_value
+    df[dendro][0] = first_usable_dendro_value
+    df[dendro] = df[dendro].ffill(axis=0)
+    df[dendro] = df[dendro] / 1000
+
+    df.insert(0, 'rad', df['dbh'][0] * 10 / 2)
+    dini = df['d_ini'][0] * 10
+    r0 = df['rad'][0]
+    c0 = math.pi * 2 * r0
+    v0 = df[dendro][0]
+    cwire = c0 + 2 * ((dini + r0 - v0) ** 2 - r0 ** 2) ** 0.5 - 2 * r0 * math.acos(r0 / (dini + r0 - v0))
+    for num, val in enumerate(df['rad'][1:], 1):
+        df['rad'][num] = (cwire
+                          - 2 * ((dini + df['rad'].iloc[num - 1] - df[dendro].iloc[num]) ** 2
+                                 - df['rad'].iloc[num - 1] ** 2) ** 0.5
+                          + 2 * df['rad'].iloc[num - 1]
+                          * math.acos(df['rad'].iloc[num - 1] / (dini + df['rad'].iloc[num - 1]
+                                                                 - df[dendro].iloc[num]))) / math.pi / 2
+    return df['rad'] * 2
+
+def dendroInterpolation(df, dendro):
+    df_reindexed = df.reindex(pd.date_range(start=df.index.min(), end=df.index.max(), freq='10min'))
+    df_reindexed = df_reindexed.interpolate(method='linear')
+    df_reindexed_regex = r'[0-9]+'
+    df_reindexed.name = f'bhd_manu_{re.findall(df_reindexed_regex, dendro)[0]} [mm]'
+    return df_reindexed
+
+
+def dendroProcess(start_date, end_date, data, manflags):
+    final_df = pd.DataFrame()
+    for dendro in data:
+        if data[dendro].isnull().all():
+            continue  # table is empty
+
+        try:
+            dropped_data = data[dendro].dropna()
+            dropped_manflags = manflags.loc[dendro].dropna(how='all')
+            concat_df = pd.concat([dropped_data, dropped_manflags], axis=1)
+        except pd.errors.InvalidIndexError:
+            continue  # concat Error
+
+        concat_df = concat_df.loc[start_date:end_date]
+
+        first_value = data.loc[data[dendro].first_valid_index(), dendro]
+        d_ini_index_list = concat_df['d_ini'].dropna().index
+        temporary_df = pd.DataFrame()
+        for num, val in enumerate(d_ini_index_list):
+            try:
+                part_concat_df = pd.concat([concat_df.loc[val:], concat_df.loc[d_ini_index_list[num:][1]:]]).drop_duplicates(keep=False)
+            except IndexError:
+                part_concat_df = concat_df.loc[val:]
+            value_df = dendroCalculation(part_concat_df, dendro, first_value)
+            temporary_df = pd.concat([temporary_df, value_df], axis=0)
+        try:
+            temporary_df_regex = r'[0-9]+'
+            temporary_df.columns = [f'bhd_auto_{re.findall(temporary_df_regex, dendro)[0]} [mm]']
+        except ValueError:
+            continue  # temporary_df is empty, because no d_ini value in this period of time
+
+        # interpolation Column
+        df = manflags['dbh'].loc[dendro]
+        df_reindexed = dendroInterpolation(df, dendro)
+
+        # Flags Column
+        flag = temporary_df.copy()
+        name_regex = r'[0-9]+'
+        name = f'bhd_{re.findall(name_regex, dendro)[0]}_f'
+        flag.columns = [name]
+        flag[name] = 9
+
+        final_df = pd.concat([final_df, temporary_df, df_reindexed * 10, flag], axis=1)
+    final_df.index.name = 'Date Time'
+    final_df = final_df.loc[start_date:end_date]
+    return final_df
+
+
+def writeData(data, device):
+    for date, out in iterDataFreq(data, "Y"):
+        fname = Path(
+            device.derivedpath,
+            f"HH_{device.station_key}_bhd_trees_10min.level2.csv"
+        )
+        writeTable(fname, out, make_path=True)
+
+
+def procDevice(logger, device, **kwargs):
+    logger.info(f"processing: {device}")
+
+    logger.debug("reading data")
+    df = device.getL1Data(
+        start_date=firstOfYear(device.start_date),
+        end_date=lastOfYear(device.end_date),
+        reindex=True, fill=True)
+    manflags = device.getManualFlags()
+    if df.empty or manflags.empty:
+        logger.info(f"{device}: no data found for period - skipping")
+
+    logger.debug("processing")
+    data, flags = splitTable(df)
+    data = filter(data, flags)
+    manflags = manflags.set_index('end', append=True)
+    manflags = manflags.drop(['start', 'comment'], axis=1)
+    manflags = manflags.dropna(subset=['dbh', 'd_ini'], how='all')
+    out = dendroProcess(logger, device.start_date, device.end_date, data, manflags)
+    out = fillNa(out)
+
+    return out
+
+
+def main(station, device, start_date, end_date, debug):
+    with initLogger(__file__, debug) as logger:
+        devices = getDevices(
+            station=station, logger=device, tag="meteo",
+            start_date=firstOfYear(start_date),
+            end_date=lastOfYear(end_date)
+        )
+
+        final_out_list = []
+        for device in devices:
+            with exceptionLogged(logger, f"{device}: failed", fail=debug):
+                final_out_list.append(procDevice(logger, device))
+
+        logger.debug("writing")
+        writeData(pd.concat(final_out_list, axis=1), device)
+
+
+if __name__ == "__main__":
+    from lib.argparser import parseArguments
+
+    args = parseArguments("Calculate dendro data", {"ndays": 1})
+    main(args.station, args.logger, args.start_date, args.end_date, args.debug)