Commit 1248b0c5 authored by David Schäfer's avatar David Schäfer
Browse files

bump saqc to version2 rc

parent 8520c81a
......@@ -10,13 +10,11 @@ import requests
import numpy as np
import pandas as pd
from saqc import fromConfig, DmpTranslator, flagging, processing, Flags, UNFLAGGED
from saqc import fromConfig, flagging, register, Flags, UNFLAGGED
from pipetools.lib import splitData, mergeData, writeParquet
CONFIG_FILE = "saqc-proc.csv"
NM_URL = "http://www.nmdb.eu/nest/draw_graph.php"
......@@ -81,20 +79,19 @@ def extractNM(data):
return nm
@processing()
@register(mask=["field"], demask=["field"], squeeze=[], handles_target=True)
def calcSoilMoisture(
data: pd.DataFrame, field: str, flags: Flags,
neutrons_field: str, n0: int,
lattice_water: float, bulk_density: float, soil_org_carbon: float,
data: pd.DataFrame, field: str, flags: Flags, target: str,
n0: int, lattice_water: float, bulk_density: float, soil_org_carbon: float,
# basically constants
a0: float = 0.0808, a1: float = 0.372, a2: float = 0.115,
**kwargs
) -> (pd.DataFrame, Flags):
neutrons = data[neutrons_field]
neutrons = data[field]
sm = (a0 / (neutrons / n0 - a1) - a2 - lattice_water - soil_org_carbon * 0.556) * bulk_density
data[field] = sm
flags[field] = pd.Series(UNFLAGGED, index=data[field].index)
data[target] = sm
flags[target] = pd.Series(UNFLAGGED, index=sm.index)
return data, flags
......@@ -111,7 +108,7 @@ def getManFlags(fname, field):
return out
@flagging(masking="field")
@flagging()
def flagManual(data, field, flags, fname, **kwargs):
mflags = getManFlags(fname, field)
......@@ -131,11 +128,9 @@ def main(infile, outfile, configfile):
data = pd.read_parquet(infile)
data.loc[:, "NM"] = extractNM(data)
saqc = fromConfig(fname=configfile, data=data, scheme=DmpTranslator(), error_policy="raise")
data, flags = saqc.getResult()
saqc = fromConfig(fname=configfile, data=data, scheme="dmp")
data = data.round(8)
df_out = mergeData(data, flags)
df_out = mergeData(saqc.data.round(8), saqc.flags)
writeParquet(df_out, outfile)
......
......@@ -3,62 +3,62 @@
varname ; test
# ----- ; ----
'.*' ;flagManual(fname="manual/saqc-flags-1488.csv")
'.*' ; breaks.flagMissing()
'.*' ; flagManual(fname="manual/saqc-flags-1488.csv")
'.*' ; flagMissing()
# quality control
# ---------------
N1 ; outliers.flagRange(min=100, max=3000)
N2 ; outliers.flagRange(min=100, max=3000)
N1ET ; outliers.flagRange(min=1, max=10000)
N2ET ; outliers.flagRange(min=1, max=10000)
p1 ; outliers.flagRange(min=1, max=1100)
p3 ; outliers.flagRange(min=1, max=1100)
p4 ; outliers.flagRange(min=1, max=1100)
RH1 ; outliers.flagRange(min=0, max=100)
RH2 ; outliers.flagRange(min=0, max=100)
N1RH ; outliers.flagRange(min=0, max=100)
N2RH ; outliers.flagRange(min=0, max=100)
RHX ; outliers.flagRange(min=0, max=100)
T1 ; outliers.flagRange(min=-60, max=60)
T2 ; outliers.flagRange(min=-60, max=60)
T3 ; outliers.flagRange(min=-60, max=60)
T4 ; outliers.flagRange(min=-60, max=60)
TX ; outliers.flagRange(min=-60, max=60)
N1T ; outliers.flagRange(min=-60, max=60)
N2T ; outliers.flagRange(min=-60, max=60)
Vbat ; outliers.flagRange(min=10, max=20)
N1 ; flagRange(min=100, max=3000)
N2 ; flagRange(min=100, max=3000)
N1ET ; flagRange(min=1, max=10000)
N2ET ; flagRange(min=1, max=10000)
p1 ; flagRange(min=1, max=1100)
p3 ; flagRange(min=1, max=1100)
p4 ; flagRange(min=1, max=1100)
RH1 ; flagRange(min=0, max=100)
RH2 ; flagRange(min=0, max=100)
N1RH ; flagRange(min=0, max=100)
N2RH ; flagRange(min=0, max=100)
RHX ; flagRange(min=0, max=100)
T1 ; flagRange(min=-60, max=60)
T2 ; flagRange(min=-60, max=60)
T3 ; flagRange(min=-60, max=60)
T4 ; flagRange(min=-60, max=60)
TX ; flagRange(min=-60, max=60)
N1T ; flagRange(min=-60, max=60)
N2T ; flagRange(min=-60, max=60)
Vbat ; flagRange(min=10, max=20)
# convert units
# -------------
N ; generic.process(func=(N1 + N2) / N1ET * 3600)
N_cleaned ; generic.process(func=N)
N ; processGeneric(field=["N1", "N2", "N1ET"], func=(N1 + N2) / N1ET * 3600)
N ; copyField(target="N_cleaned")
# quality control - new variables
# ---------------
N ; outliers.flagRange(min=500, max=2500)
N_cleaned ; outliers.flagRange(min=500, max=2500)
N ; flagRange(min=500, max=2500)
N_cleaned ; flagRange(min=500, max=2500)
# data corrections/processing
# ---------------------------
ah ; generic.process(func=6.112 * exp(17.67 * TX/(243.5 + TX))/(273.15 + TX) * 2.1674 * RHX)
correct_h ; generic.process(func=1 + 0.0054 * (ah - 0))
correct_p ; generic.process(func=exp((p4 - 1013.25) / 136))
correct_inc ; generic.process(func=1/(1 - 1 * (1 - NM / 150)))
N_corrected ; generic.process(func=N * correct_p * correct_h * correct_inc)
ah ; processGeneric(field=["TX", "RHX"], func=6.112 * exp(17.67 * TX/(243.5 + TX))/(273.15 + TX) * 2.1674 * RHX)
correct_h ; processGeneric(field="ah", func=1 + 0.0054 * (ah - 0))
correct_p ; processGeneric(field="p4", func=exp((p4 - 1013.25) / 136))
correct_inc ; processGeneric(field="NM", func=1/(1 - 1 * (1 - NM / 150)))
N_corrected ; processGeneric(field=["N", "correct_p", "correct_h", "correct_inc"], func=N * correct_p * correct_h * correct_inc)
# soil moisture calculation
# -------------------------
SM ; calcSoilMoisture(neutrons_field="N_corrected", n0=2054, lattice_water=0.0043, bulk_density=1.6, soil_org_carbon=0.0050)
SM ; outliers.flagRange(min=0, max=0.7)
SM ; calcSoilMoisture(field="N_corrected", n0=2054, lattice_water=0.0043, bulk_density=1.6, soil_org_carbon=0.0050)
SM ; flagRange(min=0, max=0.7)
# rename variables to match the DMP names
# ---------------------------------------
p4 ; tools.copy(new_field="p")
RHX ; tools.copy(new_field="rh")
TX ; tools.copy(new_field="T")
Vbat ; tools.copy(new_field="bat")
p4 ; copyField(target="p")
RHX ; copyField(target="rh")
TX ; copyField(target="T")
Vbat ; copyField(target="bat")
# flag everything
# ----------------------------
'.*' ; generic.flag(func=~isflagged(this), flag=GOOD)
'.*' ; flagUnflagged(flag=GOOD)
......@@ -4,52 +4,52 @@ varname ; test
# ----- ; ----
'.*' ; flagManual(fname="manual/saqc-flags-1490.csv")
'.*' ; breaks.flagMissing()
'.*' ; flagMissing()
# quality control
# ---------------
N1 ; outliers.flagRange(min=1, max=3000)
N1ET ; outliers.flagRange(min=1, max=10000)
p1 ; outliers.flagRange(min=1, max=1100)
p4 ; outliers.flagRange(min=1, max=1100)
RH1 ; outliers.flagRange(min=0, max=100)
N1RH ; outliers.flagRange(min=0, max=100)
RHX ; outliers.flagRange(min=0, max=100)
T1 ; outliers.flagRange(min=-60, max=60)
TX ; outliers.flagRange(min=-60, max=60)
N1T ; outliers.flagRange(min=-60, max=60)
Vbat ; outliers.flagRange(min=10, max=20)
N1 ; flagRange(min=1, max=3000)
N1ET ; flagRange(min=1, max=10000)
p1 ; flagRange(min=1, max=1100)
p4 ; flagRange(min=1, max=1100)
RH1 ; flagRange(min=0, max=100)
N1RH ; flagRange(min=0, max=100)
RHX ; flagRange(min=0, max=100)
T1 ; flagRange(min=-60, max=60)
TX ; flagRange(min=-60, max=60)
N1T ; flagRange(min=-60, max=60)
Vbat ; flagRange(min=10, max=20)
# convert units
# -------------
N ; generic.process(func=N1 / N1ET * 3600)
N_cleaned ; generic.process(func=N)
N ; processGeneric(field=["N1", "N1ET"], func=N1 / N1ET * 3600)
N ; copyField(target="N_cleaned")
# quality control - new variables
# ---------------
N ; outliers.flagRange(min=1000, max=6000)
N_cleaned ; outliers.flagRange(min=1000, max=6000)
N ; flagRange(min=1000, max=6000)
N_cleaned ; flagRange(min=1000, max=6000)
# data corrections/processing
# ---------------------------
ah ; generic.process(func=6.112 * exp(17.67 * TX/(243.5 + TX))/(273.15 + TX) * 2.1674 * RHX)
correct_h ; generic.process(func=1 + 0.0054 * (ah - 0))
correct_p ; generic.process(func=exp((p4 - 1013.25) / 136))
correct_inc ; generic.process(func=1/(1 - 1 * (1 - NM / 150)))
N_corrected ; generic.process(func=N * correct_p * correct_h * correct_inc)
ah ; processGeneric(field=["TX", "RHX"], func=6.112 * exp(17.67 * TX/(243.5 + TX))/(273.15 + TX) * 2.1674 * RHX)
correct_h ; processGeneric(field="ah", func=1 + 0.0054 * (ah - 0))
correct_p ; processGeneric(field="p4", func=exp((p4 - 1013.25) / 136))
correct_inc ; processGeneric(field="NM", func=1/(1 - 1 * (1 - NM / 150)))
N_corrected ; processGeneric(field=["N", "correct_p", "correct_h", "correct_inc"], func=N * correct_p * correct_h * correct_inc)
# soil moisture calculation
# -------------------------
SM ; calcSoilMoisture(neutrons_field="N_corrected", n0=2250, lattice_water=0.02, bulk_density=1.6, soil_org_carbon=0.0050)
SM ; outliers.flagRange(min=0, max=0.7)
SM ; calcSoilMoisture(field="N_corrected", n0=2250, lattice_water=0.02, bulk_density=1.6, soil_org_carbon=0.0050)
SM ; flagRange(min=0, max=0.7)
# rename variables to match the DMP names
# ---------------------------------------
p4 ; tools.copy(new_field="p")
RHX ; tools.copy(new_field="rh")
TX ; tools.copy(new_field="T")
Vbat ; tools.copy(new_field="bat")
p4 ; copyField(target="p")
RHX ; copyField(target="rh")
TX ; copyField(target="T")
Vbat ; copyField(target="bat")
# flag everything
# ---------------
'.*' ; generic.flag(func=~isflagged(this), flag=GOOD)
'.*' ; flagUnflagged(flag=GOOD)
......@@ -4,52 +4,52 @@ varname ; test
# ----- ; ----
'.*' ; flagManual(fname="manual/saqc-flags-1499.csv")
'.*' ; breaks.flagMissing()
'.*' ; flagMissing()
# quality control
# ---------------
N1 ; outliers.flagRange(min=1, max=5000)
N1ET ; outliers.flagRange(min=1, max=10000)
p1 ; outliers.flagRange(min=1, max=1100)
p4 ; outliers.flagRange(min=1, max=1100)
RH1 ; outliers.flagRange(min=0, max=100)
N1RH ; outliers.flagRange(min=0, max=100)
RHX ; outliers.flagRange(min=0, max=100)
T1 ; outliers.flagRange(min=-60, max=60)
TX ; outliers.flagRange(min=-60, max=60)
N1T ; outliers.flagRange(min=-60, max=60)
Vbat ; outliers.flagRange(min=10, max=20)
N1 ; flagRange(min=1, max=5000)
N1ET ; flagRange(min=1, max=10000)
p1 ; flagRange(min=1, max=1100)
p4 ; flagRange(min=1, max=1100)
RH1 ; flagRange(min=0, max=100)
N1RH ; flagRange(min=0, max=100)
RHX ; flagRange(min=0, max=100)
T1 ; flagRange(min=-60, max=60)
TX ; flagRange(min=-60, max=60)
N1T ; flagRange(min=-60, max=60)
Vbat ; flagRange(min=10, max=20)
# convert units
# -------------
N ; generic.process(func=N1 / N1ET * 3600)
N_cleaned ; generic.process(func=N)
N ; processGeneric(field=["N1", "N1ET"], func=N1 / N1ET * 3600)
N ; copyField(target="N_cleaned")
# quality control - new variables
# ---------------
N ; outliers.flagRange(min=1000, max=6000)
N_cleaned ; outliers.flagRange(min=1000, max=6000)
N ; flagRange(min=1000, max=6000)
N_cleaned ; flagRange(min=1000, max=6000)
# data corrections/processing
# ---------------------------
ah ; generic.process(func=6.112 * exp(17.67 * TX/(243.5 + TX))/(273.15 + TX) * 2.1674 * RHX)
correct_h ; generic.process(func=1 + 0.0054 * (ah - 0))
correct_p ; generic.process(func=exp((p4 - 1013.25) / 136))
correct_inc ; generic.process(func=1/(1 - 1 * (1 - NM / 150)))
N_corrected ; generic.process(func=N * correct_p * correct_h * correct_inc)
ah ; processGeneric(field=["TX", "RHX"], func=6.112 * exp(17.67 * TX/(243.5 + TX))/(273.15 + TX) * 2.1674 * RHX)
correct_h ; processGeneric(field="ah", func=1 + 0.0054 * (ah - 0))
correct_p ; processGeneric(field="p4", func=exp((p4 - 1013.25) / 136))
correct_inc ; processGeneric(field="NM", func=1/(1 - 1 * (1 - NM / 150)))
N_corrected ; processGeneric(field=["N", "correct_p", "correct_h", "correct_inc"], func=N * correct_p * correct_h * correct_inc)
# soil moisture calculation
# -------------------------
SM ; calcSoilMoisture(neutrons_field="N_corrected", n0=1830, lattice_water=0.0043, bulk_density=1.6, soil_org_carbon=0.0050)
SM ; outliers.flagRange(min=0, max=0.7)
SM ; calcSoilMoisture(field="N_corrected", n0=1830, lattice_water=0.0043, bulk_density=1.6, soil_org_carbon=0.0050)
SM ; flagRange(min=0, max=0.7)
# rename variables to match the DMP names
# ---------------------------------------
p4 ; tools.copy(new_field="p")
RHX ; tools.copy(new_field="rh")
TX ; tools.copy(new_field="T")
Vbat ; tools.copy(new_field="bat")
p4 ; copyField(target="p")
RHX ; copyField(target="rh")
TX ; copyField(target="T")
Vbat ; copyField(target="bat")
# flag everything
# ---------------
'.*' ; generic.flag(func=~isflagged(this), flag=GOOD)
'.*' ; flagUnflagged(flag=GOOD)
......@@ -4,61 +4,61 @@ varname ; test
# ----- ; ----
'.*' ; flagManual(fname="manual/saqc-flags-1501.csv")
'.*' ; breaks.flagMissing()
'.*' ; flagMissing()
# quality control
# ---------------
N1 ; outliers.flagRange(min=1, max=1000)
N2 ; outliers.flagRange(min=1, max=1000)
N1ET ; outliers.flagRange(min=1, max=10000)
N2ET ; outliers.flagRange(min=1, max=10000)
p1 ; outliers.flagRange(min=1, max=1100)
p3 ; outliers.flagRange(min=1, max=1100)
p4 ; outliers.flagRange(min=1, max=1100)
RH1 ; outliers.flagRange(min=0, max=100)
RH2 ; outliers.flagRange(min=0, max=100)
N1RH ; outliers.flagRange(min=0, max=100)
N2RH ; outliers.flagRange(min=0, max=100)
RHX ; outliers.flagRange(min=0, max=100)
T1 ; outliers.flagRange(min=-60, max=60)
T2 ; outliers.flagRange(min=-60, max=60)
T3 ; outliers.flagRange(min=-60, max=60)
T4 ; outliers.flagRange(min=-60, max=60)
TX ; outliers.flagRange(min=-60, max=60)
N1T ; outliers.flagRange(min=-60, max=60)
N2T ; outliers.flagRange(min=-60, max=60)
Vbat ; outliers.flagRange(min=10, max=20)
N1 ; flagRange(min=1, max=1000)
N2 ; flagRange(min=1, max=1000)
N1ET ; flagRange(min=1, max=10000)
N2ET ; flagRange(min=1, max=10000)
p1 ; flagRange(min=1, max=1100)
p3 ; flagRange(min=1, max=1100)
p4 ; flagRange(min=1, max=1100)
RH1 ; flagRange(min=0, max=100)
RH2 ; flagRange(min=0, max=100)
N1RH ; flagRange(min=0, max=100)
N2RH ; flagRange(min=0, max=100)
RHX ; flagRange(min=0, max=100)
T1 ; flagRange(min=-60, max=60)
T2 ; flagRange(min=-60, max=60)
T3 ; flagRange(min=-60, max=60)
T4 ; flagRange(min=-60, max=60)
TX ; flagRange(min=-60, max=60)
N1T ; flagRange(min=-60, max=60)
N2T ; flagRange(min=-60, max=60)
Vbat ; flagRange(min=10, max=20)
# convert units
# -------------
N ; generic.process(func=N1 / N1ET * 3600)
N_cleaned ; generic.process(func=N)
N ; processGeneric(field=["N1", "N1ET"], func=N1 / N1ET * 3600)
N ; copyField(target="N_cleaned")
# quality control - new variables
# ---------------
N ; outliers.flagRange(min=100, max=1000)
N_cleaned ; outliers.flagRange(min=100, max=1000)
N ; flagRange(min=100, max=1000)
N_cleaned ; flagRange(min=100, max=1000)
# data corrections/processing
# ---------------------------
ah ; generic.process(func=6.112 * exp(17.67 * TX/(243.5 + TX))/(273.15 + TX) * 2.1674 * RHX)
correct_h ; generic.process(func=1 + 0.0054 * (ah - 0))
correct_p ; generic.process(func=exp((p4 - 1013.25) / 136))
correct_inc ; generic.process(func=1/(1 - 1 * (1 - NM / 150)))
N_corrected ; generic.process(func=N * correct_p * correct_h * correct_inc)
ah ; processGeneric(field=["TX", "RHX"], func=6.112 * exp(17.67 * TX/(243.5 + TX))/(273.15 + TX) * 2.1674 * RHX)
correct_h ; processGeneric(field="ah", func=1 + 0.0054 * (ah - 0))
correct_p ; processGeneric(field="p4", func=exp((p4 - 1013.25) / 136))
correct_inc ; processGeneric(field="NM", func=1/(1 - 1 * (1 - NM / 150)))
N_corrected ; processGeneric(field=["N", "correct_p", "correct_h", "correct_inc"], func=N * correct_p * correct_h * correct_inc)
# soil moisture calculation
# -------------------------
SM ; calcSoilMoisture(neutrons_field="N_corrected", n0=828, lattice_water=0.05, bulk_density=1.05, soil_org_carbon=0.1)
SM ; outliers.flagRange(min=0, max=0.7)
SM ; calcSoilMoisture(field="N_corrected", n0=828, lattice_water=0.05, bulk_density=1.05, soil_org_carbon=0.1)
SM ; flagRange(min=0, max=0.7)
# rename variables to match the DMP names
# ---------------------------------------
p4 ; tools.copy(new_field="p")
RHX ; tools.copy(new_field="rh")
TX ; tools.copy(new_field="T")
Vbat ; tools.copy(new_field="bat")
p4 ; copyField(target="p")
RHX ; copyField(target="rh")
TX ; copyField(target="T")
Vbat ; copyField(target="bat")
# flag everything
# ----------------------------
'.*' ; generic.flag(func=~isflagged(this), flag=GOOD)
'.*' ; flagUnflagged(flag=GOOD)
......@@ -4,61 +4,61 @@ varname ; test
# ----- ; ----
'.*' ; flagManual(fname="manual/saqc-flags-1504.csv")
'.*' ; breaks.flagMissing()
'.*' ; flagMissing()
# quality control
# ---------------
N1 ; outliers.flagRange(min=1, max=3000)
N2 ; outliers.flagRange(min=1, max=3000)
N1ET ; outliers.flagRange(min=1, max=10000)
N2ET ; outliers.flagRange(min=1, max=10000)
p1 ; outliers.flagRange(min=1, max=1100)
p3 ; outliers.flagRange(min=1, max=1100)
p4 ; outliers.flagRange(min=1, max=1100)
RH1 ; outliers.flagRange(min=0, max=100)
RH2 ; outliers.flagRange(min=0, max=100)
N1RH ; outliers.flagRange(min=0, max=100)
N2RH ; outliers.flagRange(min=0, max=100)
RHX ; outliers.flagRange(min=0, max=100)
T1 ; outliers.flagRange(min=-60, max=60)
T2 ; outliers.flagRange(min=-60, max=60)
T3 ; outliers.flagRange(min=-60, max=60)
T4 ; outliers.flagRange(min=-60, max=60)
TX ; outliers.flagRange(min=-60, max=60)
N1T ; outliers.flagRange(min=-60, max=60)
N2T ; outliers.flagRange(min=-60, max=60)
Vbat ; outliers.flagRange(min=10, max=20)
N1 ; flagRange(min=1, max=3000)
N2 ; flagRange(min=1, max=3000)
N1ET ; flagRange(min=1, max=10000)
N2ET ; flagRange(min=1, max=10000)
p1 ; flagRange(min=1, max=1100)
p3 ; flagRange(min=1, max=1100)
p4 ; flagRange(min=1, max=1100)
RH1 ; flagRange(min=0, max=100)
RH2 ; flagRange(min=0, max=100)
N1RH ; flagRange(min=0, max=100)
N2RH ; flagRange(min=0, max=100)
RHX ; flagRange(min=0, max=100)
T1 ; flagRange(min=-60, max=60)
T2 ; flagRange(min=-60, max=60)
T3 ; flagRange(min=-60, max=60)
T4 ; flagRange(min=-60, max=60)
TX ; flagRange(min=-60, max=60)
N1T ; flagRange(min=-60, max=60)
N2T ; flagRange(min=-60, max=60)
Vbat ; flagRange(min=10, max=20)
# convert units
# -------------
N ; generic.process(func=N1 / N1ET * 3600)
N_cleaned ; generic.process(func=N)
N ; processGeneric(field=["N1", "N1ET"], func=N1 / N1ET * 3600)
N ; copyField(target="N_cleaned")
# quality control - new variables
# ---------------
N ; outliers.flagRange(min=100, max=950)
N_cleaned ; outliers.flagRange(min=100, max=950)
N ; flagRange(min=100, max=950)
N_cleaned ; flagRange(min=100, max=950)
# data corrections/processing
# ---------------------------
ah ; generic.process(func=6.112 * exp(17.67 * TX/(243.5 + TX))/(273.15 + TX) * 2.1674 * RHX)
correct_h ; generic.process(func=1 + 0.0054 * (ah - 0))
correct_p ; generic.process(func=exp((p4 - 1013.25) / 136))
correct_inc ; generic.process(func=1/(1 - 1 * (1 - NM / 150)))
N_corrected ; generic.process(func=N * correct_p * correct_h * correct_inc)
ah ; processGeneric(field=["TX", "RHX"], func=6.112 * exp(17.67 * TX/(243.5 + TX))/(273.15 + TX) * 2.1674 * RHX)
correct_h ; processGeneric(field=["ah"], func=1 + 0.0054 * (ah - 0))
correct_p ; processGeneric(field=["p4"], func=exp((p4 - 1013.25) / 136))
correct_inc ; processGeneric(field=["NM"], func=1/(1 - 1 * (1 - NM / 150)))
N_corrected ; processGeneric(field=["N", "correct_p", "correct_h", "correct_inc"], func=N * correct_p * correct_h * correct_inc)
# soil moisture calculation
# -------------------------
SM ; calcSoilMoisture(neutrons_field="N_corrected", n0=945, lattice_water=0.05, bulk_density=1.05, soil_org_carbon=0.1)
SM ; outliers.flagRange(min=0, max=0.7)
SM ; calcSoilMoisture(field="N_corrected", n0=945, lattice_water=0.05, bulk_density=1.05, soil_org_carbon=0.1)
SM ; flagRange(min=0, max=0.7)
# rename variables to match the DMP names
# ---------------------------------------
p4 ; tools.copy(new_field="p")
RHX ; tools.copy(new_field="rh")
TX ; tools.copy(new_field="T")
Vbat ; tools.copy(new_field="bat")
p4 ; copyField(target="p")
RHX ; copyField(target="rh")
TX ; copyField(target="T")
Vbat ; copyField(target="bat")
# flag everything
# ----------------------------
'.*' ; generic.flag(func=~isflagged(this), flag=GOOD)
'.*' ; flagUnflagged(flag=GOOD)
......@@ -4,61 +4,61 @@ varname ; test
# ----- ; ----
'.*' ; flagManual(fname="manual/saqc-flags-1505.csv")
'.*' ; breaks.flagMissing()
'.*' ; flagMissing()
# quality control
# ---------------
N1 ; outliers.flagRange(min=1, max=3000)
N2 ; outliers.flagRange(min=1, max=3000)
N1ET ; outliers.flagRange(min=1, max=10000)
N2ET ; outliers.flagRange(min=1, max=10000)
p1 ; outliers.flagRange(min=1, max=1100)
p3 ; outliers.flagRange(min=1, max=1100)
p4 ; outliers.flagRange(min=1, max=1100)
RH1 ; outliers.flagRange(min=0, max=100)
RH2 ; outliers.flagRange(min=0, max=100)
N1RH ; outliers.flagRange(min=0, max=100)
N2RH ; outliers.flagRange(min=0, max=100)
RHX ; outliers.flagRange(min=0, max=100)
T1 ; outliers.flagRange(min=-60, max=60)
T2 ; outliers.flagRange(min=-60, max=60)
T3 ; outliers.flagRange(min=-60, max=60)
T4 ; outliers.flagRange(min=-60, max=60)
TX ; outliers.flagRange(min=-60, max=60)
N1T ; outliers.flagRange(min=-60, max=60)
N2T ; outliers.flagRange(min=-60, max=60)
Vbat ; outliers.flagRange(min=10, max=20)
N1 ; flagRange(min=1, max=3000)
N2 ; flagRange(min=1, max=3000)
N1ET ; flagRange(min=1, max=10000)
N2ET ; flagRange(min=1, max=10000)
p1 ; flagRange(min=1, max=1100)
p3 ; flagRange(min=1, max=1100)
p4 ; flagRange(min=1, max=1100)
RH1 ; flagRange(min=0, max=100)
RH2 ; flagRange(min=0, max=100)
N1RH ; flagRange(min=0, max=100)
N2RH ; flagRange(min=0, max=100)
RHX ; flagRange(min=0, max=100)
T1 ; flagRange(min=-60, max=60)
T2 ; flagRange(min=-60, max=60)
T3 ; flagRange(min=-60, max=60)
T4 ; flagRange(min=-60, max=60)
TX ; flagRange(min=-60, max=60)
N1T ; flagRange(min=-60, max=60)
N2T ; flagRange(min=-60, max=60)
Vbat ; flagRange(min=10, max=20)
# convert units
# -------------
N ; generic.process(func=N1 / N1ET * 3600)
N_cleaned ; generic.process(func=N)
N ; processGeneric(field=["N1", "N1ET"], func=N1 / N1ET * 3600)
N ; copyField(target="N_cleaned")
# quality control - new variables
# ---------------
N ; outliers.flagRange(min=100, max=1000)
N_cleaned ; outliers.flagRange(min=100, max=1000)
N ; flagRange(min=100, max=1000)
N_cleaned ; flagRange(min=100, max=1000)
# data corrections/processing
# ---------------------------
ah ; generic.process(func=6.112 * exp(17.67 * TX/(243.5 + TX))/(273.15 + TX) * 2.1674 * RHX)
correct_h ; generic.process(func=1 + 0.0054 * (ah - 0))
correct_p ; generic.process(func=exp((p4 - 1013.25) / 136))
correct_inc ; generic.process(func=1/(1 - 1 * (1 - NM / 150)))
N_corrected ; generic.process(func=N * correct_p * correct_h * correct_inc)
ah ; processGeneric(field=["TX", "RHX"], func=6.112 * exp(17.67 * TX/(243.5 + TX))/(273.15 + TX) * 2.1674 * RHX)
correct_h ; processGeneric(field="ah", func=1 + 0.0054 * (ah - 0))
correct_p ; processGeneric(field="p4", func=exp((p4 - 1013.25) / 136))
correct_inc ; processGeneric(field="NM", func=1/(1 - 1 * (1 - NM / 150)))
N_corrected ; processGeneric(field=["N", "correct_p", "correct_h", "correct_inc"], func=N * correct_p * correct_h * correct_inc)
# soil moisture calculation
# -------------------------
SM ; calcSoilMoisture(neutrons_field="N_corrected", n0=1010, lattice_water=0.01, bulk_density=1.4, soil_org_carbon=0.04)
SM ; outliers.flagRange(min=0, max=0.7)
SM ; calcSoilMoisture(field="N_corrected", n0=1010, lattice_water=0.01, bulk_density=1.4, soil_org_carbon=0.04)
SM ; flagRange(min=0, max=0.7)
# rename variables to match the DMP names
# ---------------------------------------
p4 ; tools.copy(new_field="p")
RHX ; tools.copy(new_field="rh")
TX ; tools.copy(new_field="T")
Vbat ; tools.copy(new_field="bat")
p4 ; copyField(target="p")
RHX ; copyField(target="rh")
TX ; copyField(target="T")
Vbat ; copyField(target="bat")
# flag everything
# ----------------------------
'.*' ; generic.flag(func=~isflagged(this), flag=GOOD)
'.*' ; flagUnflagged(flag=GOOD)
......@@ -4,61 +4,61 @@ varname ; test
# ----- ; ----
'.*' ; flagManual(fname="manual/saqc-flags-1507.csv")
'.*' ; breaks.flagMissing()
'.*' ; flagMissing()
# quality control
# ---------------
N1 ; outliers.flagRange(min=1, max=8000)
N2 ; outliers.flagRange(min=1, max=8000)