diff --git a/flagger/baseflagger.py b/flagger/baseflagger.py
index 6fb12004c38508ff15f86e16699e4966b3201bfc..3452ec0fbb983a184385595aaa18a9607ba21f11 100644
--- a/flagger/baseflagger.py
+++ b/flagger/baseflagger.py
@@ -50,7 +50,8 @@ class BaseFlagger:
return flags.values
def initFlags(self, data: pd.DataFrame) -> pd.DataFrame:
- out = pd.DataFrame(data=self.flags[0], index=data.index, columns=data.columns)
+ out = data.copy()
+ out[:] = self.flags[0]
# astype conversion of return Dataframe performed seperately, because pd.DataFrame(...,dtype=self.flags)
# wont give you categorical flag objects:
return out.astype(self.flags)
diff --git a/funcs/functions.py b/funcs/functions.py
index 9a23cf36b2ded4c3b2c95bc57b6ddce2106c8822..89af0ede3ff87c3301650c6e8519341991d43155 100644
--- a/funcs/functions.py
+++ b/funcs/functions.py
@@ -112,7 +112,7 @@ def flagSoilMoistureBySoilFrost(data, flags, field, flagger, soil_temp_reference
Soil temperatures below "frost_level" are regarded as denoting frozen soil state.
:param data: The pandas dataframe holding the data-to-be flagged, as well as the reference
- series. Dataframe should be indexed by a datetime series.
+ series. Data must be indexed by a datetime series.
:param flags: A dataframe holding the flags/flag-entries of "data"
:param field: Fieldname of the Soil moisture measurements field in data.
:param flagger: A flagger - object.
@@ -130,9 +130,8 @@ def flagSoilMoistureBySoilFrost(data, flags, field, flagger, soil_temp_reference
# retrieve reference series
refseries = data[soil_temp_reference]
ref_flags = flags[soil_temp_reference]
- ref_unflagged = flagger.isFlagged(ref_flags, flag=flagger.flags.unflagged())
- ref_min_flagged = flagger.isFlagged(ref_flags, flag=flagger.flags.min())
- ref_use = ref_min_flagged | ref_unflagged
+ ref_use = flagger.isFlagged(ref_flags, flag=flagger.flags.min()) | \
+ flagger.isFlagged(ref_flags, flag=flagger.flags.unflagged())
# drop flagged values:
refseries = refseries[ref_use.values]
# drop nan values from reference series, since those are values you dont want to refer to.
@@ -154,7 +153,7 @@ def flagSoilMoistureBySoilFrost(data, flags, field, flagger, soil_temp_reference
# make temporal frame holding dateindex, since df.apply cant access index
temp_frame = pd.Series(data.index)
- # get flagging mask ("False" denotes "bad"="test succesfull")
+ # get flagging mask ("True" denotes "bad"="test succesfull")
mask = temp_frame.apply(check_nearest_for_frost, args=(refseries,
tolerated_deviation, frost_level))
# apply calculated flags
@@ -163,7 +162,7 @@ def flagSoilMoistureBySoilFrost(data, flags, field, flagger, soil_temp_reference
return data, flags
-def flagSoilMoistureByPrecipitationEvents(data, flags, field, flagger, precipitation_reference, sensor_meas_depth=0,
+def flagSoilMoistureByPrecipitationEvents(data, flags, field, flagger, prec_reference, sensor_meas_depth=0,
sensor_accuracy=0, soil_porosity=0, **kwargs):
"""Function flags Soil moisture measurements by flagging moisture rises that do not follow up a sufficient
precipitation event. If measurement depth, sensor accuracy of the soil moisture sensor and the porosity of the
@@ -171,27 +170,61 @@ def flagSoilMoistureByPrecipitationEvents(data, flags, field, flagger, precipita
moisture raise within 24 hours, can be estimated. If those values are not delivered, this inferior bound is set
to zero. In that case, any non zero precipitation count will justify any soil moisture raise.
- :param data: The pandas dataframe, holding the data-to-be flagged.
+
+ :param data: The pandas dataframe holding the data-to-be flagged, as well as the reference
+ series. Data must be indexed by a datetime series.
:param flags: A dataframe holding the flags/flag-entries associated with "data".
:param field: Fieldname of the Soil moisture measurements field in data.
- :param flagger: A flagger - object.
- :param precipitation_reference: Fieldname of the precipitation meassurements in data.
- :param sensor_meas_depth: Measurement depth of the soil moisture sensor in meter [m].
- :param sensor_accuracy: Accuracy of the soil moisture sensor [-].
- :param soil_porosity: Porosity of moisture sensors surrounding soil.
+ :param flagger: A flagger - object. (saqc.flagger.X)
+ :param prec_reference: Fieldname of the precipitation meassurements column in data.
+ :param sensor_meas_depth: Measurement depth of the soil moisture sensor, [m].
+ :param sensor_accuracy: Accuracy of the soil moisture sensor, [-].
+ :param soil_porosity: Porosity of moisture sensors surrounding soil, [-].
"""
# retrieve data series input:
- dataseries = pd.Series(data[field].values, index=pd.to_datetime(data.index))
+ dataseries = data[field]
+ # "nan" suspicious values (neither "unflagged" nor "min-flagged")
+ data_flags = flags[field]
+ data_use = flagger.isFlagged(data_flags, flag=flagger.flags.min()) | \
+ flagger.isFlagged(data_flags, flag=flagger.flags.unflagged())
+ dataseries[~data_use] = np.nan
+ dataseries = dataseries.dropna()
+
+ # retrieve reference series input
+ refseries = data[prec_ref]
+ # "nan" suspicious values (neither "unflagged" nor "min-flagged")
+ ref_flags = flags[prec_reference]
+ ref_use = flagger.isFlagged(ref_flags, flag=flagger.flags.min()) | \
+ flagger.isFlagged(ref_flags, flag=flagger.flags.unflagged())
+ refseries[~ref_use] = np.nan
+
+ # estimate moisture sampling frequencie (the original series sampling rate may not match data-input sample rate):
+ scnds_series = (pd.Series(dataseries.index).diff().dt.total_seconds()).dropna()
+ hist = np.histogram(scnds_series, bins=scnds_series.size)
+ moist_rate = pd.tseries.frequencies.to_offset(str(int(hist[1][hist[0].argmax()])) + 's')
+
+ # resample dataseries to its original sampling rate
+ dataseries = dataseries.resample(moist_rate).asfreq()
+
+ # get 24 h prec. monitor (we dont exclude nans, since a 24h window with
+ prec_count = refseries.rolling(window='1D').sum()
+
+ # project it onto dataseries
+ prec_count = prec_count[dataseries.index]
+
+ # make raise and std. dev tester function:
+ def prec_test(x):
+ if x[field][-1] > x[field][-2]:
+ if (x[field][-1] - x[field][0]) > 2*x[field].std():
+ return x[prec_reference][-1] <= (sensor_meas_depth*soil_porosity*sensor_accuracy)
+ else:
+ return False
+ else:
+ return False
- # if reference series is part of input data frame, evaluate input data flags:
- flag_mask = flagger.isFlagged(flags)[precipitation_reference]
- # retrieve reference series
- refseries = pd.Series(data[precipitation_reference].values, index=pd.to_datetime(data.index))
- # drop flagged values:
- refseries = refseries.loc[~np.array(flag_mask)]
- # make refseries index a datetime thingy
- refseries.index = pd.to_datetime(refseries.index)
- # drop nan values from reference series, since those are values you dont want to refer to.
- refseries = refseries.dropna()
+ # get valid moisture raises:
+ valid_raises = refseries.rolling(window='1D', closed='both', min_periods=2)\
+ .apply(prec_test).astype(bool)
+ # get 24h valid std. dev for valid moisture raises
\ No newline at end of file