Transfer ICOS data to level 1

level1/do_aggregate_soilnet.py

@@ -53,7 +53,7 @@ def aggregatePeriod(df):
     return pd.Series(out)
 
 
-def aggregateData(df, freq="30T"):
+def aggregateData(df, freq):
     df = df.sort_index(axis=1)
     groups = (reindexTable(df)
               .astype(float)
@@ -64,56 +64,63 @@ def aggregateData(df, freq="30T"):
     return out
 
 
-def _buildFname(device, date, frmt):
-    ftemplate = "SoilNet_Data_{station}_30min_{box}_{date}.level2.csv"
+def _buildFname(device, date, frmt, freq):
+    ftemplate = "SoilNet_Data_{station}_{freq}_{box}_{date}.level2.csv"
     return ftemplate.format(
         station=device.station_key, box=device.logger_key,
-        date=datetime.strftime(date, frmt))
+        date=datetime.strftime(date, frmt), freq=freq)
 
 
-def dailyPath(device, date):
+def dailyPath(device, date, freq):
     return Path(
         device.derivedpath,
         str(date.year),
-        _buildFname(device, date, "%Y%m%d"))
+        _buildFname(device, date, "%Y%m%d", freq))
 
 
-def yearlyPath(device, date):
+def yearlyPath(device, date, freq):
     return Path(
         device.derivedpath,
-        _buildFname(device, date, "%Y"))
+        _buildFname(device, date, "%Y", freq))
 
 
-def writeData(data, device, freq, pathFunc, logger):
-    for date, out in iterDataFreq(data, freq):
-        fname = pathFunc(device, date)
+def writeData(data, device, freq_write, pathFunc, logger, freq):
+    for date, out in iterDataFreq(data, freq_write):
+        fname = pathFunc(device, date, freq)
         logger.debug("writing: %s", str(fname))
         writeTable(fname, out, make_path=True)
 
 
-def main(station, device, start_date, end_date, debug):
+def main(start_date, end_date, station=None, logger=None,
+         debug=False, ICOS=False, freq="30T"):
 
-    with initLogger(__file__, debug) as logger:
+    with initLogger(__file__, debug) as log:
         devices = getDevices(
-            station=station, logger=device, tag="soilnet",
+            station=station, logger=logger, tag="soilnet",
             start_date=firstOfYear(start_date),
             end_date=lastOfYear(end_date))
 
+        if ICOS:
+            devices = getDevices(
+                station=station, logger=logger, tag="soilnet_icos",
+                start_date=firstOfYear(start_date),
+                end_date=lastOfYear(end_date))
+
         for device in devices:
             msg = "soilnet aggregation failed for: {:}".format(
                 device)
-            with exceptionLogged(logger, msg):
+            with exceptionLogged(log, msg):
                 data = device.getL1Data()
                 if data.empty:
                     continue
-                logger.info("aggregating: %s", device)
-                agg_data = aggregateData(data)
+                log.info("aggregating: %s", device)
+                agg_data = aggregateData(data, freq)
                 writeData(agg_data[start_date:end_date], device,
-                          freq="daily",
-                          pathFunc=dailyPath, logger=logger)
+                          freq_write="daily",
+                          pathFunc=dailyPath, logger=log, freq=freq)
                 # could be hidden behind a flag
-                writeData(agg_data, device, freq="yearly",
-                          pathFunc=yearlyPath, logger=logger)
+                writeData(agg_data, device, freq_write="yearly",
+                          pathFunc=yearlyPath, logger=log, freq=freq)
 
 
 if __name__ == '__main__':
@@ -122,6 +129,6 @@ if __name__ == '__main__':
     parser = ChainParser("Aggregate soilnet data to 30 minutes", {"ndays": 1})
     args = parser.parse_args()
 
-    main(station=args.station, device=args.logger,
-         start_date=args.start_date, end_date=args.end_date,
-         debug=args.debug)
+    main(start_date=args.start_date, end_date=args.end_date,
+         station=args.station, logger=args.logger, debug=args.debug,
+         ICOS=args.ICOS, freq=args.freq)