diff --git a/instantPASDy-GUI.py b/instantPASDy-GUI.py
index db069b1702d09c326f5a4d414e314d47d070a9e7..3488c3f16559fd20a95a644cc2a92aa952ede367 100644
--- a/instantPASDy-GUI.py
+++ b/instantPASDy-GUI.py
@@ -588,6 +588,8 @@ while True:
#config = configparser.ConfigParser()
window.Element('configfile').Update(window.Element('_configfile').Get())
configfile = window.Element('configfile').Get()
+
+ #config = PASDy.Config(configfile, verbose=1)
if configfile == '': configfile = 'config/config.cfg'
if os.path.isfile(configfile):
#temp = config.read(configfile)
diff --git a/instantPASDy.py b/instantPASDy.py
index d961f1ca4eec9d0311894fca8a7becd8c116cd94..2937d8524a8b9cb7be1d8ad46da879c46c65acb3 100644
--- a/instantPASDy.py
+++ b/instantPASDy.py
@@ -1,6 +1,11 @@
# Suppress nasty future warnings about pandas datetime conersion
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
+
+from PASDy.Files import *
+from PASDy.Spatiotemp import *
+from PASDy.Misc import *
+
#
from datetime import datetime, timedelta
import pytz
@@ -25,81 +30,36 @@ import simplekml
from glob import glob
#JJ packages used for calculating nearest neighbors
-from heapq import nsmallest, nlargest
-from scipy import stats
-from math import sin, cos, sqrt, atan2, radians
-import itertools
-from sklearn.neighbors import KDTree
+#from heapq import nsmallest, nlargest
+#from scipy import stats
+#from math import sin, cos, sqrt, atan2, radians
+#import itertools
+#from sklearn.neighbors import KDTree
# Overwrite print() with flush, i.e., no buffer
-_orig_print = print
-def print(*args, **kwargs):
- _orig_print(*args, flush=True, **kwargs)
+print = print_flush
+#_orig_print = print
+#def print(*args, **kwargs):
+# _orig_print(*args, flush=True, **kwargs)
#JJ recalculate strange coordinates to proper decimals
-def deg100min2dec(coord):
- coord = coord/100
- deg = np.floor(coord)
- minute = (coord - deg)/0.6
- dec = deg+minute
- return(dec)
+#def deg100min2dec(coord):
+# coord = coord/100
+# deg = np.floor(coord)
+# minute = (coord - deg)/0.6
+# dec = deg+minute
+# return(dec)
#JJ use x nearest neighbors for averaging
## find k closest lat/lon to every lat/lon coordinates
-def distance(pointA, pointB):
- return sqrt((pointB[0] - pointA[0]) ** 2 + (pointB[1] - pointA[1]) ** 2)
+#def distance(pointA, pointB):
+# return sqrt((pointB[0] - pointA[0]) ** 2 + (pointB[1] - pointA[1]) ** 2)
import zipfile
import os
-def distanceinkm(xstand,ystand,xfor,yfor): # approximate radius of earth in km
-
- R = 6373.0
- lat1 = radians(xstand)
- lon1 = radians(ystand)
- lat2 = radians(xfor)
- lon2 = radians(yfor)
-
- dlon = lon2 - lon1
- dlat = lat2 - lat1
- a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2
- c = 2 * atan2(sqrt(a), sqrt(1 - a))
- distance = R * c
- return(distance)
-
-def Neighbours(lat, lon, numpoints, data):
- numpoints = numpoints+1 # points itself included - therefore add one to get desired output
- points = np.vstack((lat, lon)).T # create numpy array from coordinates
-
- kdt = KDTree(points, leaf_size=30, metric='euclidean') # build tree with euclidean distances
- neighborpoints = kdt.query(points, k=numpoints, return_distance=False) # find the closest k points
-
- # calculate maximal distance within the k clostest coordinates
- neighbordist = []
- for i in range(len(points)):
- furthest = np.array(nlargest(1, points[neighborpoints[i]], lambda p : distance(p, (points[i])))) # find the coordinates for the k closest points
- neighbordist.append(distanceinkm(points[i][0],points[i][1], furthest[0][0],furthest[0][1])*1000)
-
- return neighbordist, neighborpoints
-
- #neighborpoints = [] # create empty list for all results
-
- #for i in range(len(points)): ## iterate for all points
- # nearest = np.array(nsmallest(numpoints, points, lambda p : distance(p, (points[i])))) # find the coordinates for the k closest points
- # print(nearest)
- # index = [] # create empty list for the index of the positions
-
- # for m in range(len(nearest)):
- # idx, rest = np.where(points==nearest[m]) # find the rows of these closest k coordinates
- # idx = stats.mode(idx) # calculate the mode - necessary because sometimes several outputs from above
- # index.append(idx.mode.tolist()) # create list from the k rows found
-
- # neighborpoints.append(index) # create list of k closest locations for all locations
- #for i in range(len(neighborpoints)):
- # neighborpoints[i] = list(itertools.chain(*neighborpoints[i])) # reduce for one nested list
-
@@ -313,26 +273,26 @@ class NM:
file_save_msg(file, '')
-def file_save_msg(file, name='File', end=''):
- size = os.stat(file).st_size
- if size < 1024: sizestr = "%.0f Bytes" % (size)
- elif size < 1024**2: sizestr = "%.0f KB" % (size/1024)
- else: sizestr = "%.1f MB" % (size/1024/1024)
- print("%s saved to %s (%s)" % (name, file, sizestr), end=end)
+#def file_save_msg(file, name='File', end=''):
+# size = os.stat(file).st_size
+# if size < 1024: sizestr = "%.0f Bytes" % (size)
+# elif size < 1024**2: sizestr = "%.0f KB" % (size/1024)
+# else: sizestr = "%.1f MB" % (size/1024/1024)
+# print("%s saved to %s (%s)" % (name, file, sizestr), end=end)
-def ifdef(a, b, convert=float):
- if not a:
- return(b)
- else:
- return(convert(a))
+#def ifdef(a, b, convert=float):
+# if not a:
+# return(b)
+# else:
+# return(convert(a))
-def xsplit(s, range=0, type=str):
- a = np.array(s.replace(',',' ').split(), dtype=type)
+#def xsplit(s, range=0, type=str):
+# a = np.array(s.replace(',',' ').split(), dtype=type)
# fill up with NaNs
- for i in np.arange(range-len(a)):
- a = np.append(a, np.nan)
- return(a)
+# for i in np.arange(range-len(a)):
+# a = np.append(a, np.nan)
+# return(a)
re_crnsdata = re.compile(r'^\d+\,')
re_dataselect = re.compile(r'//DataSelect\s*=\s*(\w+)')
@@ -407,10 +367,10 @@ def read(filename, archive=None, tz='UTC'):
return(datastr+"\n")
-def chomp(x):
- if x.endswith("\r\n"): return x[:-2]
- if x.endswith("\n") or x.endswith("\r"): return x[:-1]
- return(x)
+#def chomp(x):
+# if x.endswith("\r\n"): return x[:-2]
+# if x.endswith("\n") or x.endswith("\r"): return x[:-1]
+# return(x)
def read_header(filename, archive=None):
@@ -563,12 +523,12 @@ class Fig:
Fig.fig.savefig(bbox_inches='tight')
plt.close()
-def yesno2bool(s):
- s = s.lower()
- if s == 'yes' or s == 'y':
- return(True)
- else:
- return(False)
+#def yesno2bool(s):
+# s = s.lower()
+# if s == 'yes' or s == 'y':
+# return(True)
+# else:
+# return(False)
config = None
diff --git a/lib/CORN.py b/lib/CORN.py
deleted file mode 100644
index bec04f3674ba5c44f3a5a52495dbe2ad6d4d6938..0000000000000000000000000000000000000000
--- a/lib/CORN.py
+++ /dev/null
@@ -1 +0,0 @@
-import CORN
\ No newline at end of file
diff --git a/lib/PASDy/CORN.py b/lib/PASDy/CORN.py
new file mode 100644
index 0000000000000000000000000000000000000000..4baa30e1f1792ef98f3e190ec021c0f8febb458f
--- /dev/null
+++ b/lib/PASDy/CORN.py
@@ -0,0 +1,19 @@
+# CORNish PASDy
+
+
+## Configuration
+##
+### Inherit from PASDy.Config(...)
+###
+import PASDy.Config
+
+class Config(PASDy.Config.Config):
+
+ def __init__(self, *args, **kwargs):
+
+ # Set keys if not defined
+ if not 'unfold_numunits' in kwargs:
+ kwargs['unfold_numunits'] = ['aggregate']
+
+ # Call parent's init with the extended keys
+ super().__init__(*args, **kwargs)
\ No newline at end of file
diff --git a/lib/PASDy/Config.py b/lib/PASDy/Config.py
new file mode 100644
index 0000000000000000000000000000000000000000..31e4305892dd23e378b70c598baaedea2fffe6c3
--- /dev/null
+++ b/lib/PASDy/Config.py
@@ -0,0 +1,157 @@
+# PASDy.Config
+
+import os
+import re
+from configobj import ConfigObj # better alternative to configparser: preserves order and comments
+from copy import deepcopy
+
+import PASDy.Files as Files
+import PASDy.Misc as Misc
+from PASDy.Misc import printv
+
+#
+#
+class Config:
+
+ # config = PASDy.Config(file)
+ # creates a Config object from file.cfg
+
+ def __init__(self, file, default_file='config.cfg', encoding='cp850', verbose=2,
+ reprocess=True, clean_paths=True, convert_bool=True, unfold_range=True, join_lists=True,
+ unfold_numunits=[]):
+ self.file = file
+ self.default_file = default_file
+ self.encoding = encoding
+ self._verbose = verbose
+ self.reprocess = reprocess
+ self.clean_paths = clean_paths
+ self.convert_bool = convert_bool
+ self.unfold_range = unfold_range
+ self.join_lists = join_lists
+ self.unfold_numunits = unfold_numunits
+
+ self.c = None
+ self.read()
+
+ # Verbose print
+ def _print(self, level, s=''):
+ printv(level, self._verbose, s)
+
+ # Read
+ def read(self):
+ if self.file == '' or self.file is None or not self.file:
+ # os.path.abspath('') is better than __file__ most of the time, see https://stackoverflow.com/questions/39125532/file-does-not-exisit-in-jupyter-notebook
+ self.file = self.default_file
+ self._print(2, 'No file given, choosing default %s' % default_file)
+
+ self.c = config_read(self.file, self.encoding, self._verbose)
+
+ if not self.c is None:
+ self.sections = self.c.sections
+ if self.reprocess:
+ self.do_process()
+
+ # Process
+ def do_process(self):
+ self.cp = deepcopy(self.c)
+
+ for section in self.sections:
+ for k in self.c[section]:
+ v = self.c[section][k]
+
+ if 'path' in k:
+ if self.clean_paths:
+ self.cp[section][k] = Files.trailingslash(v)
+
+ elif isinstance(v, list):
+ if self.join_lists:
+ self.cp[section][k] = ', '.join(v)
+
+ elif v.lower() == 'yes' or v.lower() == 'no':
+ if self.convert_bool:
+ self.cp[section][k] = Misc.yesno2bool(v)
+
+ elif k.endswith('_range'):
+ if self.unfold_range:
+ if not isinstance(v, list):
+ v = ['','']
+ self.cp[section][k[:k.index('_range')]+'_min'] = v[0]
+ self.cp[section][k[:k.index('_range')]+'_max'] = v[1]
+
+ elif self.unfold_numunits:
+ if k in self.unfold_numunits:
+ if v:
+ v_bundle = re.search('^(\d+)([a-z]+)', v, re.IGNORECASE)
+ self.cp[section][k] = [v_bundle.group(1), v_bundle.group(2)]
+ else:
+ self.cp[section][k] = ['','']
+
+ self.cp[section][k+'_num'] = v[0]
+ self.cp[section][k+'_unit'] = v[1]
+
+ def undo_process(self):
+
+ for section in self.sections:
+ for k in self.cp[section]:
+ v = self.cp[section][k]
+
+ if isinstance(v, bool):
+ if self.convert_bool:
+ self.c[section][k] = Misc.bool2yesno(v)
+
+ elif ',' in v:
+ if self.join_lists:
+ self.c[section][k] = [x.strip() for x in v.split(',')]
+
+ elif k.endswith('_range'):
+ if self.unfold_range:
+ kcut = k[:k.index('_range')]
+ self.c[section][k] = [ self.cp[section][kcut+'_min'], self.cp[section][kcut+'_max'] ]
+ if self.cp[section][kcut+'_min'] == '':
+ self.c[section][k] = ''
+
+ elif self.unfold_numunits:
+ if k in self.unfold_numunits:
+ if self.cp[section][k+'_num'] == '':
+ self.c[section][k] = ''
+ else:
+ self.c[section][k] = self.cp[section][k+'_num'] + self.cp[section][k+'_unit']
+
+ # Save
+ def save(self, file=None):
+ self.undo_process()
+ # no argument given
+ if file is None: file = self.file
+ # Save
+ config_save(self.c, file, self._verbose)
+
+
+# Functions
+#
+## Read ConfigObj from file.cfg
+##
+def config_read(file, encoding='cp850', verbose=1):
+
+ if os.path.isfile(file):
+ printv(1, verbose, 'Reading sensor config: %s' % file)
+ return(ConfigObj(file, encoding=encoding))
+ else:
+ print(1, verbose, 'File does not exist: %s' % file)
+ return(None)
+
+
+## Save ConfigObj into file
+##
+def config_save(config, file, verbose=1):
+
+ if file == '' or file is None or not file:
+ # invalid file
+ printv(0, verbose, 'Cannot save config file: %s' % file)
+ return(False)
+ else:
+ config.filename = file
+ config.write_empty_values = True
+ config.write()
+ # TODO: check if write() was successful
+ Files.file_save_msg(file, name='Config')
+ return(True)
\ No newline at end of file
diff --git a/lib/PASDy/Files.py b/lib/PASDy/Files.py
new file mode 100644
index 0000000000000000000000000000000000000000..3d4ed9131ef006a4331a0546ab6c3ed65a5c3827
--- /dev/null
+++ b/lib/PASDy/Files.py
@@ -0,0 +1,30 @@
+# PASDy.Files
+
+# Formatting
+#
+### Path trailing /
+def trailingslash(p):
+ import os
+ # add / to folder
+ if os.path.isdir(p):
+ return(os.path.join(p, ''))
+ # remove / from file
+ elif os.path.isfile(p):
+ return(p.rstrip('\\').rstrip('/'))
+ else:
+ return(p)
+
+
+## Output
+##
+### Print newly created file name and size
+###
+def file_save_msg(file, name='File', end=''):
+ import os
+ size = os.stat(file).st_size
+ if size < 1024: sizestr = "%.0f Bytes" % (size)
+ elif size < 1024**2: sizestr = "%.0f KB" % (size/1024)
+ else: sizestr = "%.1f MB" % (size/1024/1024)
+ print("%s saved to %s (%s)" % (name, file, sizestr), end=end)
+
+
diff --git a/lib/PASDY.py b/lib/PASDy/Misc.py
similarity index 62%
rename from lib/PASDY.py
rename to lib/PASDy/Misc.py
index 8bae7441f516989b496dab33e3f0217332149a52..01c22c9d124eccbb4092af11b3284697350276ca 100644
--- a/lib/PASDY.py
+++ b/lib/PASDy/Misc.py
@@ -1,4 +1,4 @@
-import PASDY
+#import PASDY
# General functions
@@ -8,6 +8,12 @@ _orig_print = print
def print_flush(*args, **kwargs):
_orig_print(*args, flush=True, **kwargs)
+## Verbose print
+##
+def printv(v=1, v_ref=1, s=''):
+ if v <= v_ref:
+ print(s)
+
## System calls
##
### Launch a file using its corresponding default application
@@ -39,6 +45,7 @@ def ifdef(a, b, convert=float):
### Append some NaNs
###
def xsplit(s, range=0, type=str):
+ import numpy as np
a = np.array(s.replace(',',' ').split(), dtype=type)
# fill up with NaNs
for i in np.arange(range-len(a)):
@@ -74,36 +81,3 @@ def bool2yesno(boo):
else:
return('no')
-## Output
-##
-### Print newly created file name and size
-###
-def file_save_msg(file, name='File', end=''):
- size = os.stat(file).st_size
- if size < 1024: sizestr = "%.0f Bytes" % (size)
- elif size < 1024**2: sizestr = "%.0f KB" % (size/1024)
- else: sizestr = "%.1f MB" % (size/1024/1024)
- print("%s saved to %s (%s)" % (name, file, sizestr), end=end)
-
-
-# Spatial functions
-
-## Distance
-##
-### use x nearest neighbors for averaging
-### find k closest lat/lon to every lat/lon coordinates
-###
-def distance(pointA, pointB):
- return sqrt((pointB[0] - pointA[0]) ** 2 + (pointB[1] - pointA[1]) ** 2)
-
-
-## Coordinate conversion
-##
-### recalculate strange coordinates to proper decimals
-###
-def deg100min2dec(coord):
- coord = coord/100
- deg = np.floor(coord)
- minute = (coord - deg)/0.6
- dec = deg+minute
- return(dec)
diff --git a/lib/PASDy/NM.py b/lib/PASDy/NM.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e8a8f57323a65b9eca29517fa19d18af1f63f9a
--- /dev/null
+++ b/lib/PASDy/NM.py
@@ -0,0 +1,3 @@
+
+def whoami():
+ print("I am NM!")
\ No newline at end of file
diff --git a/lib/PASDy/Spatiotemp.py b/lib/PASDy/Spatiotemp.py
new file mode 100644
index 0000000000000000000000000000000000000000..8c5d41dfe109439411e7b03b2b8af9c6207a796c
--- /dev/null
+++ b/lib/PASDy/Spatiotemp.py
@@ -0,0 +1,92 @@
+# PASDy.Spatiotemp -- Spatiotemporal data analysis
+
+import numpy as np
+
+
+# Spatial functions
+
+## Distance
+##
+### use x nearest neighbors for averaging
+### find k closest lat/lon to every lat/lon coordinates
+###
+def distance(pointA, pointB):
+ return sqrt((pointB[0] - pointA[0]) ** 2 + (pointB[1] - pointA[1]) ** 2)
+
+
+## Coordinate conversion
+##
+### recalculate strange coordinates to proper decimals
+###
+def deg100min2dec(coord):
+ import numpy
+ coord = coord/100
+ deg = numpy.floor(coord)
+ minute = (coord - deg)/0.6
+ dec = deg+minute
+ return(dec)
+
+
+# Nearest Neighbor
+#
+## JJ packages used for calculating nearest neighbors
+##
+from heapq import nsmallest, nlargest
+from scipy import stats
+from math import sin, cos, sqrt, atan2, radians
+import itertools
+from sklearn.neighbors import KDTree
+
+
+### From lat, lon get distance in km
+###
+def distanceinkm(xstand, ystand, xfor, yfor): # approximate radius of earth in km
+
+ R = 6373.0
+ lat1 = radians(xstand)
+ lon1 = radians(ystand)
+ lat2 = radians(xfor)
+ lon2 = radians(yfor)
+
+ dlon = lon2 - lon1
+ dlat = lat2 - lat1
+ a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2
+ c = 2 * atan2(sqrt(a), sqrt(1 - a))
+ distance = R * c
+
+ return(distance)
+
+###
+###
+def Neighbours(lat, lon, numpoints, data):
+
+ numpoints = numpoints+1 # points itself included - therefore add one to get desired output
+ points = np.vstack((lat, lon)).T # create numpy array from coordinates
+
+ kdt = KDTree(points, leaf_size=30, metric='euclidean') # build tree with euclidean distances
+ neighborpoints = kdt.query(points, k=numpoints, return_distance=False) # find the closest k points
+
+ # calculate maximal distance within the k clostest coordinates
+ neighbordist = []
+ for i in range(len(points)):
+ furthest = np.array(nlargest(1, points[neighborpoints[i]], lambda p : distance(p, (points[i])))) # find the coordinates for the k closest points
+ neighbordist.append(distanceinkm(points[i][0],points[i][1], furthest[0][0],furthest[0][1])*1000)
+
+ return neighbordist, neighborpoints
+
+ #neighborpoints = [] # create empty list for all results
+
+ #for i in range(len(points)): ## iterate for all points
+ # nearest = np.array(nsmallest(numpoints, points, lambda p : distance(p, (points[i])))) # find the coordinates for the k closest points
+ # print(nearest)
+ # index = [] # create empty list for the index of the positions
+
+ # for m in range(len(nearest)):
+ # idx, rest = np.where(points==nearest[m]) # find the rows of these closest k coordinates
+ # idx = stats.mode(idx) # calculate the mode - necessary because sometimes several outputs from above
+ # index.append(idx.mode.tolist()) # create list from the k rows found
+
+ # neighborpoints.append(index) # create list of k closest locations for all locations
+ #for i in range(len(neighborpoints)):
+ # neighborpoints[i] = list(itertools.chain(*neighborpoints[i])) # reduce for one nested list
+
diff --git a/playground.py b/playground.py
new file mode 100644
index 0000000000000000000000000000000000000000..471db42197eafba9ed36bfb1d21f3c8ef44a94d3
--- /dev/null
+++ b/playground.py
@@ -0,0 +1,4 @@
+import NMxxx
+
+print("hello")
+NMxxx.whoami()
\ No newline at end of file
diff --git a/test/test.py b/test/test.py
index f32c31a545703e2c093fbd0384870758c583034f..f57ebdc63d2d5a1022eb66b4591673a8f8777cdd 100644
--- a/test/test.py
+++ b/test/test.py
@@ -4,7 +4,7 @@ from glob import glob
test_cases = [
["python", "../instantPASDy.py", "input/station.cfg", "--nonstop"],
- ["python", "../instantPASDy.py", "input/JJ-config_rover.cfg", "--nonstop"]
+# ["python", "../instantPASDy.py", "input/JJ-config_rover.cfg", "--nonstop"]
]
i = 0