diff --git a/corny/basics.py b/corny/basics.py index b2e9b723507e9e1a9e76fc8f41c739cb35105f9f..e781cb6d962aae63494b968ff4a3320dfa122bdd 100644 --- a/corny/basics.py +++ b/corny/basics.py @@ -371,6 +371,7 @@ def ftp_is_file(FTP, object): def download(archive_file='', source='FTP', server=None, user=None, pswd=None, ftp_prefix='', ftp_suffix='', sd_path='', sd_prefix='', sd_suffix='', + http_url='', http_filename='', update_all=False): # if a folder was given @@ -446,6 +447,14 @@ def download(archive_file='', source='FTP', server=None, user=None, pswd=None, remote_files['name'] = remote_list remote_files['size'] = [os.stat(sd_path+'/'+file).st_size for file in remote_list] + elif source=='HTTP': + # Select new files from FTP + from .exdata.http import download_http + download_http(http_url, http_filename) + remote_list = [http_filename] + remote_files = pandas.DataFrame() + remote_files['name'] = [http_filename] + remote_files['size'] = [os.stat(http_filename).st_size] elif source=='DMP': # Select new files from DMP @@ -483,6 +492,9 @@ def download(archive_file='', source='FTP', server=None, user=None, pswd=None, remote.retrbinary("RETR " + filename, memory.write) # FTP->memory archive.writestr(filename, memory.getvalue()) # memory->archive + elif source=='HTTP': + archive.write(filename, os.path.basename(filename)) + elif source=='SD': archive.write(os.path.join(sd_path,'') + filename, filename) diff --git a/corny/exdata/http.py b/corny/exdata/http.py new file mode 100644 index 0000000000000000000000000000000000000000..394a99e2f0b0d0da64fad8e20f1c1d0e51e72bc6 --- /dev/null +++ b/corny/exdata/http.py @@ -0,0 +1,64 @@ + +import requests +import re +import os + + +def file_save_msg(file, name='File', end=''): + size = os.stat(file).st_size + if size < 1024: sizestr = "%.0f Bytes" % (size) + elif size < 1024**2: sizestr = "%.0f KB" % (size/1024) + else: sizestr = "%.1f MB" % (size/1024/1024) + print("< %s saved to %s (%s)" % (name, file, sizestr), end=end) + + +def make_url(id): + url = "http://www.somedataserver.com/csvdata.php?ID=%s" \ + % (id) + return(url) + +def download_http( + url, + file, + head_re = r'^\#', + data_re = r'^\d', + verbose = True + ): + """ + Download data from HTTP website + + Args: + url (str): URL of the website + file (str): filename or None (then return) + + Usage: + id = 999 + file = "data/%0.f.csv" % id + url = make_url(id) + download_http(url, file) + """ + + if verbose: print('> Requesting URL: ' + url) + + try: + r = requests.get(url) + except Exception as e: + print('! Requesting HTTP data failed.') + print(e) + r = None + + if r: + if verbose: print('i Received %s characters.' % len(r.text)) + + # Write into file + re_head_line = re.compile(head_re) + re_data_line = re.compile(data_re) + + with open(file, 'w') as f: + for line in r.text.splitlines(): + if re_head_line.search(line): + f.write(line + "\n") + elif re_data_line.search(line): + f.write(line + "\n") + + if verbose: file_save_msg(file, end="\n") diff --git a/default.cfg b/default.cfg index 53b2d130b5c3e91e0739881a7eb7fbc2e8480055..589d2a01eb6f6f99d77cc66ebc8bfb20bf029d97 100644 --- a/default.cfg +++ b/default.cfg @@ -22,7 +22,7 @@ coords = ## Source # The data source to be checked can be one or a combination of the following (seperate by comma): FTP (download files from an FTP server), SD (copy files from a remote folder or SD drive). Leave blank to skip updates. -# Units: SD, FTP +# Units: SD, FTP, HTTP data_source = ## FTP server credentials @@ -43,6 +43,12 @@ SD_prefix = # Last letters of the target path. Example: .RV1 SD_suffix = +## HTTP website as a source +# URL from which data should be downloaded +http_url = +# Local filename to write downloaded data +http_filename = + [input] ## Local storage diff --git a/instantPASDy.py b/instantPASDy.py index 11716a374823c1db79c4e42da78eec5de29e8ee2..7e25a62aec2ae94d01bdc6c72ac7127bed316f44 100644 --- a/instantPASDy.py +++ b/instantPASDy.py @@ -72,6 +72,8 @@ def main(configfile=None, cfg_prefix=None): pswd = config['update']['FTP_pswd'], ftp_prefix = config['update']['FTP_prefix'], ftp_suffix = config['update']['FTP_suffix'], sd_path = config['update']['SD_path'], sd_prefix = config['update']['SD_prefix'], sd_suffix = config['update']['SD_suffix'], + http_url = gc(config,'update','http_url'), + http_filename= gc(config,'update','http_filename'), update_all = yesno2bool(gc(config,'update','update_all','no'))) coordinates = xsplit(gc(config,'info','coords'), range=3, type=float)