From a34757f93fc3122c836c44ac579ccbad0e3a4fd5 Mon Sep 17 00:00:00 2001
From: Bert Palm <bert.palm@ufz.de>
Date: Thu, 13 Feb 2020 17:16:14 +0100
Subject: [PATCH] fixed testset storage path issues
---
profiling/__init__.py | 3 ++-
profiling/generate_testsets.py | 30 ++++++++++++++++++++++--------
tests/tests.py | 4 +++-
3 files changed, 27 insertions(+), 10 deletions(-)
diff --git a/profiling/__init__.py b/profiling/__init__.py
index 139597f..34c8fe9 100644
--- a/profiling/__init__.py
+++ b/profiling/__init__.py
@@ -1,2 +1,3 @@
-
+from .generate_testsets import *
+from profiling.performance import find_index_range, gen_random_timestamps
diff --git a/profiling/generate_testsets.py b/profiling/generate_testsets.py
index 9ec68ba..9d95cc3 100644
--- a/profiling/generate_testsets.py
+++ b/profiling/generate_testsets.py
@@ -49,22 +49,36 @@ def get_random_df_and_dios(rowsz, colsz, freq='1min', disalign=True, randstart=T
return df, dios
-def get_testset(rows, cols, freq='1s', disalign=True, randstart=True, storagedir='testsets', noresult=False):
+def get_testset(rows, cols, freq='1s', disalign=True, randstart=True, storagedir=None, noresult=False):
+ if storagedir is None:
+ storagedir = os.path.dirname(__file__)
+ storagedir = os.path.join(storagedir, 'testsets')
+
fname = f'set_f{freq}_d{disalign}_r{randstart}_dim{rows}x{cols}.pkl'
fpath = os.path.join(storagedir, fname)
+
+ # try to get pickled data
try:
with open(fpath, 'rb') as fh:
if noresult:
return
tup = pickle.load(fh)
+
+ # file/data was present
+ return tup
except (pickle.UnpicklingError, FileNotFoundError):
- df, dios = _gen_testset(rowsz=rows, colsz=cols, freq=freq, disalign=disalign, randstart=randstart)
- df = df.sort_index(axis=0, level=0)
- df_type_a = df.copy().stack(dropna=False).sort_index(axis=0, level=0).copy()
- df_type_b = df.copy().unstack().sort_index(axis=0, level=0).copy()
- tup = df, df_type_a, df_type_b, dios
- with open(fpath, 'wb') as fh:
- pickle.dump(tup, fh)
+ pass
+
+ # generate testset(s)
+ df, dios = _gen_testset(rowsz=rows, colsz=cols, freq=freq, disalign=disalign, randstart=randstart)
+ df = df.sort_index(axis=0, level=0)
+ df_type_a = df.copy().stack(dropna=False).sort_index(axis=0, level=0).copy()
+ df_type_b = df.copy().unstack().sort_index(axis=0, level=0).copy()
+ tup = df, df_type_a, df_type_b, dios
+
+ # store testsets
+ with open(fpath, 'wb') as fh:
+ pickle.dump(tup, fh)
if noresult:
return
diff --git a/tests/tests.py b/tests/tests.py
index be2407e..c9c2f63 100644
--- a/tests/tests.py
+++ b/tests/tests.py
@@ -1,8 +1,10 @@
from dios import *
+from profiling import *
import pandas as pd
import datetime as dt
import numpy as np
+
v0 = 'var0'
v1 = 'var1'
v2 = 'var2'
@@ -135,7 +137,7 @@ def test_setitem():
def test_integrity():
rows = 1000
cols = 10
- df, _, _, dios = get_testset(1000, 10, storagedir='../dios/profiling/testsets')
+ df, _, _, dios = get_testset(1000, 10)
v = var_prefix + str(np.random.randint(0, cols))
t = find_index_range(dios)
--
GitLab