diff --git a/profiling/__init__.py b/profiling/__init__.py index 139597f9cb07c5d48bed18984ec4747f4b4f3438..34c8fe901fd5efd3428df1304e8159bc5f173c3c 100644 --- a/profiling/__init__.py +++ b/profiling/__init__.py @@ -1,2 +1,3 @@ - +from .generate_testsets import * +from profiling.performance import find_index_range, gen_random_timestamps diff --git a/profiling/generate_testsets.py b/profiling/generate_testsets.py index 9ec68ba9fb964249b3d13f1441212618680c0e60..9d95cc3405fa7be21ab474302cd237f3e7a54489 100644 --- a/profiling/generate_testsets.py +++ b/profiling/generate_testsets.py @@ -49,22 +49,36 @@ def get_random_df_and_dios(rowsz, colsz, freq='1min', disalign=True, randstart=T return df, dios -def get_testset(rows, cols, freq='1s', disalign=True, randstart=True, storagedir='testsets', noresult=False): +def get_testset(rows, cols, freq='1s', disalign=True, randstart=True, storagedir=None, noresult=False): + if storagedir is None: + storagedir = os.path.dirname(__file__) + storagedir = os.path.join(storagedir, 'testsets') + fname = f'set_f{freq}_d{disalign}_r{randstart}_dim{rows}x{cols}.pkl' fpath = os.path.join(storagedir, fname) + + # try to get pickled data try: with open(fpath, 'rb') as fh: if noresult: return tup = pickle.load(fh) + + # file/data was present + return tup except (pickle.UnpicklingError, FileNotFoundError): - df, dios = _gen_testset(rowsz=rows, colsz=cols, freq=freq, disalign=disalign, randstart=randstart) - df = df.sort_index(axis=0, level=0) - df_type_a = df.copy().stack(dropna=False).sort_index(axis=0, level=0).copy() - df_type_b = df.copy().unstack().sort_index(axis=0, level=0).copy() - tup = df, df_type_a, df_type_b, dios - with open(fpath, 'wb') as fh: - pickle.dump(tup, fh) + pass + + # generate testset(s) + df, dios = _gen_testset(rowsz=rows, colsz=cols, freq=freq, disalign=disalign, randstart=randstart) + df = df.sort_index(axis=0, level=0) + df_type_a = df.copy().stack(dropna=False).sort_index(axis=0, level=0).copy() + df_type_b = df.copy().unstack().sort_index(axis=0, level=0).copy() + tup = df, df_type_a, df_type_b, dios + + # store testsets + with open(fpath, 'wb') as fh: + pickle.dump(tup, fh) if noresult: return diff --git a/tests/tests.py b/tests/tests.py index be2407e86f50ce5b8fdba35d57e781ee31af628a..c9c2f63f31ae768392a3bccb08642405a8df3d54 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,8 +1,10 @@ from dios import * +from profiling import * import pandas as pd import datetime as dt import numpy as np + v0 = 'var0' v1 = 'var1' v2 = 'var2' @@ -135,7 +137,7 @@ def test_setitem(): def test_integrity(): rows = 1000 cols = 10 - df, _, _, dios = get_testset(1000, 10, storagedir='../dios/profiling/testsets') + df, _, _, dios = get_testset(1000, 10) v = var_prefix + str(np.random.randint(0, cols)) t = find_index_range(dios)