add benchmarking for data to and from disk and a run with memory limit

benchmarks/data/test_tools.py

@@ -69,6 +69,27 @@ class TestPrepare(unittest.TestCase):
         xdata = full(0.0, info)
         _result = self.benchmark(prepare, data=xdata, info=info)
 
+    @pytest.mark.benchmark(group="data-tools")
+    def test_to_xarray_np_units_01_2x1(self):
+        time = dt.datetime(2000, 1, 1)
+        info = fm.Info(time=time, grid=fm.UniformGrid((2, 1)), units="m")
+        xdata = full(0.0, info).magnitude
+        _result = self.benchmark(to_xarray, data=xdata, info=info)
+
+    @pytest.mark.benchmark(group="data-tools")
+    def test_to_xarray_np_units_02_512x256(self):
+        time = dt.datetime(2000, 1, 1)
+        info = fm.Info(time=time, grid=fm.UniformGrid((512, 256)), units="m")
+        xdata = full(0.0, info).magnitude
+        _result = self.benchmark(to_xarray, data=xdata, info=info)
+
+    @pytest.mark.benchmark(group="data-tools")
+    def test_to_xarray_np_units_03_2048x1024(self):
+        time = dt.datetime(2000, 1, 1)
+        info = fm.Info(time=time, grid=fm.UniformGrid((2048, 1024)), units="m")
+        xdata = full(0.0, info).magnitude
+        _result = self.benchmark(to_xarray, data=xdata, info=info)
+
     @pytest.mark.benchmark(group="data-tools-slow")
     def test_cp_prepare_np_01_2x1(self):
         time = dt.datetime(2000, 1, 1)

benchmarks/numpy/test_save_load.py

+import os.path
+import tempfile
+import unittest
+
+import numpy as np
+import pytest
+
+import finam as fm
+
+
+class TestCreateUniform(unittest.TestCase):
+    @pytest.fixture(autouse=True)
+    def setupBenchmark(self, benchmark):
+        self.benchmark = benchmark
+
+    @pytest.mark.benchmark(group="np-save-load")
+    def test_save_01_64x32(self):
+        xdata = np.full((1, 64, 32), 1.0, dtype=np.dtype(np.float64))
+        with tempfile.TemporaryDirectory() as d:
+            fp = os.path.join(d, "temp.npy")
+            _result = self.benchmark(np.save, file=fp, arr=xdata)
+
+    @pytest.mark.benchmark(group="np-save-load")
+    def test_save_02_512x256(self):
+        xdata = np.full((1, 512, 256), 1.0, dtype=np.dtype(np.float64))
+        with tempfile.TemporaryDirectory() as d:
+            fp = os.path.join(d, "temp.npy")
+            _result = self.benchmark(np.save, file=fp, arr=xdata)
+
+    @pytest.mark.benchmark(group="np-save-load")
+    def test_save_03_1024x512(self):
+        xdata = np.full((1, 1024, 512), 1.0, dtype=np.dtype(np.float64))
+        with tempfile.TemporaryDirectory() as d:
+            fp = os.path.join(d, "temp.npy")
+            _result = self.benchmark(np.save, file=fp, arr=xdata)
+
+    @pytest.mark.benchmark(group="np-save-load")
+    def test_save_04_2048x1024(self):
+        xdata = np.full((1, 2048, 1024), 1.0, dtype=np.dtype(np.float64))
+        with tempfile.TemporaryDirectory() as d:
+            fp = os.path.join(d, "temp.npy")
+            _result = self.benchmark(np.save, file=fp, arr=xdata)
+
+    @pytest.mark.benchmark(group="np-save-load")
+    def test_load_01_64x32(self):
+        xdata = np.full((1, 64, 32), 1.0, dtype=np.dtype(np.float64))
+        with tempfile.TemporaryDirectory() as d:
+            fp = os.path.join(d, "temp.npy")
+            np.save(fp, xdata)
+            _result = self.benchmark(np.load, file=fp)
+
+    @pytest.mark.benchmark(group="np-save-load")
+    def test_load_02_512x256(self):
+        xdata = np.full((1, 512, 256), 1.0, dtype=np.dtype(np.float64))
+        with tempfile.TemporaryDirectory() as d:
+            fp = os.path.join(d, "temp.npy")
+            np.save(fp, xdata)
+            _result = self.benchmark(np.load, file=fp)
+
+    @pytest.mark.benchmark(group="np-save-load")
+    def test_load_03_1024x512(self):
+        xdata = np.full((1, 1024, 512), 1.0, dtype=np.dtype(np.float64))
+        with tempfile.TemporaryDirectory() as d:
+            fp = os.path.join(d, "temp.npy")
+            np.save(fp, xdata)
+            _result = self.benchmark(np.load, file=fp)
+
+    @pytest.mark.benchmark(group="np-save-load")
+    def test_load_04_2048x1024(self):
+        xdata = np.full((1, 2048, 1024), 1.0, dtype=np.dtype(np.float64))
+        with tempfile.TemporaryDirectory() as d:
+            fp = os.path.join(d, "temp.npy")
+            np.save(fp, xdata)
+            _result = self.benchmark(np.load, file=fp)

benchmarks/profiling/mem_huge_memory.py → benchmarks/profiling/profile_huge_memory.py

+import cProfile
 import datetime as dt
+import io
+import pstats
+import sys
+import time
 
 import numpy as np
 
 import finam as fm
 
-if __name__ == "__main__":
+
+def run_model():
+    t = time.time()
+
     start_time = dt.datetime(2000, 1, 1)
     end_time = dt.datetime(2002, 1, 1)
 
@@ -29,9 +37,23 @@ if __name__ == "__main__":
         step=dt.timedelta(days=365),
     )
 
-    composition = fm.Composition([source, sink])
+    composition = fm.Composition([source, sink], slot_memory_limit=500 * 2**20)
     composition.initialize()
 
     source["Out"] >> sink["In"]
 
     composition.run(end_time=end_time)
+
+    print("Total time:", time.time() - t)
+
+
+if __name__ == "__main__":
+    pr = cProfile.Profile()
+    pr.enable()
+
+    run_model()
+
+    pr.disable()
+    s = io.StringIO()
+    ps = pstats.Stats(pr, stream=s).sort_stats(pstats.SortKey.CUMULATIVE)
+    ps.dump_stats(sys.argv[1])