From a09002a4233852b642fd48865cfae21d25066a93 Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:02:07 -0400 Subject: [PATCH] Setup benchmark suite Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --- .gitignore | 5 +++ Makefile | 23 ++++++++++++ ci/benchmarks/benchmarks.json | 33 ++++++++++++++++ conda/dev-environment-unix.yml | 1 + conda/dev-environment-win.yml | 1 + csp/benchmarks/__init__.py | 0 csp/benchmarks/asv.conf.jsonc | 33 ++++++++++++++++ csp/benchmarks/stats/__init__.py | 0 csp/benchmarks/stats/basic.py | 64 ++++++++++++++++++++++++++++++++ pyproject.toml | 2 + 10 files changed, 162 insertions(+) create mode 100644 ci/benchmarks/benchmarks.json create mode 100644 csp/benchmarks/__init__.py create mode 100644 csp/benchmarks/asv.conf.jsonc create mode 100644 csp/benchmarks/stats/__init__.py create mode 100644 csp/benchmarks/stats/basic.py diff --git a/.gitignore b/.gitignore index 7c508dbd..3b6280a2 100644 --- a/.gitignore +++ b/.gitignore @@ -104,6 +104,11 @@ csp/lib/ *.so *.tsbuildinfo +# Benchmarks +.asv +ci/benchmarks/* +!ci/benchmarks/benchmarks.json + # Jupyter / Editors .ipynb_checkpoints .autoversion diff --git a/Makefile b/Makefile index d37aa8f6..07742fd1 100644 --- a/Makefile +++ b/Makefile @@ -118,6 +118,29 @@ dockerps: ## spin up docker compose services for adapter testing dockerdown: ## spin up docker compose services for adapter testing $(DOCKER) compose -f ci/$(ADAPTER)/docker-compose.yml down +############## +# BENCHMARKS # +############## +.PHONY: benchmark benchmarks benchmark-regen benchmark-view benchmarks-regen benchmarks-view +benchmark: ## run benchmarks + python -m asv run --config csp/benchmarks/asv.conf.jsonc --verbose `git rev-parse HEAD` + +# https://github.com/airspeed-velocity/asv/issues/1027 +# https://github.com/airspeed-velocity/asv/issues/488 +benchmark-regen: + python -m asv run --config csp/benchmarks/asv.conf.jsonc --verbose v0.0.3^! + python -m asv run --config csp/benchmarks/asv.conf.jsonc --verbose v0.0.4^! + python -m asv run --config csp/benchmarks/asv.conf.jsonc --verbose v0.0.5^! + +benchmark-view: ## generate viewable website of benchmark results + python -m asv publish --config csp/benchmarks/asv.conf.jsonc + python -m asv preview --config csp/benchmarks/asv.conf.jsonc + +# Alias +benchmarks: benchmark +benchmarks-regen: benchmark-regen +benchmarks-view: benchmark-view + ########### # VERSION # ########### diff --git a/ci/benchmarks/benchmarks.json b/ci/benchmarks/benchmarks.json new file mode 100644 index 00000000..48924a9a --- /dev/null +++ b/ci/benchmarks/benchmarks.json @@ -0,0 +1,33 @@ +{ + "stats.basic.StatsBenchmarkSuite.time_stats_qtl": { + "code": "class StatsBenchmarkSuite:\n def time_stats_qtl(self):\n def g_qtl():\n data = csp.curve(typ=np.ndarray, data=self.DATA)\n median = csp.stats.median(data, interval=self.INTERVAL)\n csp.add_graph_output(\"final_median\", median, tick_count=1)\n \n qtl_times = []\n \n for _ in range(self.NUM_SAMPLES):\n start = time.time()\n csp.run(g_qtl, realtime=False, starttime=self.st, endtime=timedelta(seconds=self.N))\n post_qtl = time.time()\n qtl_times.append(post_qtl - start)\n \n avg_med = sum(qtl_times) / self.NUM_SAMPLES\n print(\n f\"Average time in {self.NUM_SAMPLES} tests for median with {self.N=}, {self.ARRAY_SIZE=}, {self.INTERVAL=}: {round(avg_med, 2)} s\"\n )\n return avg_med\n\n def setup(self):\n self.st = datetime(2020, 1, 1)\n self.N = 1_000\n self.ARRAY_SIZE = 100\n self.TEST_TIMES = [self.st + timedelta(seconds=i) for i in range(self.N)]\n self.RANDOM_VALUES = [np.random.normal(size=(self.ARRAY_SIZE,)) for i in range(self.N)] # 100 element np array\n self.DATA = list(zip(self.TEST_TIMES, self.RANDOM_VALUES))\n self.INTERVAL = 500\n self.NUM_SAMPLES = 100", + "min_run_count": 2, + "name": "stats.basic.StatsBenchmarkSuite.time_stats_qtl", + "number": 0, + "param_names": [], + "params": [], + "repeat": 0, + "rounds": 2, + "sample_time": 0.01, + "type": "time", + "unit": "seconds", + "version": "21f280e4eeceac0ca2172bed432939c57f2b2618bd26bd27d15d4ca177e2ab26", + "warmup_time": -1 + }, + "stats.basic.StatsBenchmarkSuite.time_stats_rank": { + "code": "class StatsBenchmarkSuite:\n def time_stats_rank(self):\n def g_rank():\n data = csp.curve(typ=np.ndarray, data=self.DATA)\n rank = csp.stats.rank(data, interval=self.INTERVAL)\n csp.add_graph_output(\"final_rank\", rank, tick_count=1)\n \n rank_times = []\n \n for _ in range(self.NUM_SAMPLES):\n start = time.time()\n csp.run(g_rank, realtime=False, starttime=self.st, endtime=timedelta(seconds=self.N))\n post_rank = time.time()\n rank_times.append(post_rank - start)\n \n avg_rank = sum(rank_times) / self.NUM_SAMPLES\n print(\n f\"Average time in {self.NUM_SAMPLES} tests for rank with {self.N=}, {self.ARRAY_SIZE=}, {self.INTERVAL=}: {round(avg_rank, 2)} s\"\n )\n return avg_rank\n\n def setup(self):\n self.st = datetime(2020, 1, 1)\n self.N = 1_000\n self.ARRAY_SIZE = 100\n self.TEST_TIMES = [self.st + timedelta(seconds=i) for i in range(self.N)]\n self.RANDOM_VALUES = [np.random.normal(size=(self.ARRAY_SIZE,)) for i in range(self.N)] # 100 element np array\n self.DATA = list(zip(self.TEST_TIMES, self.RANDOM_VALUES))\n self.INTERVAL = 500\n self.NUM_SAMPLES = 100", + "min_run_count": 2, + "name": "stats.basic.StatsBenchmarkSuite.time_stats_rank", + "number": 0, + "param_names": [], + "params": [], + "repeat": 0, + "rounds": 2, + "sample_time": 0.01, + "type": "time", + "unit": "seconds", + "version": "4c302ccf942084ac2367999fc84b2ba882c2ff74cddd80a3c27c8f8a1aee333d", + "warmup_time": -1 + }, + "version": 2 +} \ No newline at end of file diff --git a/conda/dev-environment-unix.yml b/conda/dev-environment-unix.yml index fa3740a5..618210d2 100644 --- a/conda/dev-environment-unix.yml +++ b/conda/dev-environment-unix.yml @@ -3,6 +3,7 @@ channels: - conda-forge - nodefaults dependencies: + - asv - bison - brotli - build diff --git a/conda/dev-environment-win.yml b/conda/dev-environment-win.yml index 8ca2482d..89f8cb98 100644 --- a/conda/dev-environment-win.yml +++ b/conda/dev-environment-win.yml @@ -3,6 +3,7 @@ channels: - conda-forge - nodefaults dependencies: + - asv - brotli - build - bump2version>=1 diff --git a/csp/benchmarks/__init__.py b/csp/benchmarks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/csp/benchmarks/asv.conf.jsonc b/csp/benchmarks/asv.conf.jsonc new file mode 100644 index 00000000..5f1f6725 --- /dev/null +++ b/csp/benchmarks/asv.conf.jsonc @@ -0,0 +1,33 @@ +// https://asv.readthedocs.io/en/v0.6.3/asv.conf.json.html +{ + "version": 1, + "project": "csp", + "project_url": "https://github.com/Point72/csp", + "repo": "../..", + "branches": ["main"], + "dvcs": "git", + + "install_command": ["in-dir={env_dir} python -mpip install {wheel_file}"], + "uninstall_command": ["return-code=any python -mpip uninstall -y {project}"], + "build_command": [ + "python -m pip install build", + "python -m build --wheel -o {build_cache_dir} {build_dir}" + ], + "environment_type": "virtualenv", + "install_timeout": 600, + "show_commit_url": "http://github.com/point72/csp/commit/", + + "pythons": ["3.11"], + + // "environment_type": "mamba", + // "conda_channels": ["conda-forge"], + // "conda_environment_file": "conda/dev-environment-unix.yml", + + "benchmark_dir": "../../csp/benchmarks", + "env_dir": "../../.asv/env", + "results_dir": "../../ci/benchmarks", + "html_dir": "../../.asv/html", + + "hash_length": 8, + "build_cache_size": 2 +} diff --git a/csp/benchmarks/stats/__init__.py b/csp/benchmarks/stats/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/csp/benchmarks/stats/basic.py b/csp/benchmarks/stats/basic.py new file mode 100644 index 00000000..6556c9b1 --- /dev/null +++ b/csp/benchmarks/stats/basic.py @@ -0,0 +1,64 @@ +import numpy as np +import time +from datetime import datetime, timedelta + +import csp + + +class StatsBenchmarkSuite: + def setup(self): + self.st = datetime(2020, 1, 1) + self.N = 1_000 + self.ARRAY_SIZE = 100 + self.TEST_TIMES = [self.st + timedelta(seconds=i) for i in range(self.N)] + self.RANDOM_VALUES = [np.random.normal(size=(self.ARRAY_SIZE,)) for i in range(self.N)] # 100 element np array + self.DATA = list(zip(self.TEST_TIMES, self.RANDOM_VALUES)) + self.INTERVAL = 500 + self.NUM_SAMPLES = 100 + + def time_stats_qtl(self): + def g_qtl(): + data = csp.curve(typ=np.ndarray, data=self.DATA) + median = csp.stats.median(data, interval=self.INTERVAL) + csp.add_graph_output("final_median", median, tick_count=1) + + qtl_times = [] + + for _ in range(self.NUM_SAMPLES): + start = time.time() + csp.run(g_qtl, realtime=False, starttime=self.st, endtime=timedelta(seconds=self.N)) + post_qtl = time.time() + qtl_times.append(post_qtl - start) + + avg_med = sum(qtl_times) / self.NUM_SAMPLES + print( + f"Average time in {self.NUM_SAMPLES} tests for median with {self.N=}, {self.ARRAY_SIZE=}, {self.INTERVAL=}: {round(avg_med, 2)} s" + ) + return avg_med + + def time_stats_rank(self): + def g_rank(): + data = csp.curve(typ=np.ndarray, data=self.DATA) + rank = csp.stats.rank(data, interval=self.INTERVAL) + csp.add_graph_output("final_rank", rank, tick_count=1) + + rank_times = [] + + for _ in range(self.NUM_SAMPLES): + start = time.time() + csp.run(g_rank, realtime=False, starttime=self.st, endtime=timedelta(seconds=self.N)) + post_rank = time.time() + rank_times.append(post_rank - start) + + avg_rank = sum(rank_times) / self.NUM_SAMPLES + print( + f"Average time in {self.NUM_SAMPLES} tests for rank with {self.N=}, {self.ARRAY_SIZE=}, {self.INTERVAL=}: {round(avg_rank, 2)} s" + ) + return avg_rank + + +if __name__ == "__main__": + sbs = StatsBenchmarkSuite() + sbs.setup() + sbs.time_stats_qtl() + sbs.time_stats_rank() diff --git a/pyproject.toml b/pyproject.toml index 511405e8..35be4b30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,8 @@ develop = [ "sqlalchemy", # db "threadpoolctl", # test_random "tornado", # profiler, perspective, websocket + # benchmarks + "asv", ] showgraph = [ "graphviz",