test: Run bench sanity checks in parallel with functional tests

.github/workflows/ci.yml

@@ -484,9 +484,6 @@ jobs:
           ./src/univalue/object.exe
           ./src/univalue/unitester.exe
 
-      - name: Run benchmarks
-        run: ./bin/bench_bitcoin.exe -sanity-check
-
       - name: Adjust paths in test/config.ini
         shell: pwsh
         run: |

src/bench/CMakeLists.txt

@@ -84,8 +84,4 @@ if(ENABLE_WALLET)
   target_link_libraries(bench_bitcoin bitcoin_wallet)
 endif()
 
-add_test(NAME bench_sanity_check
-  COMMAND bench_bitcoin -sanity-check
-)
-
 install_binary_component(bench_bitcoin INTERNAL)

test/functional/test_runner.py

@@ -29,6 +29,11 @@ import sys
 import tempfile
 import re
 import logging
+from test_framework.util import (
+    Binaries,
+    export_env_build_path,
+    get_binary_paths,
+)
 
 # Minimum amount of space to run the tests.
 MIN_FREE_SPACE = 1.1 * 1024 * 1024 * 1024
@@ -87,7 +92,12 @@ EXTENDED_SCRIPTS = [
     'feature_index_prune.py',
 ]
 
+# Special script to run each bench sanity check
+TOOL_BENCH_SANITY_CHECK = "tool_bench_sanity_check.py"
+
 BASE_SCRIPTS = [
+    # Special scripts that are "expanded" later
+    TOOL_BENCH_SANITY_CHECK,
     # Scripts that are run by default.
     # Longest test should go first, to favor running tests in parallel
     # vv Tests less than 5m vv
@@ -457,6 +467,8 @@ def main():
         print("Re-compile with the -DBUILD_DAEMON=ON build option")
         sys.exit(1)
 
+    export_env_build_path(config)
+
     # Build tests
     test_list = deque()
     if tests:
@@ -511,6 +523,15 @@ def main():
                 # Exclude all variants of a test
                 remove_tests([test for test in test_list if test.split('.py')[0] == exclude_test.split('.py')[0]])
 
+    if config["components"].getboolean("BUILD_BENCH") and TOOL_BENCH_SANITY_CHECK in test_list:
+        # Remove it, and expand it for each bench in the list
+        test_list.remove(TOOL_BENCH_SANITY_CHECK)
+        bench_cmd = Binaries(get_binary_paths(config), bin_dir=None).bench_argv() + ["-list"]
+        bench_list = subprocess.check_output(bench_cmd, text=True).splitlines()
+        bench_list = [f"{TOOL_BENCH_SANITY_CHECK} --bench={b}" for b in bench_list]
+        # Start with special scripts (variable, unknown runtime)
+        test_list.extendleft(reversed(bench_list))
+
     if args.filter:
         test_list = deque(filter(re.compile(args.filter).search, test_list))
 

test/functional/tool_bench_sanity_check.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+# Copyright (c) The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://opensource.org/license/mit/.
+"""Special script to run each bench sanity check
+"""
+import shlex
+import subprocess
+
+from test_framework.test_framework import BitcoinTestFramework
+
+
+class BenchSanityCheck(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 0  # No node/datadir needed
+
+    def setup_network(self):
+        pass
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_bitcoin_bench()
+
+    def add_options(self, parser):
+        parser.add_argument(
+            "--bench",
+            default=".*",
+            help="Regex to filter the bench to run (default=%(default)s)",
+        )
+
+    def run_test(self):
+        cmd = self.get_binaries().bench_argv() + [
+            f"-filter={self.options.bench}",
+            "-sanity-check",
+        ]
+        self.log.info(f"Starting: {shlex.join(cmd)}")
+        subprocess.run(cmd, check=True)
+        self.log.info("Success!")
+
+
+if __name__ == "__main__":
+    BenchSanityCheck(__file__).main()