extras/benchmark/benchmark.py

0001 # SPDX-PackageName = "traccc, a part of the ACTS project"
0002 # SPDX-FileCopyrightText: CERN
0003 # SPDX-License-Identifier: MPL-2.0
0004
0005 import argparse
0006 import sys
0007 import csv
0008 import git
0009 import pathlib
0010 import shutil
0011 import logging
0012 import tempfile
0013 import subprocess
0014 import os
0015 import time
0016
0017 from traccc_bench_tools import parse_profile, types, build, profile, benchmark
0018
0019 log = logging.getLogger("traccc_benchmark")
0020
0021
0022 KNOWN_BROKEN_COMMITS = [
0023     "9444f505d62d1213da7b3a502da3a233d524d264",
0024 ]
0025
0026
0027 def main():
0028     parser = argparse.ArgumentParser()
0029
0030     parser.add_argument(
0031         "repo",
0032         type=pathlib.Path,
0033         help="the traccc build repository",
0034     )
0035
0036     parser.add_argument(
0037         "db",
0038         type=pathlib.Path,
0039         help="the CSV database",
0040     )
0041
0042     parser.add_argument(
0043         "data",
0044         type=pathlib.Path,
0045         help="the traccc dataset to use",
0046     )
0047
0048     parser.add_argument(
0049         "-f",
0050         "--from",
0051         type=str,
0052         help="the first commit in range (exclusive)",
0053         required=True,
0054     )
0055
0056     parser.add_argument(
0057         "-t",
0058         "--to",
0059         type=str,
0060         help="the last commit in range (inclusive)",
0061         default="HEAD",
0062     )
0063
0064     parser.add_argument(
0065         "-v",
0066         "--verbose",
0067         help="enable verbose output",
0068         action="store_true",
0069     )
0070
0071     parser.add_argument(
0072         "-j",
0073         "--parallel",
0074         help="number of threads to use for compilation",
0075         default=1,
0076         type=int,
0077     )
0078
0079     parser.add_argument(
0080         "-e",
0081         "--events",
0082         help="number of events to process per commit",
0083         type=int,
0084         default=10,
0085     )
0086
0087     parser.add_argument(
0088         "--cc",
0089         help="Compute Capability of the modelled GPU",
0090         type=str,
0091         dest="cc",
0092     )
0093
0094     parser.add_argument(
0095         "--num-sm",
0096         help="number of SMs in the modelled GPU",
0097         type=int,
0098         required=True,
0099         dest="num_sm",
0100     )
0101
0102     parser.add_argument(
0103         "--num-threads-per-sm",
0104         help="number of thread slots per SM in the modelled GPU",
0105         type=int,
0106         required=True,
0107         dest="num_threads_per_sm",
0108     )
0109
0110     parser.add_argument(
0111         "--ncu-wrapper",
0112         help="wrapper to use around the ncu command",
0113         type=str,
0114         dest="ncu_wrapper",
0115     )
0116
0117     args = parser.parse_args()
0118
0119     logging.basicConfig(
0120         level=logging.DEBUG if (args.verbose or False) else logging.INFO,
0121         format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
0122     )
0123
0124     log.info(
0125         "Using GPU with %d SMs and %d thread slots per SM (%d thread slots total)",
0126         getattr(args, "num_sm"),
0127         getattr(args, "num_threads_per_sm"),
0128         getattr(args, "num_sm") * getattr(args, "num_threads_per_sm"),
0129     )
0130
0131     repo = git.Repo(args.repo)
0132
0133     log.info("Using git repository at %s", repo.git_dir)
0134
0135     if repo.is_dirty():
0136         e = "Repository is dirty; please clean it before use!"
0137         log.fatal(e)
0138         raise RuntimeError(e)
0139
0140     if "TRACCC_TEST_DATA_DIR" not in os.environ:
0141         e = 'Environment variable "TRACCC_TEST_DATA_DIR" is not set; aborting!'
0142         log.fatal(e)
0143         raise RuntimeError(e)
0144
0145     for exec in ["ncu", "g++", "nvcc", "cmake"]:
0146         if shutil.which(exec) is None:
0147             e = 'Executable "%s" is not available; aborting' % exec
0148             log.fatal(e)
0149             raise RuntimeError(e)
0150
0151     old_commit_hash = repo.head.object.hexsha
0152     log.info("Current commit hash is %s", old_commit_hash)
0153
0154     commit_range = repo.iter_commits(
0155         "%s...%s" % (getattr(args, "from"), getattr(args, "to"))
0156     )
0157
0158     commit_list = list(commit_range)
0159     commit_str_list = list(str(x) for x in commit_list)
0160
0161     log.info(
0162         "Examining a total of %d commits between %s and %s",
0163         len(commit_list),
0164         getattr(args, "from"),
0165         getattr(args, "to"),
0166     )
0167
0168     if args.db.is_file():
0169         log.info('Database file "%s" already exists; creating a backup', args.db)
0170         shutil.copy(str(args.db), str(args.db) + ".bak")
0171         results = []
0172         with open(args.db, "r") as f:
0173             reader = csv.DictReader(f)
0174             for i in reader:
0175                 results.append(i)
0176         log.info("Database contained %d pre-existing results", len(results))
0177     else:
0178         log.info('Database file "%s" does not exist; starting from scratch', args.db)
0179         results = []
0180
0181     known_commits = set(x["commit"] for x in results)
0182
0183     log.info("Currently have pre-existing results for %d commits", len(known_commits))
0184
0185     commits_to_skip = set()
0186
0187     for progress_id, x in enumerate(commit_list):
0188         log.info(
0189             "Considering commit %s (%u out of %u)", x, progress_id, len(commit_list)
0190         )
0191
0192         parents = x.parents
0193
0194         # If this commit has a parent with a single parent, and that
0195         # grandparent is also a parent of this current commit, we have a
0196         # single-commit merge which we don't need to measure separately.
0197         if len(parents) == 2:
0198             for p1 in parents:
0199                 if len(p1.parents) == 1 and p1.parents[0] in parents:
0200                     log.info("Commit %s is a triangle commit; adding to skip list", p1)
0201                     commits_to_skip.add(str(p1))
0202
0203         # Skip commits which we have already benchmarked
0204         if str(x) in known_commits:
0205             log.info("Commit %s is already know; skipping", x)
0206             continue
0207
0208         if str(x) in commits_to_skip:
0209             log.info("Commit %s is marked for skipping", x)
0210             continue
0211
0212         if str(x) in KNOWN_BROKEN_COMMITS:
0213             log.info("Commit %s is known to be broken; skipping", x)
0214             continue
0215
0216         try:
0217             log.info("Running benchmark for commit %s", x)
0218
0219             log.info("Running checkout step")
0220
0221             start_time = time.time()
0222             repo.git.checkout(x)
0223             end_time = time.time()
0224
0225             log.info("Completed checkout step in %.1f seconds", end_time - start_time)
0226
0227             result_df = benchmark.run_benchmark(
0228                 source_dir=args.repo,
0229                 data_dir=args.data,
0230                 commit=x,
0231                 gpu_spec=types.GpuSpec(
0232                     getattr(args, "num_sm"), getattr(args, "num_threads_per_sm")
0233                 ),
0234                 parallel=args.parallel,
0235                 events=args.events,
0236                 ncu_wrapper=getattr(args, "ncu_wrapper", None),
0237                 cc=getattr(args, "cc", None),
0238             )
0239
0240             for y in result_df.iloc:
0241                 results.append(
0242                     {
0243                         "commit": str(x),
0244                         "kernel": y["Kernel Name"],
0245                         "throughput": y["ThroughputMean"],
0246                         "throughput_dev": y["ThroughputStd"],
0247                         "rec_throughput": y["RecThroughputMean"],
0248                         "rec_throughput_dev": y["RecThroughputStd"],
0249                     }
0250                 )
0251
0252         except Exception as e:
0253             log.exception(e)
0254         except KeyboardInterrupt as e:
0255             log.info("Received keyboard interrupt; skipping to post-processing")
0256             break
0257
0258     log.info("Gathered a total of %d results (incl. pre-existing)", len(results))
0259     output_results = sorted(
0260         [x for x in results if x["commit"] in commit_str_list],
0261         key=lambda x: -commit_str_list.index(x["commit"]),
0262     )
0263     log.info("Keeping a total of %d results after pruning", len(output_results))
0264
0265     log.info("Writing data to %s", args.db)
0266     with open(args.db, "w") as f:
0267         writer = csv.DictWriter(
0268             f,
0269             fieldnames=[
0270                 "commit",
0271                 "kernel",
0272                 "throughput",
0273                 "throughput_dev",
0274                 "rec_throughput",
0275                 "rec_throughput_dev",
0276             ],
0277         )
0278         writer.writeheader()
0279
0280         for i in output_results:
0281             writer.writerow(i)
0282
0283     log.info("Checking out repository to previous commit %s", old_commit_hash)
0284     repo.git.checkout(old_commit_hash)
0285     log.info("Processing complete; goodbye!")
0286
0287
0288 if __name__ == "__main__":
0289     main()