physics_benchmarks/bin/collect_benchmarks.py

0001 #!/usr/bin/env python3
0002
0003 """
0004 Combine the json files from the individual benchmark tests into
0005 a final master json file combining all benchmarks.
0006
0007 Benchmark results are expected to be all json files in the results
0008 directory.
0009 """
0010
0011 ## Our master definition file, the benchmark project directory
0012 MASTER_FILE=r'benchmarks/benchmarks.json'
0013
0014 ## Our results directory
0015 RESULTS_PATH=r'results'
0016
0017 ## Output json file with all benchmark results
0018 OUTPUT_FILE=r'results/summary.json'
0019
0020 import argparse
0021 import json
0022 from pathlib import Path
0023
0024 ## Exceptions for this module
0025 class Error(Exception):
0026     '''Base class for exceptions in this module.'''
0027     pass
0028 class FileNotFoundError(Error):
0029     '''File does not exist.
0030
0031     Attributes:
0032         file: the file name
0033         message: error message
0034     '''
0035     def __init__(self, file):
0036         self.file = file
0037         self.message = 'No such file or directory: {}'.format(file)
0038
0039 class InvalidDefinitionError(Error):
0040     '''Raised for missing keys in the definitions.
0041
0042     Attributes:
0043         key: the missing key
0044         file: the definition file
0045         message: error message
0046     '''
0047     def __init__(self, key, file):
0048         self.key = key
0049         self.file = file
0050         self.message = "key '{}' not found in '{}'".format(key, file)
0051
0052 class InvalidResultError(Error):
0053     '''Raised for invalid benchmark result value.
0054
0055     Attributes:
0056         key: the missing key
0057         value: the invalid value
0058         file: the benchmark definition file
0059         message: error message
0060     '''
0061     def __init__(self, key, value, file):
0062         self.key = key
0063         self.value = value
0064         self.file = file
0065         self.message = "value '{}' for key '{}' invalid in benchmark file '{}'".format(
0066                 value, key, file)
0067
0068 def collect_benchmarks():
0069     '''Collect all benchmark results and write results to a single file.'''
0070     print("Collecting all benchmark results")
0071
0072     ## load the test definition for this benchmark
0073     results = _load_master()
0074
0075     ## collect the test results
0076     results['benchmarks'] = _load_benchmarks()
0077
0078     ## calculate aggregate test statistics
0079     results = _aggregate_results(results)
0080
0081     ## save results to output file
0082     _save(results)
0083
0084     ## Summarize results
0085     for bm in results['benchmarks']:
0086         _print_benchmark(bm)
0087     _print_summary(results)
0088
0089 def _load_master():
0090     '''Load master definition.'''
0091     master_file = Path(MASTER_FILE)
0092     if not master_file.exists():
0093         raise FileNotFoundError(master_file)
0094     print('  --> Loading master definition from:', master_file)
0095     results = None
0096     with master_file.open() as f:
0097         results = json.load(f)
0098     ## ensure this is a valid benchmark file
0099     for key in ('name', 'title', 'description'):
0100         if not key in results:
0101             raise InvalidDefinitionError('target', master_file)
0102     return results
0103
0104 def _load_benchmarks():
0105     '''Load all benchmark results from the results folder.'''
0106     print('  --> Collecting all benchmarks')
0107     rootdir = Path(RESULTS_PATH)
0108     results = []
0109     for file in rootdir.glob('*.json'):
0110         print('    --> Loading file:', file, '... ', end='')
0111         with open(file) as f:
0112             bm = json.load(f)
0113             ## skip files that don't include test results
0114             if not 'tests' in bm:
0115                 print('skipped (does not contain benchmark results).')
0116                 continue
0117             ## check if these are valid benchmark results,
0118             ## raise exception otherwise
0119             for key in ('name', 'title', 'description', 'target', 'n_tests',
0120                     'n_pass', 'n_fail', 'n_error', 'maximum', 'sum', 'value',
0121                     'result'):
0122                 if not key in bm:
0123                     raise InvalidDefinitionError(key, file)
0124             if bm['result'] not in ('pass', 'fail', 'error'):
0125                 raise InvalidResultError('result', bm['result'], file)
0126             ## Append to our test results
0127             results.append(bm)
0128             print('done')
0129     return results
0130
0131 def _aggregate_results(results):
0132     '''Aggregate benchmark results.'''
0133     print('  --> Aggregating benchmark statistics')
0134     results['n_benchmarks'] = len(results['benchmarks'])
0135     results['n_pass'] = len([1 for t in results['benchmarks'] if t['result'] == 'pass'])
0136     results['n_fail'] = len([1 for t in results['benchmarks'] if t['result'] == 'fail'])
0137     results['n_error'] = len([1 for t in results['benchmarks'] if t['result'] == 'error'])
0138     if results['n_error'] > 0:
0139         results['result'] = 'error'
0140     elif results['n_fail'] == 0:
0141         results['result'] = 'pass'
0142     else:
0143         results['result'] = 'fail'
0144     return results
0145
0146 def _save(results):
0147     '''Save aggregated benchmark results'''
0148     ofile = Path(OUTPUT_FILE)
0149     print('  --> Saving results to:', ofile)
0150     with ofile.open('w') as f:
0151         json.dump(results, f, indent=4)
0152
0153 def _print_benchmark(bm):
0154     '''Print benchmark summary to the terminal.'''
0155     print('====================================================================')
0156     print('  Summary for:', bm['title'])
0157     print('  Pass: {}, Fail: {}, Error: {} out of {} total tests'.format(
0158         bm['n_pass'], bm['n_fail'], bm['n_error'],
0159         bm['n_tests']))
0160     print('  Weighted sum: {} / {}'.format(bm['sum'], bm['maximum']))
0161     print('  kBenchmark value: {} (target: {})'.format(
0162         bm['value'], bm['target']))
0163     print('  ===> status:', bm['result'])
0164
0165 def _print_summary(results):
0166     '''Print master benchmark summary to the terminal.'''
0167     print('====================================================================')
0168     print('MASTER BENCHMARK SUMMARY FOR:', results['title'].upper())
0169     print('Pass: {}, Fail: {}, Error: {} out of {} total benchmarks'.format(
0170         results['n_pass'], results['n_fail'], results['n_error'],
0171         results['n_benchmarks']))
0172     print('===> status:', results['result'])
0173     print('====================================================================')
0174
0175
0176 if __name__ == "__main__":
0177     try:
0178         collect_benchmarks()
0179     except Error as e:
0180         print()
0181         print('ERROR', e.message)