mirror of
https://github.com/RPCS3/llvm-mirror.git
synced 2024-11-25 04:19:43 +00:00
81032e2009
The Args field of the remark which consists of a list of mappings in YAML is translated into a list of (small) dicts on Python. An empty dict is 280 bytes on my system so we can save memory by using a tuple of tuples instead. Making a tuple of tuples rather than a list of tuples allows Args to be shared with the key of the remark. This is actually an even greater saving. (Keys are alive throughout the entire run in all_remarks.) Here are a few opt-stats runs with different input sizes while measuring heap usage with heapy. Avg remark size is simply estimated as heap-size / # of remarks: | # of files | 60 | 114 | 308 | 605 | 1370 | | # of remarks | 20K | 37K | 146K | 180K | 640K | | total file size (MB) | 22 | 51 | 219 | 202 | 1034 | |------------------------+------+------+------+------+------| | Avg remark size before | 4339 | 4792 | 4761 | 4096 | 4607 | | Avg remark size after | 3446 | 3641 | 3567 | 3146 | 3347 | | Rate | 0.79 | 0.76 | 0.75 | 0.77 | 0.73 | Differential Revision: https://reviews.llvm.org/D35611 llvm-svn: 308538
70 lines
2.3 KiB
Python
Executable File
70 lines
2.3 KiB
Python
Executable File
#!/usr/bin/env python2.7
|
|
|
|
from __future__ import print_function
|
|
|
|
desc = '''Generate the difference of two YAML files into a new YAML file (works on
|
|
pair of directories too). A new attribute 'Added' is set to True or False
|
|
depending whether the entry is added or removed from the first input to the
|
|
next.
|
|
|
|
The tools requires PyYAML.'''
|
|
|
|
import yaml
|
|
# Try to use the C parser.
|
|
try:
|
|
from yaml import CLoader as Loader
|
|
except ImportError:
|
|
from yaml import Loader
|
|
|
|
import optrecord
|
|
import argparse
|
|
from collections import defaultdict
|
|
from multiprocessing import cpu_count, Pool
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description=desc)
|
|
parser.add_argument(
|
|
'yaml_dir_or_file_1',
|
|
help='An optimization record file or a directory searched for optimization '
|
|
'record files that are used as the old version for the comparison')
|
|
parser.add_argument(
|
|
'yaml_dir_or_file_2',
|
|
help='An optimization record file or a directory searched for optimization '
|
|
'record files that are used as the new version for the comparison')
|
|
parser.add_argument(
|
|
'--jobs',
|
|
'-j',
|
|
default=cpu_count(),
|
|
type=int,
|
|
help='Max job count (defaults to %(default)s, the current CPU count)')
|
|
parser.add_argument(
|
|
'--no-progress-indicator',
|
|
'-n',
|
|
action='store_true',
|
|
default=False,
|
|
help='Do not display any indicator of how many YAML files were read.')
|
|
parser.add_argument('--output', '-o', default='diff.opt.yaml')
|
|
args = parser.parse_args()
|
|
|
|
files1 = optrecord.find_opt_files([args.yaml_dir_or_file_1])
|
|
files2 = optrecord.find_opt_files([args.yaml_dir_or_file_2])
|
|
|
|
print_progress = not args.no_progress_indicator
|
|
all_remarks1, _, _ = optrecord.gather_results(files1, args.jobs, print_progress)
|
|
all_remarks2, _, _ = optrecord.gather_results(files2, args.jobs, print_progress)
|
|
|
|
added = set(all_remarks2.values()) - set(all_remarks1.values())
|
|
removed = set(all_remarks1.values()) - set(all_remarks2.values())
|
|
|
|
for r in added:
|
|
r.Added = True
|
|
for r in removed:
|
|
r.Added = False
|
|
|
|
result = added | removed
|
|
for r in result:
|
|
r.recover_yaml_structure()
|
|
|
|
with open(args.output, 'w') as stream:
|
|
yaml.dump_all(result, stream)
|