mirror of
https://github.com/EleutherAI/lm-evaluation-harness.git
synced 2026-10-12 02:02:05 +00:00
Lintang SutawikaandBaber
1ba35e623b
* changed source of eval_logger * allow eval_logger to be set from args * removed verbosity arg from non-main methods * fix logging * pre-commit * set verbosity in eval logger * replace utils.eval_logger * fix logging in main * add logging to docs * add logging message * nit * add logging to docs * refactor setup_logging to utils --------- Co-authored-by: Baber <[email protected]>
75 lines
2.4 KiB
Python
75 lines
2.4 KiB
Python
"""
|
|
Usage:
|
|
python make_table_tasks.py --output <markdown_filename>
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
|
|
from pytablewriter import LatexTableWriter, MarkdownTableWriter
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def make_table(result_dict):
|
|
"""Generate table of results."""
|
|
md_writer = MarkdownTableWriter()
|
|
latex_writer = LatexTableWriter()
|
|
md_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"]
|
|
latex_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"]
|
|
|
|
values = []
|
|
|
|
for k, dic in sorted(result_dict["results"].items()):
|
|
version = result_dict["versions"][k]
|
|
percent = k == "squad2"
|
|
for m, v in dic.items():
|
|
if m.endswith("_stderr"):
|
|
continue
|
|
|
|
if m + "_stderr" in dic:
|
|
se = dic[m + "_stderr"]
|
|
if percent or m == "ppl":
|
|
values.append([k, version, m, "%.2f" % v, "±", "%.2f" % se])
|
|
else:
|
|
values.append(
|
|
[k, version, m, "%.2f" % (v * 100), "±", "%.2f" % (se * 100)]
|
|
)
|
|
else:
|
|
if percent or m == "ppl":
|
|
values.append([k, version, m, "%.2f" % v, "", ""])
|
|
else:
|
|
values.append([k, version, m, "%.2f" % (v * 100), "", ""])
|
|
k = ""
|
|
version = ""
|
|
md_writer.value_matrix = values
|
|
latex_writer.value_matrix = values
|
|
|
|
# todo: make latex table look good
|
|
# print(latex_writer.dumps())
|
|
|
|
return md_writer.dumps()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# loop dirs and subdirs in results dir
|
|
# for each dir, load json files
|
|
for dirpath, dirnames, filenames in os.walk("../results"):
|
|
# skip dirs without files
|
|
if not filenames:
|
|
continue
|
|
path_readme = os.path.join(dirpath, "README.md")
|
|
with open(path_readme, "w", encoding="utf-8") as f:
|
|
# get path name, only last folder
|
|
path_name = dirpath.split("/")[-1]
|
|
f.write(f"# {path_name} \n\n")
|
|
for filename in sorted([f for f in filenames if f.endswith(".json")]):
|
|
path = os.path.join(dirpath, filename)
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
result_dict = json.load(f)
|
|
with open(path_readme, "a", encoding="utf-8") as f:
|
|
f.write(f"## {filename} \n")
|
|
f.write(f"{make_table(result_dict)} \n")
|