libretro/libretro-database
1
0 Fork 0
mirror of https://github.com/libretro/libretro-database.git synced 2026-11-07 11:06:04 -10:00
Code Issues Packages Projects Releases 1 Wiki Activity

Improve FBNeo dat generator script (#1677)

Browse Source 
* FBNeo_dat_gen: change text encoding, change year to releaseyear and pass parser where missing

* FBNeo_dat_gen: standardize cli args

* gitattributes: Add .py EOL LF

---------

Co-authored-by: Rob Loach <[email protected]>
This commit is contained in:
AramisandRob Loach authored and GitHub committed 2026-06-19 11:37:59 -04:00
1 parent 864ff2b27a
commit 4e74868001
2 files changed
+240 -235
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitattributes
+1
View File
@@ -2,3 +2,4 @@
*.dat text eol=lf
*.xml text eol=lf
*.cht text eol=lf
*.py text eol=lf
scripts/FBNeo_dat_gen.py
+239 -235
View File
@@ -1,235 +1,239 @@
#!/usr/bin/env python3
# FBNeo_dat_gen.py
# Written in 2017 by SpiralBrad <[email protected]>
## usage: FBNeo_dat_gen.py [-h] -dat DAT -path PATH [-output_file OUTPUT_FILE]
## [-header_name HEADER_NAME]
## [-header_description HEADER_DESCRIPTION]
## [-header_version HEADER_VERSION]
##
## Generates Final Burn Neo .dat file needed for RetroArch/libretro-
## db/c_converter
##
## optional arguments:
## -h, --help show this help message and exit
## -output_file OUTPUT_FILE
## Path to the target output file; example: FBNeo -
## Arcade games.dat
## -header_name HEADER_NAME
## Override the clrmamepro(name) in the output .dat
## -header_description HEADER_DESCRIPTION
## Override the clrmamepro(description) in the output
## .dat
## -header_version HEADER_VERSION
## Override the clrmamepro(version) in the output .dat
##
## required arguments:
## -dat DAT Misc -> Generate dat file -> Generate dat (Arcade
## only)
## -path PATH Path to a split, ClrMamePro verified and TorrentZipped
## ROM set matching the Arcade only dat
##
## example usage: python3 FBA_dat_gen.py -dat "FBNeo v0.2.97.42 (ClrMame Pro
## XML).dat" -path "/path/to/split/verified/torrentzipped/roms/"
## -output_file "FBNeo - Arcade Games.dat"
#- To the extent possible under law, the author(s) have dedicated all
#- copyright and related and neighboring rights to this software to the
#- public domain worldwide. This software is distributed without any
#- warranty.
#-
#- You should have received a copy of the CC0 Public Domain Dedication
#- along with this software. If not, see
#- <http://creativecommons.org/publicdomain/zero/1.0/>.
import argparse
import hashlib
import operator
import os
import os.path
import sys
import xml.etree.ElementTree as ET
import zlib
def main():
parser = setup_argparse()
args = parser.parse_args()
dat_root = get_datroot(args.dat, parser)
header_name = get_header_name(args)
header_version = get_header_version(args, dat_root, parser)
header_description = get_header_description(args, header_version)
header = generate_dat_header(header_name, header_description, header_version)
game_list = generate_game_list(dat_root, args.path)
output(args, header, game_list)
def setup_argparse():
"""Set up the argparse arguments and return the argparse instance"""
parser = argparse.ArgumentParser(prog='FBgen.py', description='Generates Final Burn Neo .dat file needed for RetroArch/libretro-db/c_converter')
required_arguments = parser.add_argument_group('required arguments')
required_arguments.add_argument('-dat', help='Misc -> Generate dat file -> Generate dat (Arcade only)', required=True)
required_arguments.add_argument('-path', help='Path to a split, ClrMamePro verified and TorrentZipped ROM set matching the Arcade only dat', required=True)
parser.add_argument('-output_file', help='Path to the target output file; example: FBNeo - Arcade games.dat')
parser.add_argument('-header_name', help='Override the clrmamepro(name) in the output .dat')
parser.add_argument('-header_description', help='Override the clrmamepro(description) in the output .dat')
parser.add_argument('-header_version', help='Override the clrmamepro(version) in the output .dat')
if len(sys.argv[1:])==0:
parser.print_help()
print('\n'.join(['',
'example usage: python3 FBA_dat_gen.py -dat "FBNeo v0.2.97.42 (ClrMame Pro ',
' XML).dat" -path "/path/to/split/verified/torrentzipped/roms/" ',
' -output_file "FBNeo - Arcade Games.dat"']))
parser.exit()
return parser
def get_datroot(dat, parser):
"""Get the ElementTree root from the specified <dat> file and return the ElementTree root element"""
if not os.path.isfile(dat):
print('File not found: ' + dat)
print('')
parser.print_help()
parser.exit()
else:
dat_tree = ET.parse(dat)
dat_root = dat_tree.getroot()
return dat_root
def get_header_name(args):
"""Return the default header name or the -header_name argument"""
if not args.header_name:
header_name = 'FBNeo - Arcade Games'
else:
header_name = args.header_name
return header_name
def get_header_version(args, dat_root, parser):
"""Return the version from the <dat_root> header version, or the -header_version argument"""
if not args.header_version:
header_version = dat_root.find('header').find('version').text
if not header_version:
print('Version not found in ClrMame Pro XML dat. Please specify with -header_version')
print('')
parser.print_help()
parser.exit()
raise SystemExit()
else:
header_version = args.header_version
return header_version
def get_header_description(args, version):
"""Return the description by generationg via the <version> or the -header_description argument"""
if not args.header_description:
header_description = 'FBNeo v' + version + ' Arcade Games'
else:
header_description = args.header_description
return header_description
def generate_dat_header(name, description, version):
"""Return the textual DAT header from a given <name>, <description>, and <version>"""
header = ['clrmamepro (',
'\t' + 'name "%s"' % name,
'\t' + 'description "%s"' % description,
'\t' + 'version %s' % version,
')']
return '\n'.join(header)
def generate_game_list(dat_root, path):
"""Generate the sorted list of games with all metadata and return a textual dat list"""
game_list = []
if not os.path.isdir(path):
print('Path not found: ' + path)
print('')
parser.print_help()
parser.exit()
else:
game_entries = []
for game in dat_root.iter('game'):
# 'gpriders.zip' kludge hack, as of at least 0.2.97.42
# - rom is exact to 'gprider.zip' and does not exist in a split set
if game.get('name') != 'gpriders':
# same 0.2.97.44
if game.get('name') != 'natodefa':
if game.get('name') != 'marioo':
entry = GameEntry()
# set name ('description' in FBA-generated-dat)
entry.name = game.find('description').text
# set year
entry.year = game.find('year').text
# set publisher ('manufacturer' in FBA-generated-dat)
entry.publisher = game.find('manufacturer').text
# set zip filename
entry.zip = game.get('name') + '.zip'
zip_path = os.path.join(path, entry.zip)
# set zip size
entry.size = os.path.getsize(zip_path)
# set zip crc
entry.crc = get_crc(zip_path)
# set zip md5
entry.md5 = get_md5(zip_path)
# set zip sha1
entry.sha1 = get_sha1(zip_path)
# Add to game_entries list
game_entries.append(entry)
# Sort game_entries list
game_entries.sort(key=operator.attrgetter('name'))
# Generate formatted textual list
for entry in game_entries:
text_entry = ['game (',
'\t' + 'name "%s"' % entry.name,
'\t' + 'year "%s"' % entry.year,
'\t' + 'publisher "%s"' % entry.publisher,
'\t' + 'rom ( name %s size %s crc %s md5 %s sha1 %s )' % (entry.zip, entry.size, entry.crc, entry.md5, entry.sha1),
')',
'']
game_list.append('\n'.join(text_entry))
return '\n'.join(game_list)
def output(args, header, game_list):
"""Combine <header> and <game_list> and print to stdout, or write to specified -output_file"""
if not args.output_file:
# print to stdout
print('\n'.join([header, '', game_list]).rstrip())
else:
# write to args.output_file
with open(args.output_file, 'w') as f:
f.write('\n'.join([header, '', game_list]))
class GameEntry:
name = None
year = None
publisher = None
zip = None
size = None
crc = None
md5 = None
sha1 = None
def get_crc(file):
"""Return the CRC32 hash of <file>"""
prev = 0
for eachLine in open(file, 'rb'):
prev = zlib.crc32(eachLine, prev)
return str("%X"%(prev & 0xFFFFFFFF)).zfill(8)
def get_md5(file):
"""Return the MD5 hash of <file>"""
m = hashlib.md5()
with open(file, 'rb') as f:
for chunk in iter(lambda: f.read(4096), b""):
m.update(chunk)
return m.hexdigest().zfill(32)
def get_sha1(file):
"""Return the SHA1 hash of <file>"""
m = hashlib.sha1()
with open(file, 'rb') as f:
for chunk in iter(lambda: f.read(4096), b""):
m.update(chunk)
return m.hexdigest().zfill(40)
if __name__ == '__main__':
main()
#!/usr/bin/env python3
# FBNeo_dat_gen.py
# Written in 2017 by SpiralBrad <[email protected]>
## usage: FBNeo_dat_gen.py [-h] --dat DAT --path PATH [--output-file OUTPUT_FILE]
## [--header-name HEADER_NAME]
## [--header-description HEADER_DESCRIPTION]
## [--header-version HEADER_VERSION]
##
## Generates Final Burn Neo .dat file needed for RetroArch/libretro-
## db/c_converter
##
## optional arguments:
## -h, --help show this help message and exit
## -o, --output_file OUTPUT_FILE
## Path to the target output file; example: FBNeo -
## Arcade games.dat
## --header-name HEADER_NAME
## Override the clrmamepro(name) in the output .dat
## --header-description HEADER_DESCRIPTION
## Override the clrmamepro(description) in the output
## .dat
## --header-version HEADER_VERSION
## Override the clrmamepro(version) in the output .dat
##
## required arguments:
## -d, --dat DAT Misc -> Generate dat file -> Generate dat (Arcade
## only)
## -p, --path PATH Path to a split, ClrMamePro verified and TorrentZipped
## ROM set matching the Arcade only dat
##
## example usage: python3 FBA_dat_gen.py --dat "FBNeo v0.2.97.42 (ClrMame Pro
## XML).dat" --path "/path/to/split/verified/torrentzipped/roms/"
## --output-file "FBNeo - Arcade Games.dat"
#- To the extent possible under law, the author(s) have dedicated all
#- copyright and related and neighboring rights to this software to the
#- public domain worldwide. This software is distributed without any
#- warranty.
#-
#- You should have received a copy of the CC0 Public Domain Dedication
#- along with this software. If not, see
#- <http://creativecommons.org/publicdomain/zero/1.0/>.
import argparse
import hashlib
import operator
import os
import os.path
import sys
import xml.etree.ElementTree as ET
import zlib
def main():
parser = setup_argparse()
args = parser.parse_args()
if parser == None:
print("parser is none")
dat_root = get_datroot(args.dat, parser)
header_name = get_header_name(args)
header_version = get_header_version(args, dat_root, parser)
header_description = get_header_description(args, header_version)
header = generate_dat_header(header_name, header_description, header_version)
game_list = generate_game_list(dat_root, args.path, parser)
output(args, header, game_list)
def setup_argparse():
"""Set up the argparse arguments and return the argparse instance"""
parser = argparse.ArgumentParser(prog='FBgen.py', description='Generates Final Burn Neo .dat file needed for RetroArch/libretro-db/c_converter')
required_arguments = parser.add_argument_group('required arguments')
required_arguments.add_argument('-d', '--dat', help='Misc -> Generate dat file -> Generate dat (Arcade only)', required=True)
required_arguments.add_argument('-p', '--path', help='Path to a split, ClrMamePro verified and TorrentZipped ROM set matching the Arcade only dat', required=True)
parser.add_argument('-o', '--output-file', help='Path to the target output file; example: FBNeo - Arcade games.dat')
parser.add_argument('--header-name', help='Override the clrmamepro(name) in the output .dat')
parser.add_argument('--header-description', help='Override the clrmamepro(description) in the output .dat')
parser.add_argument('--header-version', help='Override the clrmamepro(version) in the output .dat')
if len(sys.argv) == 1:
parser.print_help()
print('\n'.join(['',
'example usage: python3 FBA_dat_gen.py --dat "FBNeo v0.2.97.42 (ClrMame Pro ',
' XML).dat" -path "/path/to/split/verified/torrentzipped/roms/" ',
' --output-file "FBNeo - Arcade Games.dat"']))
parser.exit()
return parser
def get_datroot(dat, parser):
"""Get the ElementTree root from the specified <dat> file and return the ElementTree root element"""
if not os.path.isfile(dat):
print('File not found: ' + dat)
print('')
parser.print_help()
parser.exit()
else:
dat_tree = ET.parse(dat)
dat_root = dat_tree.getroot()
return dat_root
def get_header_name(args):
"""Return the default header name or the -header_name argument"""
if not args.header_name:
header_name = 'FBNeo - Arcade Games'
else:
header_name = args.header_name
return header_name
def get_header_version(args, dat_root, parser):
"""Return the version from the <dat_root> header version, or the -header_version argument"""
if not args.header_version:
header_version = dat_root.find('header').find('version').text
if not header_version:
print('Version not found in ClrMame Pro XML dat. Please specify with -header_version')
print('')
parser.print_help()
parser.exit()
raise SystemExit()
else:
header_version = args.header_version
return header_version
def get_header_description(args, version):
"""Return the description by generationg via the <version> or the -header_description argument"""
if not args.header_description:
header_description = 'FBNeo v' + version + ' Arcade Games'
else:
header_description = args.header_description
return header_description
def generate_dat_header(name, description, version):
"""Return the textual DAT header from a given <name>, <description>, and <version>"""
header = ['clrmamepro (',
'\t' + 'name "%s"' % name,
'\t' + 'description "%s"' % description,
'\t' + 'version %s' % version,
')']
return '\n'.join(header)
def generate_game_list(dat_root, path, parser):
"""Generate the sorted list of games with all metadata and return a textual dat list"""
game_list = []
if not os.path.isdir(path):
print(f'Path is not dir or not found: {path}\n')
parser.print_help()
parser.exit()
else:
game_entries = []
for game in dat_root.iter('game'):
# 'gpriders.zip' kludge hack, as of at least 0.2.97.42
# - rom is exact to 'gprider.zip' and does not exist in a split set
if game.get('name') != 'gpriders':
# same 0.2.97.44
if game.get('name') != 'natodefa':
if game.get('name') != 'marioo':
entry = GameEntry()
# set name ('description' in FBA-generated-dat)
entry.name = game.find('description').text
# set year
entry.year = game.find('year').text
# set publisher ('manufacturer' in FBA-generated-dat)
entry.publisher = game.find('manufacturer').text
# set zip filename
entry.zip = game.get('name') + '.zip'
zip_path = os.path.join(path, entry.zip)
if not os.path.exists(zip_path):
print(f'Did not find "{zip_path}", skipping.')
continue
# set zip size
entry.size = os.path.getsize(zip_path)
# set zip crc
entry.crc = get_crc(zip_path)
# set zip md5
entry.md5 = get_md5(zip_path)
# set zip sha1
entry.sha1 = get_sha1(zip_path)
# Add to game_entries list
game_entries.append(entry)
# Sort game_entries list
game_entries.sort(key=operator.attrgetter('name'))
# Generate formatted textual list
for entry in game_entries:
text_entry = ['game (',
'\t' + 'name "%s"' % entry.name,
'\t' + 'releaseyear "%s"' % entry.year,
'\t' + 'publisher "%s"' % entry.publisher,
'\t' + 'rom ( name %s size %s crc %s md5 %s sha1 %s )' % (entry.zip, entry.size, entry.crc, entry.md5, entry.sha1),
')',
'']
game_list.append('\n'.join(text_entry))
return '\n'.join(game_list)
def output(args, header, game_list):
"""Combine <header> and <game_list> and print to stdout, or write to specified --output-file"""
if not args.output_file:
# print to stdout
print('\n'.join([header, '', game_list]).rstrip())
else:
# write to args.output_file
with open(args.output_file, 'w') as f:
f.write('\n'.join([header, '', game_list]))
class GameEntry:
name = None
year = None
publisher = None
zip = None
size = None
crc = None
md5 = None
sha1 = None
def get_crc(file):
"""Return the CRC32 hash of <file>"""
prev = 0
for eachLine in open(file, 'rb'):
prev = zlib.crc32(eachLine, prev)
return str("%X"%(prev & 0xFFFFFFFF)).zfill(8)
def get_md5(file):
"""Return the MD5 hash of <file>"""
m = hashlib.md5()
with open(file, 'rb') as f:
for chunk in iter(lambda: f.read(4096), b""):
m.update(chunk)
return m.hexdigest().zfill(32)
def get_sha1(file):
"""Return the SHA1 hash of <file>"""
m = hashlib.sha1()
with open(file, 'rb') as f:
for chunk in iter(lambda: f.read(4096), b""):
m.update(chunk)
return m.hexdigest().zfill(40)
if __name__ == '__main__':
main()