mirror of
https://github.com/libretro/pcsx2.git
synced 2024-12-22 17:58:38 +00:00
dfe08e3c96
Now also summarizes regions by default and also lists compatibility improvements by default (both can be configured to be silent).
456 lines
15 KiB
HTML
456 lines
15 KiB
HTML
<html>
|
|
<meta charset="utf-8">
|
|
<title>PCSX2 games index compatibility updater</title>
|
|
<head>
|
|
<script>
|
|
/*
|
|
Quick and (very) dirty gameindex.dbf update tool from website compatibility db data.csv.
|
|
- works in Firefox since Chrome doesn't allow loading local files.
|
|
- updates compatibility info at the dbf from data.csv
|
|
- add entries to the dbf if they only appear on data.csv
|
|
- sorts the dbf entries according to the serial number
|
|
- keeps hacks/comments/etc untouched (except when modifying compatibility info - and prints a warning)
|
|
- can import external html files with games info and compare the name/region to our own data
|
|
*/
|
|
var CONFIG_SKIP_SORT_1 = 0; // sort after initial dbf import
|
|
var CONFIG_SKIP_REGIONS_SUMMARY = 0;
|
|
var CONFIG_SKIP_IMPORT_CSV = 0;
|
|
var CONFIG_SKIP_ADD_MISSING_FROM_CSV = 0;
|
|
var CONFIG_SKIP_LOG_CSV_MISSING = 0; // files which have compat info at the dbf but not at the csv
|
|
var CONFIG_SKIP_LOG_IMPROVEMENTS = 0; // the common case of data.csv having better compat than the current dbf.
|
|
var CONFIG_SKIP_SORT_2 = 0; // sort after adding entries from data.csv
|
|
var CONFIG_SKIP_HTML_INDEX_IMPORTS = 1;
|
|
|
|
var DBF_PATH = "GameIndex.dbf";
|
|
var COMPAT_PATH = "data.csv";
|
|
var HTML_INDEX_FILES = ["serials_1.htm", "serials_2.htm", "serials_3.htm"];
|
|
|
|
var COMPAT_DELIMITER = "\t";
|
|
var COMPAT_COLUMNS = {Serial: 0, Compat: 1, Name: 5, Region: 6};
|
|
|
|
var DBF_KEYS = ["Serial", "Name", "Region", "Compat"];
|
|
var DBF_DELIMITER = "\n---------------------------------------------\n";
|
|
var DBF_HEADER_END = DBF_DELIMITER + "-- Game List" + DBF_DELIMITER;
|
|
|
|
|
|
function escapeHtml(str) { return str.split("&").join("&")
|
|
.split("<").join("<")
|
|
.split(">").join(">")
|
|
.split("\"").join("""); }
|
|
function $(id){ return document.getElementById(id); }
|
|
function log(text) { $("logout").innerHTML += escapeHtml(text); }
|
|
function logln(text) { log(text + "\n"); }
|
|
|
|
function serPretty(ser) { ser = ser.toUpperCase(); return ser.substr(0,4) + "-" + ser.substr(4); }
|
|
function noComment(txt) { return txt.split("//")[0].trim(); }
|
|
function serNor(ser) { return noComment(ser).split("-").join("").toLowerCase(); }
|
|
|
|
function pad(txt, len, ch) {
|
|
txt = "" + txt;
|
|
ch = ch || " ";
|
|
while (txt.length < len)
|
|
txt += ch;
|
|
return txt;
|
|
}
|
|
|
|
// Synchronous. Returns "" on failure, or file content on success.
|
|
// Use Firefox. chrome doesn't allow loading local files even if at the same dir as this file.
|
|
function getFile(URI, failQuietly) {
|
|
var result = "";
|
|
|
|
try {
|
|
function reqListener () {
|
|
result = this.responseText;
|
|
logln("Got file '" + URI + "', length: " + result.length);
|
|
}
|
|
|
|
var oReq = new XMLHttpRequest();
|
|
oReq.onload = reqListener;
|
|
oReq.open("get", URI, false);
|
|
oReq.overrideMimeType("text/plain; charset=x-user-defined");
|
|
oReq.send();
|
|
} catch(e) {
|
|
if (!failQuietly)
|
|
logln("Error file '" + URI + "' (" + e + ")");
|
|
return "";
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
var exportData = [];
|
|
function parseDbf(dbf, startLine) {
|
|
|
|
function importGame(section, firstLine) {
|
|
if (!firstLine) firstLine = 0;
|
|
var result = {};
|
|
var extra = [];
|
|
var lines = String(section).trim().split("\n");
|
|
for (var i in lines) {
|
|
var line = lines[i].trim();
|
|
if (line.indexOf("--------") == 0 ){
|
|
logln("Warning: invalid separator? : '" + line + "'");
|
|
}
|
|
var lineLow = line.toLowerCase();
|
|
var foundKey = false;
|
|
|
|
for (var k in DBF_KEYS) {
|
|
var rest = line.substr(DBF_KEYS[k].length).trim();
|
|
if (lineLow.indexOf(DBF_KEYS[k].toLowerCase()) == 0 && rest.indexOf("=") == 0) {
|
|
// known key
|
|
var key = DBF_KEYS[k];
|
|
var value = rest.substr(1).trim();
|
|
if (!(key in result)) {
|
|
if (key == "Serial") {
|
|
value = value.split("-").join("");
|
|
}
|
|
result[key] = value;
|
|
foundKey = true;
|
|
break;
|
|
|
|
} else {
|
|
logln ("Error: rejecting DBF Line: " + (firstLine + i) + " redefine: " + DBF_KEYS[k]);
|
|
}
|
|
} else if (lineLow.indexOf(DBF_KEYS[k].toLowerCase()) == 0) {
|
|
// known yet without '=', warn.
|
|
logln("Warning: Line " + (firstLine + i) + ", no '='? '" + line + "'");
|
|
}
|
|
}
|
|
|
|
if (!foundKey) {
|
|
// unknown/no key (comment, patch, hack, empty, etc)
|
|
extra.push(lines[i]); // because 'line' is trimmed
|
|
}
|
|
} // lines
|
|
|
|
result["extra"] = extra;
|
|
return result;
|
|
}
|
|
|
|
var result = {header: "", games: []};
|
|
|
|
logln("import-dbf-start");
|
|
var dbf = getFile(DBF_PATH);
|
|
|
|
var dbfs = dbf.split(DBF_HEADER_END);
|
|
logln("found main sections: " + dbfs.length);
|
|
|
|
var gameSections = String(dbfs[1]).split(DBF_DELIMITER);
|
|
if (gameSections[gameSections.length-1].length == 0) {
|
|
gameSections.pop();
|
|
}
|
|
logln("Found game sections: " + gameSections.length);
|
|
|
|
logln("Importing games ...");
|
|
var gamesDb = [];
|
|
var serialIndex= {};
|
|
var compatInfo = [];
|
|
var dbfRegions = {};
|
|
for (var g in gameSections) {
|
|
var game = importGame(gameSections[g]);
|
|
if (!("Serial" in game) && !("Name" in game) && !("Region" in game) && !("Compat" in game) && game.extra.length == 1) {
|
|
logln("Warning: Empty section (skipping)");
|
|
} else {
|
|
if (!("Serial" in game) || !("Name" in game) || !("Region" in game)) {
|
|
logln("Warning: Incomplete: ser="+game.Serial.substr(0,4)+"-"+game.Serial.substr(4)+" name="+game.Name+ " region="+game.Region);
|
|
}
|
|
gamesDb.push(game);
|
|
if (game.Serial) {
|
|
var ser = serNor(game.Serial);
|
|
if (ser in serialIndex) {
|
|
logln("Warning: duplicate serial at dbf: " + serPretty(ser));
|
|
} else {
|
|
serialIndex[ser] = gamesDb.length - 1;
|
|
}
|
|
}
|
|
if ("Compat" in game) {
|
|
var c = game.Compat.substr(0,1);
|
|
if (!(c in compatInfo)) compatInfo[c] = 0;
|
|
compatInfo[c]++;
|
|
}
|
|
}
|
|
if (!dbfRegions[game.Region])
|
|
dbfRegions[game.Region] = 0;
|
|
dbfRegions[game.Region]++;
|
|
}
|
|
logln("Imported entries: " + gamesDb.length);
|
|
for (var i in compatInfo)
|
|
logln("Compatibility " + i + ": " + compatInfo[i] + " games");
|
|
|
|
if (!CONFIG_SKIP_REGIONS_SUMMARY) {
|
|
for (var r in dbfRegions)
|
|
logln("" + r + ": " + dbfRegions[r]);
|
|
}
|
|
|
|
|
|
function sortDbf() {
|
|
gamesDb.sort(function(a, b) {
|
|
// no need for equal since there are no duplicates
|
|
return serNor(a.Serial) > serNor(b.Serial) ? 1 : -1;
|
|
});
|
|
for (var i = 0; i < gamesDb.length; i++) {
|
|
serialIndex[serNor(gamesDb[i].Serial)] = i;
|
|
}
|
|
}
|
|
|
|
logln("Checking sort ...");
|
|
var prevg = "";
|
|
var sorted = true;
|
|
for (var g in serialIndex) {
|
|
if (g < prevg) {
|
|
logln ("bad sort: " + serPretty(prevg) + " before " + serPretty(g));
|
|
sorted = false;
|
|
}
|
|
prevg = g;
|
|
}
|
|
|
|
if (sorted) {
|
|
logln("dbf sort: no need (already sorted)");
|
|
} else {
|
|
if (CONFIG_SKIP_SORT_1) {
|
|
logln("skipping sort1");
|
|
} else {
|
|
logln("Sorting dbf...");
|
|
sortDbf();
|
|
logln("Done Sorting dbf.");
|
|
}
|
|
}
|
|
|
|
logln("import-dbf-end");
|
|
|
|
function dbfRegionFromCompatRegion(csvRegion) {
|
|
// currently data.csv only has NTSC/PAL/JAP
|
|
// we're translating to the dbf as: NTSC-U/PAL-Unk/NTSC-J respectively
|
|
// can later manually fix or set more specific region data.
|
|
// return undefined if unexpected region name at data.csv
|
|
var trans = {
|
|
"NTSC": "NTSC-U",
|
|
"PAL" : "PAL-Unk",
|
|
"JAP" : "NTSC-J"
|
|
};
|
|
return trans[csvRegion];
|
|
}
|
|
|
|
logln("");
|
|
if (CONFIG_SKIP_IMPORT_CSV) {
|
|
logln("skipping data.csv import");
|
|
} else {
|
|
logln("start import compatibility ...");
|
|
var compat = getFile(COMPAT_PATH);
|
|
compat = compat.split("\n")
|
|
logln("Found " + compat.length + " compatibility info lines");
|
|
var updateSummary = {same: 0, "new": 0, better: 0, worse: 0, missingAtCsv: 0};
|
|
var compatVerification = {};
|
|
|
|
var compatRegions = {};
|
|
for (var i in compat) {
|
|
if (!compat[i].trim().length) continue;
|
|
var cols = compat[i].split(COMPAT_DELIMITER);
|
|
var found = false;
|
|
var ser = cols[COMPAT_COLUMNS.Serial].trim().toLowerCase();
|
|
var com = Number(cols[COMPAT_COLUMNS.Compat].trim());
|
|
var reg = cols[COMPAT_COLUMNS.Region].trim();
|
|
|
|
if (!compatRegions[reg])
|
|
compatRegions[reg] = 0;
|
|
compatRegions[reg]++;
|
|
|
|
if (!(ser in compatVerification)) {
|
|
compatVerification[ser] = com;
|
|
} else {
|
|
logln("Warning (skipping): duplicate compat info (" + compatVerification[ser] + "/" + com + ") for: " + serPretty(ser));
|
|
continue;
|
|
}
|
|
if (ser.toLowerCase() in serialIndex) {
|
|
var game = gamesDb[serialIndex[ser]];
|
|
var dbser = serNor(game.Serial);
|
|
var dbcom = Number(("Compat" in game) ? game.Compat.split("//")[0].trim() : "0");
|
|
if (dbser != ser) {
|
|
logln("Error: internal: invalid index at serialIndex for " + serPretty(ser));
|
|
continue;
|
|
}
|
|
|
|
if (com != dbcom) {
|
|
//logln("Updating compat from " + dbcom + " to " + com + " for " + ser + ": " + game.Name);
|
|
if (("Compat" in game) && game.Compat.split("//").length > 1) {
|
|
logln("Warning: removing compat comment for serial " + serPretty(ser) + " : //" + game.Compat.split("//")[1]);
|
|
}
|
|
game.Compat = com;
|
|
|
|
if (com > dbcom) {
|
|
updateSummary.better++
|
|
if (!CONFIG_SKIP_LOG_IMPROVEMENTS)
|
|
logln("--> ++ Improving compat from " + dbcom + " to " + com + " for " + serPretty(ser) + ": " + game.Name);
|
|
} else {
|
|
updateSummary.worse++;
|
|
logln("--> -- Degrading compat from " + dbcom + " to " + com + " for " + serPretty(ser) + ": " + game.Name);
|
|
}
|
|
} else {
|
|
updateSummary.same++;
|
|
}
|
|
} else {
|
|
updateSummary.new++;
|
|
logln ("not at dbf: " + serPretty(ser) + " - " + (CONFIG_SKIP_ADD_MISSING_FROM_CSV ? "skipping" : "adding:"));
|
|
if (!CONFIG_SKIP_ADD_MISSING_FROM_CSV) {
|
|
var newDbfReg = dbfRegionFromCompatRegion(reg);
|
|
if (newDbfReg) {
|
|
gamesDb.push({Serial: ser, Compat: com, Region: newDbfReg, Name: cols[COMPAT_COLUMNS.Name].trim(), extra: ""});
|
|
log(getExportEntry(gamesDb[gamesDb.length - 1]) + DBF_DELIMITER);
|
|
} else {
|
|
logln("Error: skipping due to unexpected region at data.csv: " + reg);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (compat && compat.length > 1) {
|
|
if (updateSummary.new) {
|
|
if (CONFIG_SKIP_SORT_2) {
|
|
logln("skipping sort after add");
|
|
} else {
|
|
logln("sorting new entries into the dbf...");
|
|
sortDbf();
|
|
logln("sort - done.");
|
|
}
|
|
}
|
|
logln("-- Update summary --");
|
|
|
|
if (!CONFIG_SKIP_REGIONS_SUMMARY) {
|
|
logln("data.csv regions:");
|
|
for (var r in compatRegions)
|
|
logln("" + r + ": " + compatRegions[r]);
|
|
}
|
|
if (!CONFIG_SKIP_LOG_CSV_MISSING) {
|
|
for (var csvSer in serialIndex) {
|
|
var game = gamesDb[serialIndex[csvSer]];
|
|
if (game.Compat && !compatVerification[serNor(csvSer)]) {
|
|
logln("serial missing from CSV: " + pad(serPretty(csvSer), 12) + " / " + pad(noComment(game.Region), 8) + " / " + game.Compat + " / " + game.Name);
|
|
updateSummary.missingAtCsv++;
|
|
}
|
|
}
|
|
}
|
|
|
|
logln("Not at dbf (and " +(CONFIG_SKIP_ADD_MISSING_FROM_CSV ? "skipped" : "added")+ "): " + updateSummary.new);
|
|
logln("Not at csv (but have compat info at the dbf): " + updateSummary.missingAtCsv);
|
|
logln("Unchanged: " + updateSummary.same);
|
|
logln("Better compat: " + updateSummary.better);
|
|
logln("Worse compat: " + updateSummary.worse);
|
|
logln("End import compatibility.");
|
|
}
|
|
}
|
|
|
|
|
|
logln("");
|
|
if (CONFIG_SKIP_HTML_INDEX_IMPORTS) {
|
|
logln("skipping html index files");
|
|
} else {
|
|
logln("Scanning html index files");
|
|
var in_incomplete = 0;
|
|
var in_identical = 0;
|
|
var in_diff = 0;
|
|
var in_new = 0;
|
|
var in_err = 0;
|
|
var in_usBetter = 0;
|
|
var files = HTML_INDEX_FILES;
|
|
for (var f in files) {
|
|
var serials = getFile(files[f]);
|
|
logln("file sections: " + serials.split('<table border="1"').length);
|
|
if (!serials.length) {
|
|
logln("Warning: file not found. skipping.");
|
|
continue;
|
|
}
|
|
var list = serials.split('<table border="1"')[1].split("</tr>");
|
|
logln("found rows: " + list.length);
|
|
for (var l in list) {
|
|
var s = list[l].split("</td>");
|
|
if (!(s.length==4)) {
|
|
in_incomplete++;
|
|
continue;
|
|
}
|
|
try {
|
|
var ser = serNor(s[0].split('.htm">')[1].split("</pre>")[0]);
|
|
var name = s[1].split(' 0">')[1].split("</pre>")[0];
|
|
var region = s[2].split(' 0">')[1].split("</pre>")[0];
|
|
|
|
if (ser in serialIndex) {
|
|
var game = gamesDb[serialIndex[ser]];
|
|
function norName(n) {
|
|
n = n.toLowerCase().split("//")[0];
|
|
var cs = [", ", ".", "-", ":"];
|
|
for (var i in cs)
|
|
n = n.split(cs[i]).join("");
|
|
return n.split(" ").join("").split(" ").join("").trim();
|
|
}
|
|
if (norName(game.Name) == norName(name) && norName(game.Region) == norName(region)) {
|
|
in_identical++
|
|
} else {
|
|
if ((norName(region).indexOf("unk")>=0 && norName(game.Region).indexOf("unk")<0)|| norName(name).indexOf("zzz")==0)
|
|
in_usBetter++
|
|
else {
|
|
logln("diff: " + serPretty(ser) + " DB: [" + game.Region.split("//")[0].trim() + "] " + game.Name.split("//")[0].trim());
|
|
logln(" index: [" + region + "] "+ name);
|
|
logln("");
|
|
in_diff++;
|
|
}
|
|
}
|
|
} else {
|
|
in_new++;
|
|
}
|
|
} catch (e) {
|
|
in_err++;
|
|
logln(e);
|
|
}
|
|
}
|
|
}
|
|
logln("--- Html index summary: ---");
|
|
logln("errors: " + in_err);
|
|
logln("incomplete data: " + in_incomplete);
|
|
logln("identical: " + in_identical);
|
|
logln("we're better: " + in_usBetter);
|
|
logln("different: "+ in_diff);
|
|
logln("not in db: " + in_new);
|
|
}
|
|
|
|
|
|
// start: test export dbf
|
|
function getExportEntry(gameItem) {
|
|
var entry = [];
|
|
for (var k in DBF_KEYS) {
|
|
if (DBF_KEYS[k] in gameItem) {
|
|
var v = gameItem[DBF_KEYS[k]];
|
|
var fill = "";
|
|
if (DBF_KEYS[k] == "Serial") v = (v.substr(0,4) + "-" + v.substr(4)).toUpperCase();
|
|
if (DBF_KEYS[k] == "Name") fill = " ";
|
|
entry.push(DBF_KEYS[k] + fill + " = " + v);
|
|
}
|
|
}
|
|
if (gameItem.extra.length) {
|
|
entry.push(gameItem.extra.join("\n"));
|
|
}
|
|
return entry.join("\n");
|
|
}
|
|
exportData = [];
|
|
for (var g in gamesDb) {
|
|
exportData.push(getExportEntry(gamesDb[g]));
|
|
}
|
|
exportData = dbfs[0] + DBF_HEADER_END + exportData.join(DBF_DELIMITER) + DBF_DELIMITER;
|
|
$("saveUpdated").href = "data:bin/plain;base64," + window.btoa(exportData);
|
|
$("saveUpdatedWrapper").style.display = "inline";
|
|
// end: export
|
|
|
|
}
|
|
|
|
</script>
|
|
</head>
|
|
|
|
<body onload="javascript:parseDbf()">
|
|
Please copy GameIndex.dbf (from the bin folder) and data.csv (compat info) to the folder of this html file<br/>
|
|
<span id="saveUpdatedWrapper" style="display:none;">
|
|
<a id="saveUpdated" href="" download="GameIndex.new.dbf">Save updated DBF</a>
|
|
</span>
|
|
|
|
<h4>Log</h4>
|
|
<pre><div id="logout"></div></pre>
|
|
|
|
</body>
|
|
</html>
|