Update to v091r06 release.

byuu says:

This release adds initial database support.

The way it works is you can now load game folders as you always have, or
you can load a game file. If you load a game file, it tries to create
a game folder for you by looking up the file's sha256 in a database. If
it can't find it, sorry, the game won't play. I'm not hooking up the
oldschool "make up a manifest" code here. The easiest way to handle this
is to get me every game so I can dump it and add it to the database :D

The database entries are complete entries that can be copied directly.
So it describes the board, the information, file layout, etc. That'll be
what comes with higan releases in the future.

Internally, I'm separating the information and board descriptions, and
will use a tool to merge the two together.

Here's a current database copy, with one game in it. Still hammering out
some details, but it's mostly how it's going to look.

    cartridge region=NTSC
	board type=1CB5B-20
	    superfx revision=2
		rom name=program.rom size=0x200000
		ram name=save.rwm size=0x8000
		map id=io address=00-3f,80-bf:3000-32ff
		map id=rom address=00-3f:8000-ffff mask=0x8000
		map id=rom address=40-5f:0000-ffff
		map id=ram address=00-3f,80-bf:6000-7fff size=0x2000
		map id=ram address=70-71:0000-ffff
	information
	    name:   Super Mario World 2 - Yoshi's Island (SNS) (1.1)
	    title:  Super Mario World 2: Yoshi's Island
	    sha256: bd763c1a56365c244be92e6cffefd318780a2a19eda7d5baf1c6d5bd6c1b3e06
	    board:  SHVC-1CB5B-20
	    rom:    0x200000
	    ram:    0x8000
	layout
	    file name=program.rom size=0x200000

higan/emulator/emulator.hpp

@@ -3,7 +3,7 @@
 
 namespace Emulator {
   static const char Name[] = "higan";
-  static const char Version[] = "091.05";
+  static const char Version[] = "091.06";
   static const char Author[] = "byuu";
   static const char License[] = "GPLv3";
 }

higan/nall/crc16.hpp

@@ -0,0 +1,25 @@
+#ifndef NALL_CRC16_HPP
+#define NALL_CRC16_HPP
+
+#include <nall/stdint.hpp>
+
+namespace nall {
+  inline uint16_t crc16_adjust(uint16_t crc16, uint8_t data) {
+    for(unsigned n = 0; n < 8; n++) {
+      if((crc16 & 1) ^ (data & 1)) crc16 = (crc16 >> 1) ^ 0x8408;
+      else crc16 >>= 1;
+      data >>= 1;
+    }
+    return crc16;
+  }
+
+  inline uint16_t crc16_calculate(const uint8_t *data, unsigned length) {
+    uint16_t crc16 = ~0;
+    for(unsigned n = 0; n < length; n++) {
+      crc16 = crc16_adjust(crc16, data[n]);
+    }
+    return ~crc16;
+  }
+}
+
+#endif

higan/nall/file.hpp

@@ -90,6 +90,11 @@ namespace nall {
       return true;
     }
 
+    static string sha256(const string &filename) {
+      auto buffer = read(filename);
+      return nall::sha256(buffer.data(), buffer.size());
+    }
+
     uint8_t read() {
       if(!fp) return 0xff;                       //file not open
       if(file_mode == mode::write) return 0xff;  //reads not permitted

higan/nall/nall.hpp

@@ -13,6 +13,7 @@
 #include <nall/bit.hpp>
 #include <nall/bmp.hpp>
 #include <nall/config.hpp>
+#include <nall/crc16.hpp>
 #include <nall/crc32.hpp>
 #include <nall/directory.hpp>
 #include <nall/dl.hpp>

higan/sfc/cartridge/cartridge.hpp

@@ -76,7 +76,7 @@ private:
   void parse_markup_map(Mapping&, Markup::Node&);
   void parse_markup_memory(MappedRAM&, Markup::Node&, unsigned id, bool writable);
 
-  void parse_markup_cartridge(Markup::Node&);
+  void parse_markup_board(Markup::Node&);
   void parse_markup_icd2(Markup::Node&);
   void parse_markup_bsx(Markup::Node&);
   void parse_markup_bsxslot(Markup::Node&);

higan/sfc/cartridge/markup.cpp

@@ -5,26 +5,27 @@ void Cartridge::parse_markup(const char *markup) {
 
   auto document = Markup::Document(markup);
   auto &cartridge = document["cartridge"];
+  auto &board = cartridge["board"];
   region = cartridge["region"].data != "PAL" ? Region::NTSC : Region::PAL;
 
-  parse_markup_cartridge(cartridge);
-  parse_markup_icd2(cartridge["icd2"]);
-  parse_markup_bsx(cartridge["bsx"]);
-  parse_markup_bsxslot(cartridge["bsxslot"]);
-  parse_markup_sufamiturbo(cartridge["sufamiturbo"]);
-  parse_markup_nss(cartridge["nss"]);
-  parse_markup_event(cartridge["event"]);
-  parse_markup_sa1(cartridge["sa1"]);
-  parse_markup_superfx(cartridge["superfx"]);
-  parse_markup_armdsp(cartridge["armdsp"]);
-  parse_markup_hitachidsp(cartridge["hitachidsp"]);
-  parse_markup_necdsp(cartridge["necdsp"]);
-  parse_markup_epsonrtc(cartridge["epsonrtc"]);
-  parse_markup_sharprtc(cartridge["sharprtc"]);
-  parse_markup_spc7110(cartridge["spc7110"]);
-  parse_markup_sdd1(cartridge["sdd1"]);
-  parse_markup_obc1(cartridge["obc1"]);
-  parse_markup_msu1(cartridge["msu1"]);
+  parse_markup_board(board);
+  parse_markup_icd2(board["icd2"]);
+  parse_markup_bsx(board["bsx"]);
+  parse_markup_bsxslot(board["bsxslot"]);
+  parse_markup_sufamiturbo(board["sufamiturbo"]);
+  parse_markup_nss(board["nss"]);
+  parse_markup_event(board["event"]);
+  parse_markup_sa1(board["sa1"]);
+  parse_markup_superfx(board["superfx"]);
+  parse_markup_armdsp(board["armdsp"]);
+  parse_markup_hitachidsp(board["hitachidsp"]);
+  parse_markup_necdsp(board["necdsp"]);
+  parse_markup_epsonrtc(board["epsonrtc"]);
+  parse_markup_sharprtc(board["sharprtc"]);
+  parse_markup_spc7110(board["spc7110"]);
+  parse_markup_sdd1(board["sdd1"]);
+  parse_markup_obc1(board["obc1"]);
+  parse_markup_msu1(board["msu1"]);
 }
 
 //
@@ -48,7 +49,7 @@ void Cartridge::parse_markup_memory(MappedRAM &ram, Markup::Node &node, unsigned
 
 //
 
-void Cartridge::parse_markup_cartridge(Markup::Node &root) {
+void Cartridge::parse_markup_board(Markup::Node &root) {
   if(root.exists() == false) return;
 
   parse_markup_memory(rom, root["rom"], ID::ROM, false);

higan/target-ethos/configuration/configuration.cpp

@@ -25,6 +25,7 @@ Configuration::Configuration() {
   append(input.focusAllow = false, "Input::Focus::AllowInput");
   append(timing.video = 60.0, "Timing::Video");
   append(timing.audio = 48000.0, "Timing::Audio");
+  append(path.game = {userpath(), "Emulation/"}, "Path::Game");
   load();
 }
 

higan/target-ethos/configuration/configuration.hpp

@@ -34,6 +34,10 @@ struct Configuration : configuration {
     double audio;
   } timing;
 
+  struct Path {
+    string game;
+  } path;
+
   void load();
   void save();
   Configuration();

higan/target-ethos/ethos.cpp

@@ -93,6 +93,7 @@ Application::Application(int argc, char **argv) {
   inputSettings = new InputSettings;
   hotkeySettings = new HotkeySettings;
   timingSettings = new TimingSettings;
+  pathSettings = new PathSettings;
   driverSettings = new DriverSettings;
   settings = new Settings;
   cheatDatabase = new CheatDatabase;

higan/target-ethos/general/browser.cpp

@@ -157,14 +157,14 @@ void Browser::setPath(const string &path, unsigned selection) {
   fileList.reset();
   filenameList.reset();
 
-  lstring contents = directory::folders(path);
+  lstring contents = directory::contents(path);
 
   for(auto &filename : contents) {
-    if(!filename.wildcard(R"(*.??/)") && !filename.wildcard(R"(*.???/)")) {
+    if(!filename.wildcard(R"(*.??/)") && !filename.wildcard(R"(*.???/)") && filename.endswith("/")) {
       string name = filename;
       name.rtrim<1>("/");
       fileList.append(name);
-      fileList.setImage(filenameList.size(), 0, image(resource::folder, sizeof resource::folder));
+      fileList.setImage(filenameList.size(), 0, image(resource::systemFolder, sizeof resource::systemFolder));
       filenameList.append(filename);
     }
   }
@@ -176,7 +176,19 @@ void Browser::setPath(const string &path, unsigned selection) {
         string name = filename;
         name.rtrim<1>(suffix);
         fileList.append(name);
-        fileList.setImage(filenameList.size(), 0, image(resource::game, sizeof resource::game));
+        fileList.setImage(filenameList.size(), 0, image(resource::gameFolder, sizeof resource::gameFolder));
+        filenameList.append(filename);
+      }
+    }
+  }
+
+  for(auto &filename : contents) {
+    string suffix = {".", this->extension};
+    if(filename.wildcard(R"(*.??)") || filename.wildcard(R"(*.???)")) {
+      if(filename.endswith(suffix)) {
+        string name = filename;
+        fileList.append(name);
+        fileList.setImage(filenameList.size(), 0, image(resource::gameFile, sizeof resource::gameFile));
         filenameList.append(filename);
       }
     }
@@ -190,8 +202,7 @@ void Browser::setPath(const string &path, unsigned selection) {
 void Browser::fileListActivate() {
   unsigned selection = fileList.selection();
   string filename = filenameList[selection];
-  string suffix = {this->extension, "/"};
-  if(filename.endswith(suffix) == false) return setPath({path, filename});
+  if(string{filename}.rtrim<1>("/").endswith(this->extension) == false) return setPath({path, filename});
 
   dialogActive = false;
   outputFilename = {path, filename};

higan/target-ethos/resource/resource.cpp

@@ -1,6 +1,30 @@
 namespace resource {
 
-const uint8_t folder[1176] = {
+const uint8_t systemFolder[667] = {
+  137,80,78,71,13,10,26,10,0,0,0,13,73,72,68,82,0,0,0,32,0,0,0,32,8,6,0,0,0,115,122,122,
+  244,0,0,0,4,115,66,73,84,8,8,8,8,124,8,100,136,0,0,2,82,73,68,65,84,88,133,237,148,205,107,19,
+  81,20,197,127,153,36,218,84,48,65,138,164,74,186,16,108,236,34,75,65,148,110,252,8,214,165,127,128,165,160,224,206,
+  86,93,136,46,92,186,16,138,244,95,40,93,235,170,136,85,112,163,130,11,65,92,104,235,194,133,133,150,154,106,172,249,
+  232,100,230,189,235,98,38,166,38,157,201,100,38,208,133,30,120,48,195,220,243,238,185,119,238,61,240,31,255,58,98,221,
+  2,46,93,200,223,32,158,124,148,140,119,13,237,128,105,105,12,81,183,22,151,62,205,134,82,119,177,56,54,51,115,245,
+  108,93,153,191,68,180,45,90,53,68,219,166,104,171,46,186,81,21,213,168,136,50,183,68,109,255,20,85,255,33,118,109,
+  83,236,90,73,236,234,134,216,149,117,217,46,125,150,235,147,227,245,137,226,137,219,94,57,60,203,154,40,230,239,141,100,
+  15,223,157,155,91,24,180,54,222,83,253,178,212,115,1,169,163,167,48,210,199,152,190,115,179,190,250,109,243,254,211,103,
+  203,15,3,11,56,127,110,76,22,159,188,196,92,123,75,45,68,242,38,6,142,156,36,150,62,206,229,169,41,158,191,248,
+  216,145,47,225,71,54,146,41,222,205,95,9,157,220,193,60,133,201,199,158,95,125,5,104,219,4,224,195,87,155,181,178,
+  238,57,245,112,198,160,144,75,32,170,17,82,128,178,0,88,47,107,78,143,38,187,175,204,14,8,240,102,197,162,144,195,
+  87,128,225,123,137,75,20,2,236,107,27,98,46,15,90,133,132,16,224,77,236,5,161,59,160,125,136,189,192,239,158,64,
+  67,184,27,94,173,116,118,231,204,104,210,227,158,144,2,154,191,160,249,63,219,231,32,63,84,253,243,188,92,58,240,55,
+  119,71,124,132,45,112,136,217,140,193,235,93,42,110,79,218,222,149,225,140,225,10,240,158,165,64,29,40,228,18,20,114,
+  126,145,254,8,61,3,205,214,69,53,162,240,67,232,18,247,204,136,116,223,140,40,162,19,70,133,223,16,118,233,64,159,
+  156,48,180,15,248,16,123,50,162,168,67,24,213,136,34,175,97,31,141,40,5,212,131,8,24,132,150,242,62,26,209,8,
+  80,6,74,128,242,18,16,3,210,224,40,207,141,79,35,162,64,43,68,52,162,21,136,66,180,123,68,187,223,154,239,10,
+  180,118,98,155,188,150,128,44,206,224,91,192,119,47,1,113,247,112,237,193,66,248,178,119,199,190,246,156,94,254,114,8,
+  24,2,50,192,0,93,214,53,0,76,55,113,21,167,253,171,128,246,19,0,206,28,28,4,246,247,65,128,237,38,220,2,
+  42,180,76,114,239,241,27,4,79,89,165,234,42,236,66,0,0,0,0,73,69,78,68,174,66,96,130,
+};
+
+const uint8_t gameFolder[1176] = {
   137,80,78,71,13,10,26,10,0,0,0,13,73,72,68,82,0,0,0,32,0,0,0,32,8,6,0,0,0,115,122,122,
   244,0,0,0,4,115,66,73,84,8,8,8,8,124,8,100,136,0,0,0,9,112,72,89,115,0,0,13,215,0,0,13,
   215,1,66,40,155,120,0,0,0,25,116,69,88,116,83,111,102,116,119,97,114,101,0,119,119,119,46,105,110,107,115,99,
@@ -40,7 +64,7 @@ const uint8_t folder[1176] = {
   235,6,248,7,188,50,165,151,203,8,55,43,0,0,0,0,73,69,78,68,174,66,96,130,
 };
 
-const uint8_t game[844] = {
+const uint8_t gameFile[844] = {
   137,80,78,71,13,10,26,10,0,0,0,13,73,72,68,82,0,0,0,32,0,0,0,32,8,6,0,0,0,115,122,122,
   244,0,0,0,4,115,66,73,84,8,8,8,8,124,8,100,136,0,0,3,3,73,68,65,84,88,133,229,151,79,110,212,
   48,24,197,127,182,227,56,81,39,116,36,132,0,169,167,226,2,101,209,93,239,192,150,37,171,46,43,245,4,92,164,183,

higan/target-ethos/resource/resource.hpp

@@ -1,6 +1,7 @@
 namespace resource {
-  extern const uint8_t folder[1176];
-  extern const uint8_t game[844];
+  extern const uint8_t systemFolder[667];
+  extern const uint8_t gameFolder[1176];
+  extern const uint8_t gameFile[844];
   extern const uint8_t home[1774];
   extern const uint8_t up[1193];
 };

higan/target-ethos/resource/resource.xml

@@ -1,7 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <resource name="resource">
-  <binary id="folder" name="folder.png"/>
-  <binary id="game" name="game.png"/>
+  <binary id="systemFolder" name="system-folder.png"/>
+  <binary id="gameFolder" name="game-folder.png"/>
+  <binary id="gameFile" name="game-file.png"/>
   <binary id="home" name="home.png"/>
   <binary id="up" name="up.png"/>
 </resource>

higan/target-ethos/settings/path.cpp

@@ -0,0 +1,21 @@
+PathSettings *pathSettings = nullptr;
+
+PathSettings::PathSettings() {
+  title.setFont(application->titleFont);
+  title.setText("Path Settings");
+  gameLabel.setText("Games:");
+  gamePath.setEditable(false);
+  gamePath.setText(config->path.game);
+  gameBrowse.setText("Browse ...");
+
+  append(title, {~0, 0}, 5);
+  append(gameLayout, {~0, 0});
+    gameLayout.append(gameLabel, {0, 0}, 5);
+    gameLayout.append(gamePath, {~0, 0}, 5);
+    gameLayout.append(gameBrowse, {80, 0});
+
+  gameBrowse.onActivate = [&] {
+    string path = DialogWindow::folderSelect(*settings, userpath());
+    if(path.empty() == false) gamePath.setText(config->path.game = path);
+  };
+}

higan/target-ethos/settings/path.hpp

@@ -0,0 +1,11 @@
+struct PathSettings : SettingsLayout {
+  Label title;
+  HorizontalLayout gameLayout;
+    Label gameLabel;
+    LineEdit gamePath;
+    Button gameBrowse;
+
+  PathSettings();
+};
+
+extern PathSettings *pathSettings;

higan/target-ethos/settings/settings.cpp

@@ -4,6 +4,7 @@
 #include "input.cpp"
 #include "hotkey.cpp"
 #include "timing.cpp"
+#include "path.cpp"
 #include "driver.cpp"
 Settings *settings = nullptr;
 
@@ -31,6 +32,7 @@ Settings::Settings() {
   panelList.append("Input");
   panelList.append("Hotkeys");
   panelList.append("Timing");
+  panelList.append("Paths");
   panelList.append("Driver");
 
   append(layout);
@@ -40,6 +42,7 @@ Settings::Settings() {
   append(*inputSettings);
   append(*hotkeySettings);
   append(*timingSettings);
+  append(*pathSettings);
   append(*driverSettings);
 
   onClose = [&] {
@@ -59,6 +62,7 @@ void Settings::panelChanged() {
   inputSettings->setVisible(false);
   hotkeySettings->setVisible(false);
   timingSettings->setVisible(false);
+  pathSettings->setVisible(false);
   driverSettings->setVisible(false);
   if(panelList.selected() == false) return;
 
@@ -68,6 +72,7 @@ void Settings::panelChanged() {
   case 2: return inputSettings->setVisible();
   case 3: return hotkeySettings->setVisible();
   case 4: return timingSettings->setVisible();
-  case 5: return driverSettings->setVisible();
+  case 5: return pathSettings->setVisible();
+  case 6: return driverSettings->setVisible();
   }
 }

higan/target-ethos/settings/settings.hpp

@@ -11,6 +11,7 @@ struct SettingsLayout : HorizontalLayout {
 #include "input.hpp"
 #include "hotkey.hpp"
 #include "timing.hpp"
+#include "path.hpp"
 #include "driver.hpp"
 
 struct Settings : Window {

higan/target-ethos/utility/database.cpp

@@ -0,0 +1,33 @@
+void Utility::loadMediaFromDatabase(Emulator::Interface *emulator, Emulator::Interface::Media &media, const string &filename) {
+  string sha256 = file::sha256(filename);
+  string systemPath = {configpath(), "higan/", emulator->information.name, ".sys/"};
+  string outputPath = {config->path.game, emulator->information.name, "/"};
+
+  string databaseText = string::read({systemPath, "database.bml"}).rtrim("\n");
+  lstring databaseItem = databaseText.split("\n\n");
+
+  for(auto &item : databaseItem) {
+    item.append("\n");
+    auto document = Markup::Document(item);
+    if(document["cartridge"]["information"]["sha256"].content() == sha256) {
+      string name = {document["cartridge"]["information"]["name"].content(), ".", extension(filename), "/"};
+      directory::create({outputPath, name});
+      file::write({outputPath, name, "manifest.bml"}, (const uint8_t*)(const char*)item, item.length());
+      auto buffer = file::read(filename);
+      unsigned offset = 0;
+      for(auto &node : document["cartridge"]["layout"]) {
+        if(node.name != "file") continue;
+        string filename = node["name"].content();
+        unsigned filesize = numeral(node["size"].content());
+        file::write({outputPath, name, filename}, buffer.data() + offset, filesize);
+        offset += filesize;
+      }
+      return loadMedia(emulator, media, {outputPath, name});
+    }
+  }
+
+  MessageWindow::warning(*presentation,
+    "Game not found in database. Mapping information unavailable.\n\n"
+    "As such, emulation cannot proceed."
+  );
+}

higan/target-ethos/utility/utility.cpp

@@ -1,4 +1,5 @@
 #include "../ethos.hpp"
+#include "database.cpp"
 
 Utility *utility = nullptr;
 
@@ -12,9 +13,12 @@ void Utility::loadMedia(Emulator::Interface *emulator, Emulator::Interface::Medi
   string pathname;
   if(!media.load.empty()) pathname = application->path({media.load, "/"});
   if(!directory::exists(pathname)) pathname = browser->select("Load Media", media.type);
-  if(!directory::exists(pathname)) return;
-  if(!file::exists({pathname, "manifest.xml"})) return;
-  loadMedia(emulator, media, pathname);
+  if(directory::exists(pathname) && file::exists({pathname, "manifest.bml"})) {
+    return loadMedia(emulator, media, pathname);
+  }
+  if(file::exists(pathname)) {
+    return loadMediaFromDatabase(emulator, media, pathname);
+  }
 }
 
 //load menu cartridge selected or command-line load
@@ -26,7 +30,7 @@ void Utility::loadMedia(Emulator::Interface *emulator, Emulator::Interface::Medi
   if(media.load.empty()) this->pathname.append(pathname);
 
   string manifest;
-  manifest.readfile({pathname, "manifest.xml"});
+  manifest.readfile({pathname, "manifest.bml"});
   system().load(media.id, manifest);
   system().power();
 
@@ -43,7 +47,7 @@ void Utility::loadRequest(unsigned id, const string &name, const string &type) {
   this->pathname.append(pathname);
 
   string manifest;
-  manifest.readfile({pathname, "manifest.xml"});
+  manifest.readfile({pathname, "manifest.bml"});
   system().load(id, manifest);
 }
 

higan/target-ethos/utility/utility.hpp

@@ -3,6 +3,7 @@ struct Utility {
 
   void loadMedia(Emulator::Interface *emulator, Emulator::Interface::Media &media);
   void loadMedia(Emulator::Interface *emulator, Emulator::Interface::Media &media, const string &pathname);
+  void loadMediaFromDatabase(Emulator::Interface *emulator, Emulator::Interface::Media &media, const string &filename);
 
   void loadRequest(unsigned id, const string &name, const string &type);
   void loadRequest(unsigned id, const string &path);