Implement MP3 file support (hackily: immediately decoded to PCM)

.gitmodules

@@ -8,3 +8,6 @@
 [submodule "openal-soft"]
 	path = vendor/openal-soft
 	url = https://github.com/kcat/openal-soft
+[submodule "vendor/dr_libs"]
+	path = vendor/dr_libs
+	url = https://github.com/mackron/dr_libs.git

CHANGELOG.md

@@ -18,6 +18,7 @@ Compatibility:
 - API support improvements:
   - Various small contributions. (@hikari-no-yume, @nitinseshadri)
   - Some key parts of `UIImage`, `CGImage` and `CGBitmapContext` used by Apple's `Texture2D` sample code are now implemented. Loading textures from PNG files in this way should now work. (@hikari-no-yume)
+  - MP3 is now a supported audio file format in Audio Toolbox. This is done in a fairly hacky way so it might not work for some apps. (@hikari-no-yume)
 - New supported app: Super Monkey Ball Lite.
 
 Quality:

Cargo.lock

@@ -655,6 +655,7 @@ dependencies = [
  "rusttype",
  "sdl2",
  "sdl2-sys",
+ "touchHLE_dr_mp3_wrapper",
  "touchHLE_dynarmic_wrapper",
  "touchHLE_gl_bindings",
  "touchHLE_openal_soft_wrapper",
@@ -662,6 +663,13 @@ dependencies = [
  "zip",
 ]
 
+[[package]]
+name = "touchHLE_dr_mp3_wrapper"
+version = "0.1.1"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "touchHLE_dynarmic_wrapper"
 version = "0.1.1"

Cargo.toml

@@ -37,6 +37,7 @@ rusttype = "0.9.3"
 # _CHHapticDynamicParameterIDHapticIntensityControl etc)
 sdl2 = { version = "=0.35.1", features = ["bundled", "static-link"] }
 sdl2-sys = "=0.35.1"
+touchHLE_dr_mp3_wrapper = { path = "src/audio/dr_mp3_wrapper" }
 touchHLE_dynarmic_wrapper = { path = "src/cpu/dynarmic_wrapper" }
 touchHLE_gl_bindings = { path = "src/window/gl_bindings" }
 touchHLE_openal_soft_wrapper = { path = "src/audio/openal_soft_wrapper" }

README.md

@@ -85,7 +85,7 @@ Please note that different licensing terms apply to the bundled dynamic librarie
 We stand on the shoulders of giants. Thank you to:
 
 * Everyone who has contributed to the project or supported it financially.
-* The authors of and contributors to the many libraries used by this project: [dynarmic](https://github.com/merryhime/dynarmic), [rust-macho](https://github.com/flier/rust-macho), [SDL](https://libsdl.org/), [rust-sdl2](https://github.com/Rust-SDL2/rust-sdl2), [stb\_image](https://github.com/nothings/stb), [openal-soft](https://github.com/kcat/openal-soft), [hound](https://github.com/ruuda/hound), [caf](https://github.com/rustaudio/caf), [RustType](https://gitlab.redox-os.org/redox-os/rusttype), [the Liberation fonts](https://github.com/liberationfonts/liberation-fonts), [the Noto CJK fonts](https://github.com/googlefonts/noto-cjk), [rust-plist](https://github.com/ebarnard/rust-plist), [gl-rs](https://github.com/brendanzab/gl-rs), [cargo-license](https://github.com/onur/cargo-license), [cc-rs](https://github.com/rust-lang/cc-rs), [cmake-rs](https://github.com/rust-lang/cmake-rs), and the Rust standard library.
+* The authors of and contributors to the many libraries used by this project: [dynarmic](https://github.com/merryhime/dynarmic), [rust-macho](https://github.com/flier/rust-macho), [SDL](https://libsdl.org/), [rust-sdl2](https://github.com/Rust-SDL2/rust-sdl2), [stb\_image](https://github.com/nothings/stb), [openal-soft](https://github.com/kcat/openal-soft), [hound](https://github.com/ruuda/hound), [caf](https://github.com/rustaudio/caf), [dr\_mp3](https://github.com/mackron/dr_libs), [RustType](https://gitlab.redox-os.org/redox-os/rusttype), [the Liberation fonts](https://github.com/liberationfonts/liberation-fonts), [the Noto CJK fonts](https://github.com/googlefonts/noto-cjk), [rust-plist](https://github.com/ebarnard/rust-plist), [gl-rs](https://github.com/brendanzab/gl-rs), [cargo-license](https://github.com/onur/cargo-license), [cc-rs](https://github.com/rust-lang/cc-rs), [cmake-rs](https://github.com/rust-lang/cmake-rs), and the Rust standard library.
 * The [Rust project](https://www.rust-lang.org/) generally.
 * The various people out there who've documented the iPhone OS platform, officially or otherwise. Much of this documentation is linked to within this codebase!
 * The iOS hacking/jailbreaking community.

src/audio.rs

@@ -6,8 +6,8 @@
 //! Audio file decoding and OpenAL bindings.
 //!
 //! The audio file decoding support is an abstraction over various libraries
-//! (currently [caf] and [hound]), usage of which should be confined to this
-//! module.
+//! (currently [caf], [hound], and dr_mp3), usage of which should be confined to
+//! this module.
 //!
 //! Resources:
 //! - [Apple Core Audio Format Specification 1.0](https://developer.apple.com/library/archive/documentation/MusicAudio/Reference/CAFSpec/CAF_intro/CAF_intro.html)
@@ -16,6 +16,7 @@ mod ima4;
 
 pub use ima4::decode_ima4;
 pub use touchHLE_openal_soft_wrapper as openal;
+use touchHLE_dr_mp3_wrapper as dr_mp3;
 
 use crate::fs::{Fs, GuestPath};
 use std::io::Cursor;
@@ -46,6 +47,7 @@ pub struct AudioFile(AudioFileInner);
 enum AudioFileInner {
     Wave(hound::WavReader<Cursor<Vec<u8>>>),
     Caf(caf::CafPacketReader<Cursor<Vec<u8>>>),
+    Mp3(dr_mp3::Mp3DecodedToPcm),
 }
 
 impl AudioFile {
@@ -66,6 +68,12 @@ impl AudioFile {
         } else if caf::CafPacketReader::new(Cursor::new(&bytes), vec![]).is_ok() {
             let reader = caf::CafPacketReader::new(Cursor::new(bytes), vec![]).unwrap();
             Ok(AudioFile(AudioFileInner::Caf(reader)))
+        // TODO: Real MP3 container handling. Currently we are immediately
+        // decoding the entire file to PCM and acting as if it's a PCM file,
+        // simply because because this is easier. Full MP3 support would require
+        // a lot of changes in Audio Toolbox.
+        } else if let Ok(pcm) = dr_mp3::decode_mp3_to_pcm(&bytes) {
+            Ok(AudioFile(AudioFileInner::Mp3(pcm)))
         } else {
             // We may eventually want to return an error here, this is just more
             // useful currently.
@@ -142,6 +150,18 @@ impl AudioFile {
                     bits_per_channel,
                 }
             }
+            AudioFileInner::Mp3(dr_mp3::Mp3DecodedToPcm{
+                sample_rate,
+                channels,
+                ..
+            }) => AudioDescription {
+                sample_rate: f64::from(sample_rate),
+                format: AudioFormat::LinearPcm { is_float: false, is_little_endian: true },
+                bytes_per_packet: channels * 2,
+                frames_per_packet: 1,
+                channels_per_frame: channels,
+                bits_per_channel: 16,
+            }
         }
     }
 
@@ -169,12 +189,15 @@ impl AudioFile {
                 // variable size not implemented
                 u64::from(self.packet_size_fixed()) * self.packet_count()
             }
+            AudioFileInner::Mp3(dr_mp3::Mp3DecodedToPcm { ref bytes, .. }) => {
+                bytes.len() as u64
+            }
         }
     }
 
     pub fn packet_count(&self) -> u64 {
         match self.0 {
-            AudioFileInner::Wave(_) => {
+            AudioFileInner::Wave(_) | AudioFileInner::Mp3(dr_mp3::Mp3DecodedToPcm { .. }) => {
                 // never variable-size
                 self.byte_count() / u64::from(self.packet_size_fixed())
             }
@@ -252,6 +275,13 @@ impl AudioFile {
                 }
                 Ok(byte_offset)
             }
+            AudioFileInner::Mp3(dr_mp3::Mp3DecodedToPcm { ref bytes, .. }) => {
+                let bytes = bytes.get(offset as usize..).ok_or(())?;
+                let bytes_to_read = buffer.len().min(bytes.len());
+                let bytes = &bytes[..bytes_to_read];
+                buffer[..bytes_to_read].copy_from_slice(bytes);
+                Ok(bytes_to_read)
+            }
         }
     }
 }

src/audio/dr_mp3_wrapper/Cargo.toml

@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+[package]
+name = "touchHLE_dr_mp3_wrapper"
+version = { workspace = true }
+edition = { workspace = true }
+license = { workspace = true }
+authors = { workspace = true }
+homepage = { workspace = true }
+
+[lib]
+path = "lib.rs"
+
+[build-dependencies]
+cc = { workspace = true }

src/audio/dr_mp3_wrapper/build.rs

@@ -0,0 +1,21 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+use std::path::Path;
+
+fn rerun_if_changed(path: &Path) {
+    println!("cargo:rerun-if-changed={}", path.to_str().unwrap());
+}
+
+fn main() {
+    let package_root = Path::new(env!("CARGO_MANIFEST_DIR"));
+    let workspace_root = package_root.join("../../..");
+
+    cc::Build::new()
+        .file(package_root.join("lib.c"))
+        .compile("dr_mp3_wrapper");
+    rerun_if_changed(&package_root.join("lib.c"));
+    rerun_if_changed(&workspace_root.join("vendor/dr_libs/dr_mp3.h"));
+}

src/audio/dr_mp3_wrapper/lib.c

@@ -0,0 +1,30 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+#define DR_MP3_IMPLEMENTATION
+#define DR_MP3_NO_STDIO
+#include "../../../vendor/dr_libs/dr_mp3.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+
+int16_t *touchHLE_decode_mp3_to_pcm(const uint8_t *data, size_t data_size,
+                                    uint32_t *channels,
+                                    uint32_t *sample_rate,
+                                    uint64_t *frame_count) {
+  drmp3_config config;
+  int16_t *samples = drmp3_open_memory_and_read_pcm_frames_s16(
+      data, data_size, &config, frame_count,
+      /* pAllocationCallbacks: */ NULL);
+  if (samples) {
+    *channels = config.channels;
+    *sample_rate = config.sampleRate;
+  }
+  return samples;
+}
+
+void touchHLE_free_decoded_mp3_pcm(int16_t *samples) {
+  drmp3_free(samples, /* pAllocationCallbacks: */ NULL);
+}

src/audio/dr_mp3_wrapper/lib.rs

@@ -0,0 +1,47 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+//! This is separated out into its own package so that we can avoid rebuilding
+//! dr_mp3 more often than necessary, and to improve build-time parallelism.
+
+// Allow the crate to have a non-snake-case name (touchHLE).
+// This also allows items in the crate to have non-snake-case names.
+#![allow(non_snake_case)]
+
+// See build.rs and lib.c
+extern "C" {
+    fn touchHLE_decode_mp3_to_pcm(data: *const u8, data_size: usize, channels: *mut u32, sample_rate: *mut u32, frame_count: *mut u64) -> *mut i16;
+    fn touchHLE_free_decoded_mp3_pcm(samples: *mut i16);
+}
+
+/// PCM data decoded from an MP3 file.
+pub struct Mp3DecodedToPcm {
+    /// 16-bit little-endian PCM samples, grouped in frames (one sample per
+    /// channel in each frame).
+    pub bytes: Vec<u8>,
+    /// Sample rate in Hz.
+    pub sample_rate: u32,
+    /// Channel count.
+    pub channels: u32,
+}
+
+pub fn decode_mp3_to_pcm(data: &[u8]) -> Result<Mp3DecodedToPcm, ()> {
+    let mut channels = 0;
+    let mut sample_rate = 0;
+    let mut frame_count = 0;
+    let samples_ptr = unsafe { touchHLE_decode_mp3_to_pcm(data.as_ptr(), data.len(), &mut channels, &mut sample_rate, &mut frame_count) };
+    if samples_ptr.is_null() {
+        return Err(());
+    }
+
+    let bytes = unsafe { std::slice::from_raw_parts(samples_ptr as *const _, std::mem::size_of::<i16>() * (frame_count as usize)) }.to_vec();
+    unsafe { touchHLE_free_decoded_mp3_pcm(samples_ptr) };
+
+    Ok(Mp3DecodedToPcm {
+        bytes,
+        sample_rate,
+        channels,
+    })
+}

src/licenses.rs

@@ -116,6 +116,14 @@ which is available either as Public Domain or under the terms of the MIT
 license.
 ";
 
+const DR_MP3: &str = "
+touchHLE, and therefore this executable, incorporates the library dr_mp3,
+which is available either under The Unlicense (which is a public domain
+dedication) or under the terms of the MIT license. dr_mp3 is in turn derived
+from the library minimp3, which is licensed under the Creative Commons CC0 1.0
+Universal Public Domain Dedication.
+";
+
 pub fn divider() {
     println!("---");
 }
@@ -142,4 +150,6 @@ pub fn print() {
     println!("{}", OPENAL_SOFT);
     divider();
     println!("{}", STB_IMAGE);
+    divider();
+    println!("{}", DR_MP3);
 }