Add basic Microphone support on Windows.

Core/Config.h

@@ -142,6 +142,7 @@ public:
 	std::string sVulkanDevice;
 	std::string sD3D11Device;  // Windows only
 	std::string sCameraDevice;
+	std::string sMicDevice;
 
 	bool bSoftwareRendering;
 	bool bHardwareTransform; // only used in the GLES backend

Core/HLE/sceAudio.cpp

@@ -23,6 +23,7 @@
 #include "Core/HLE/FunctionWrappers.h"
 #include "Core/HLE/sceKernelThread.h"
 #include "Core/HLE/sceAudio.h"
+#include "Core/HLE/sceUsbMic.h"
 #include "Core/HLE/__sceAudio.h"
 #include "Core/Reporting.h"
 
@@ -471,6 +472,11 @@ static u32 sceAudioSRCOutputBlocking(u32 vol, u32 buf) {
 	return hleLogSuccessI(SCEAUDIO, result);
 }
 
+static int sceAudioInputBlocking(u32 maxSamples, u32 sampleRate, u32 bufAddr) {
+	ERROR_LOG(HLE, "sceAudioInputBlocking: maxSamples: %d, samplerate: %d, bufAddr: %08x", maxSamples, sampleRate, bufAddr);
+	return __MicInputBlocking(maxSamples, sampleRate, bufAddr);
+}
+
 static u32 sceAudioRoutingSetMode(u32 mode) {
 	ERROR_LOG_REPORT(SCEAUDIO, "sceAudioRoutingSetMode(%08x)", mode);
 	int previousMode = defaultRoutingMode;
@@ -535,7 +541,7 @@ const HLEFunction sceAudio[] =
 	{0X7DE61688, nullptr,                                   "sceAudioInputInit",             '?', ""    },
 	{0XE926D3FB, nullptr,                                   "sceAudioInputInitEx",           '?', ""    },
 	{0X6D4BEC68, nullptr,                                   "sceAudioInput",                 '?', ""    },
-	{0X086E5895, nullptr,                                   "sceAudioInputBlocking",         '?', ""    },
+	{0X086E5895, &WrapI_UUU<sceAudioInputBlocking>,         "sceAudioInputBlocking",         'i', "xxx"    },
 	{0XA708C6A6, nullptr,                                   "sceAudioGetInputLength",        '?', ""    },
 	{0XA633048E, nullptr,                                   "sceAudioPollInputEnd",          '?', ""    },
 	{0X87B2E651, nullptr,                                   "sceAudioWaitInputEnd",          '?', ""    },

Core/HLE/sceKernel.cpp

@@ -77,12 +77,13 @@
 #include "sceImpose.h"
 #include "sceUsb.h"
 #include "sceUsbGps.h"
+#include "sceUsbCam.h"
+#include "sceUsbMic.h"
 #include "scePspNpDrm_user.h"
 #include "sceVaudio.h"
 #include "sceHeap.h"
 #include "sceDmac.h"
 #include "sceMp4.h"
-#include "sceUsbCam.h"
 
 #include "../Util/PPGeDraw.h"
 
@@ -146,6 +147,7 @@ void __KernelInit()
 	__VideoPmpInit();
 	__UsbGpsInit();
 	__UsbCamInit();
+	__UsbMicInit();
 	
 	SaveState::Init();  // Must be after IO, as it may create a directory
 	Reporting::Init();
@@ -170,6 +172,7 @@ void __KernelShutdown()
 	kernelObjects.Clear();
 
 	__UsbCamShutdown();
+	__UsbMicShutdown();
 	__UsbGpsShutdown();
 
 	__AudioCodecShutdown();

Core/HLE/sceKernelThread.cpp

@@ -79,6 +79,7 @@ const WaitTypeNames waitTypeNames[] = {
 	{ WAITTYPE_TLSPL,           "TLS" },
 	{ WAITTYPE_VMEM,            "Volatile Mem" },
 	{ WAITTYPE_ASYNCIO,         "AsyncIO" },
+	{ WAITTYPE_MICINPUT,        "Microphone input"},
 };
 
 const char *getWaitTypeName(WaitType type)

Core/HLE/sceKernelThread.h

@@ -106,6 +106,7 @@ enum WaitType : int
 	WAITTYPE_TLSPL        = 21,
 	WAITTYPE_VMEM         = 22,
 	WAITTYPE_ASYNCIO      = 23,
+	WAITTYPE_MICINPUT     = 24, // fake
 
 	NUM_WAITTYPES
 };

Core/HLE/sceUsb.cpp

@@ -83,7 +83,7 @@ static int sceUsbGetState() {
 			| (usbConnected ? USB_STATUS_CONNECTED : USB_STATUS_DISCONNECTED)
 			| (usbActivated ? USB_STATUS_ACTIVATED : USB_STATUS_DEACTIVATED);
 	}
-	INFO_LOG(HLE, "sceUsbGetState: 0x%x", state);
+	DEBUG_LOG(HLE, "sceUsbGetState: 0x%x", state);
 	return state;
 }
 

Core/HLE/sceUsbMic.cpp

@@ -21,29 +21,328 @@
 #include "Common/ChunkFile.h"
 #include "Core/HLE/HLE.h"
 #include "Core/HLE/FunctionWrappers.h"
+#include "Core/HLE/sceKernelThread.h"
 #include "Core/HLE/sceUsbMic.h"
+#include "Core/CoreTiming.h"
 #include "Core/MemMapHelpers.h"
+#include "Windows/CaptureDevice.h"
 
-static int sceUsbMicInputBlocking(u32 size, u32 samplerate, u32 bufAddr) {
-	INFO_LOG(HLE, "UNIMPL sceUsbMicInputBlocking: size: %d, samplerate: %d", size, samplerate);
-	for (unsigned int i = 0; i < size; i++) {
-		if (Memory::IsValidAddress(bufAddr + i)) {
-			Memory::Write_U8(i & 0xFF, bufAddr + i);
+#if defined(_WIN32) && !PPSSPP_PLATFORM(UWP) && !defined(__LIBRETRO__)
+#define HAVE_WIN32_MICROPHONE
+#endif
+
+enum {
+	SCE_USBMIC_ERROR_INVALID_MAX_SAMPLES = 0x80243806,
+	SCE_USBMIC_ERROR_INVALID_SAMPLERATE  = 0x8024380A,
+};
+
+int eventUsbMicAudioUpdate = -1;
+
+QueueBuf *audioBuf = nullptr;
+u32 audioBufSize = 0;
+u32 numNeedSamples = 0;
+static std::vector<MicWaitInfo> waitingThreads;
+std::mutex wtMutex;
+bool isNeedInput = false;
+
+static void __UsbMicAudioUpdate(u64 userdata, int cyclesLate) {
+	SceUID threadID = (SceUID)userdata;
+	u32 error;
+	int count = 0;
+	std::unique_lock<std::mutex> lock(wtMutex);
+	for (auto waitingThread : waitingThreads) {
+		if (waitingThread.threadID == threadID) {
+			SceUID waitID = __KernelGetWaitID(threadID, WAITTYPE_MICINPUT, error);
+			if (waitID == 0)
+				continue;
+			if (Microphone::isHaveDevice()) {
+				if (Microphone::availableAudioBufSize() >= waitingThread.needSize) {
+					u8 *tempbuf8 = new u8[waitingThread.needSize];
+					Microphone::getAudioData(tempbuf8, waitingThread.needSize);
+					u16 *tempbuf16 = (u16*)tempbuf8;
+					for (u32 i = 0; i < waitingThread.needSize / 2; i++) {
+						if (Memory::IsValidAddress(waitingThread.addr + i * 2)) {
+							Memory::Write_U16(tempbuf16[i] & 0xFFFF, waitingThread.addr + i * 2);
+						}
+					}
+					delete[] tempbuf8;
+					u32 ret = __KernelGetWaitValue(threadID, error);
+					DEBUG_LOG(HLE, "sceUsbMic: Waking up thread(%d)", (int)waitingThread.threadID);
+					__KernelResumeThreadFromWait(threadID, ret);
+					waitingThreads.erase(waitingThreads.begin() + count);
+					if (waitingThreads.size() == 0)
+						isNeedInput = false;
+				} else {
+					u64 waitTimeus = (waitingThread.needSize - Microphone::availableAudioBufSize()) * 1000000 / 2 / waitingThread.sampleRate;
+					CoreTiming::ScheduleEvent(usToCycles(waitTimeus), eventUsbMicAudioUpdate, userdata);
+				}
+			} else {
+				for (u32 i = 0; i < waitingThread.needSize; i++) {
+					if (Memory::IsValidAddress(waitingThread.addr + i)) {
+						Memory::Write_U8(i & 0xFF, waitingThread.addr + i);
+					}
+				}
+				u32 ret = __KernelGetWaitValue(threadID, error);
+				DEBUG_LOG(HLE, "sceUsbMic: Waking up thread(%d)", (int)waitingThread.threadID);
+				__KernelResumeThreadFromWait(threadID, ret);
+				waitingThreads.erase(waitingThreads.begin() + count);
+				if (waitingThreads.size() == 0)
+					isNeedInput = false;
+			}
 		}
+		++count;
 	}
-	hleEatMicro(1000000 / samplerate * (size / 2));
+	lock.unlock();
+}
+
+void __UsbMicInit() {
+	eventUsbMicAudioUpdate = CoreTiming::RegisterEvent("UsbMicAudioUpdate", &__UsbMicAudioUpdate);
+}
+
+void __UsbMicShutdown() {
+	if (audioBuf) {
+		delete audioBuf;
+		audioBuf = nullptr;
+	}
+	Microphone::stopMic();
+}
+
+QueueBuf::QueueBuf(u32 size) : start(0), end(0), capacity(size) {
+	buf_ = new u8[size];
+}
+
+QueueBuf::~QueueBuf() {
+	delete[] buf_;
+}
+
+void QueueBuf::push(u8 *buf, u32 size) {
+	if (getRemainingSize() < size) {
+		resize((capacity + size - getRemainingSize()) * 3 / 2);
+	}
+	std::unique_lock<std::mutex> lock(mutex);
+	if (end + size <= capacity) {
+		memcpy(buf_ + end, buf, size);
+		end += size;
+	} else {
+		memcpy(buf_ + end, buf, capacity - end);
+		size -= capacity - end;
+		memcpy(buf_, buf + capacity - end, size);
+		end = size;
+	}
+	lock.unlock();
+}
+
+u32 QueueBuf::pop(u8 *buf, u32 size) {
+	u32 ret = 0;
+	if (getAvailableSize() < size)
+		size = getAvailableSize();
+	ret = size;
+
+	std::unique_lock<std::mutex> lock(mutex);
+	if (start + size <= capacity) {
+		memcpy(buf, buf_ + start, size);
+		start += size;
+	} else {
+		memcpy(buf, buf_ + start, capacity - start);
+		size -= capacity - start;
+		memcpy(buf + capacity - start, buf_, size);
+		start = size;
+	}
+	lock.unlock();
+	return ret;
+}
+
+void QueueBuf::resize(u32 newSize) {
+	u32 availableSize = getAvailableSize();
+	u8 *oldbuf = buf_;
+
+	std::unique_lock<std::mutex> lock(mutex);
+	buf_ = new u8[newSize];
+	if (end >= start) {
+		memcpy(buf_, oldbuf + start, availableSize);
+	} else {
+		memcpy(buf_, oldbuf + start, capacity - start);
+		memcpy(buf_ + capacity - start, oldbuf, availableSize - (capacity - start));
+	}
+	start = 0;
+	end = availableSize;
+	capacity = newSize;
+	delete[] oldbuf;
+	lock.unlock();
+}
+
+u32 QueueBuf::getAvailableSize() {
+	u32 availableSize = 0;
+	if (end >= start) {
+		availableSize = end - start;
+	} else {
+		availableSize = end + capacity - start;
+	}
+	return availableSize;
+}
+
+u32 QueueBuf::getRemainingSize() {
+	return capacity - getAvailableSize();
+}
+
+static int sceUsbMicPollInputEnd() {
+	ERROR_LOG(HLE, "UNIMPL sceUsbMicPollInputEnd");
+	return 0;
+}
+
+static int sceUsbMicInputBlocking(u32 maxSamples, u32 sampleRate, u32 bufAddr) {
+	INFO_LOG(HLE, "sceUsbMicInputBlocking: maxSamples: %d, samplerate: %d, bufAddr: %08x", maxSamples, sampleRate, bufAddr);
+	if (maxSamples <= 0 || (maxSamples & 0x3F) != 0) {
+		return SCE_USBMIC_ERROR_INVALID_MAX_SAMPLES;
+	}
+
+	if (sampleRate != 44100 && sampleRate != 22050 && sampleRate != 11025) {
+		return SCE_USBMIC_ERROR_INVALID_SAMPLERATE;
+	}
+	return __MicInputBlocking(maxSamples, sampleRate, bufAddr);
+}
+
+static int sceUsbMicInputInitEx(u32 paramAddr) {
+	ERROR_LOG(HLE, "UNIMPL sceUsbMicInputInitEx: %08x", paramAddr);
+	return 0;
+}
+
+static int sceUsbMicInput() {
+	ERROR_LOG(HLE, "UNIMPL sceUsbMicInput");
+	return 0;
+}
+static int sceUsbMicGetInputLength() {
+	int ret = Microphone::availableAudioBufSize() / 2;
+	ERROR_LOG(HLE, "UNTEST sceUsbMicGetInputLength(ret: %d)", ret);
+	return ret;
+}
+
+static int sceUsbMicInputInit(int unknown1, int inputVolume, int unknown2) {
+	ERROR_LOG(HLE, "UNIMPL sceUsbMicInputInit(unknown1: %d, inputVolume: %d, unknown2: %d)", unknown1, inputVolume, unknown2);
+	return 0;
+}
+
+static int sceUsbMicWaitInputEnd() {
+	ERROR_LOG(HLE, "UNIMPL sceUsbMicWaitInputEnd");
+	return 0;
+}
+
+int Microphone::startMic(void *param) {
+#ifdef HAVE_WIN32_MICROPHONE
+	if (winMic)
+		winMic->sendMessage({ CAPTUREDEVIDE_COMMAND::START, param });
+#endif
+
+	return 0;
+}
+
+int Microphone::stopMic() {
+#ifdef HAVE_WIN32_MICROPHONE
+	if (winMic)
+		winMic->sendMessage({ CAPTUREDEVIDE_COMMAND::STOP, nullptr });
+#endif
+	return 0;
+}
+
+bool Microphone::isHaveDevice() {
+#ifdef HAVE_WIN32_MICROPHONE
+	return winMic->getDeviceCounts() >= 1;
+#endif
+	return false;
+}
+
+bool Microphone::isMicStarted() {
+#ifdef HAVE_WIN32_MICROPHONE
+	if(winMic)
+		return winMic->isStarted();
+#endif
+	return false;
+}
+
+bool Microphone::isNeedInput() {
+	return ::isNeedInput;
+}
+
+u32 Microphone::numNeedSamples() {
+	return ::numNeedSamples;
+}
+
+u32 Microphone::availableAudioBufSize() {
+	return audioBuf->getAvailableSize();
+}
+
+int Microphone::addAudioData(u8 *buf, u32 size) {
+	if (audioBuf)
+		audioBuf->push(buf, size);
+	else
+		return 0;
+
 	return size;
 }
 
+u32 Microphone::getAudioData(u8 *buf, u32 size) {
+	if(audioBuf)
+		return audioBuf->pop(buf, size);
+
+	return 0;
+}
+
+std::vector<std::string> Microphone::getDeviceList() {
+#ifdef HAVE_WIN32_MICROPHONE
+	if (winMic) {
+		return winMic->getDeviceList();
+	}
+#endif
+	return std::vector<std::string>();
+}
+
+void Microphone::onMicDeviceChange() {
+	if (Microphone::isMicStarted()) {
+		Microphone::stopMic();
+		// Just use the last param.
+		Microphone::startMic(nullptr);
+	}
+}
+
+u32 __MicInputBlocking(u32 maxSamples, u32 sampleRate, u32 bufAddr) {
+	u32 size = maxSamples << 1;
+	numNeedSamples = maxSamples;
+	if (size != audioBufSize) {
+		if (audioBuf) {
+			delete audioBuf;
+		}
+		audioBuf = new QueueBuf(size);
+		audioBufSize = size;
+	}
+	if (!Microphone::isMicStarted()) {
+		std::vector<u32> *param = new std::vector<u32>({ sampleRate, 1 });
+		Microphone::startMic(param);
+	}
+	u64 waitTimeus = 0;
+	if (Microphone::availableAudioBufSize() < size) {
+		waitTimeus = (size - Microphone::availableAudioBufSize()) * 1000000 / 2 / sampleRate;
+		isNeedInput = true;
+	}
+	CoreTiming::ScheduleEvent(usToCycles(waitTimeus), eventUsbMicAudioUpdate, __KernelGetCurThread());
+	MicWaitInfo waitInfo = { __KernelGetCurThread(), bufAddr, size, sampleRate };
+	std::unique_lock<std::mutex> lock(wtMutex);
+	waitingThreads.push_back(waitInfo);
+	lock.unlock();
+	DEBUG_LOG(HLE, "MicInputBlocking: blocking thread(%d)", (int)__KernelGetCurThread());
+	__KernelWaitCurThread(WAITTYPE_MICINPUT, 1, size, 0, false, "blocking microphone");
+
+	return maxSamples;
+}
+
 const HLEFunction sceUsbMic[] =
 {
-	{0x06128E42, nullptr,                            "sceUsbMicPollInputEnd",         '?', "" },
+	{0x06128E42, &WrapI_V<sceUsbMicPollInputEnd>,    "sceUsbMicPollInputEnd",         'i', "" },
 	{0x2E6DCDCD, &WrapI_UUU<sceUsbMicInputBlocking>, "sceUsbMicInputBlocking",        'i', "xxx" },
-	{0x45310F07, nullptr,                            "sceUsbMicInputInitEx",          '?', "" },
-	{0x5F7F368D, nullptr,                            "sceUsbMicInput",                '?', "" },
-	{0x63400E20, nullptr,                            "sceUsbMicGetInputLength",       '?', "" },
-	{0xB8E536EB, nullptr,                            "sceUsbMicInputInit",            '?', "" },
-	{0xF899001C, nullptr,                            "sceUsbMicWaitInputEnd",         '?', "" },
+	{0x45310F07, &WrapI_U<sceUsbMicInputInitEx>,     "sceUsbMicInputInitEx",          'i', "x" },
+	{0x5F7F368D, &WrapI_V<sceUsbMicInput> ,          "sceUsbMicInput",                'i', "" },
+	{0x63400E20, &WrapI_V<sceUsbMicGetInputLength>,  "sceUsbMicGetInputLength",       'i', "" },
+	{0xB8E536EB, &WrapI_III<sceUsbMicInputInit>,     "sceUsbMicInputInit",            'i', "iii" },
+	{0xF899001C, &WrapI_V<sceUsbMicWaitInputEnd>,    "sceUsbMicWaitInputEnd",         'i', "" },
 };
 
 void Register_sceUsbMic()

Core/HLE/sceUsbMic.h

@@ -16,4 +16,55 @@
 // https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.
 
 #pragma once
+
+#include "sceKernel.h"
+#include <mutex>
+
 void Register_sceUsbMic();
+
+void __UsbMicInit();
+void __UsbMicShutdown();
+
+struct MicWaitInfo {
+	SceUID threadID;
+	u32 addr;
+	u32 needSize;
+	u32 sampleRate;
+};
+
+class QueueBuf {
+public:
+	QueueBuf(u32 size);
+	~QueueBuf();
+
+	void push(u8 *buf, u32 size);
+	u32 pop(u8 *buf, u32 size);
+	void resize(u32 newSize);
+	u32 getAvailableSize();
+	u32 getRemainingSize();
+
+private:
+	u32 start;
+	u32 end;
+	u32 capacity;
+	u8 *buf_;
+	std::mutex mutex;
+};
+
+namespace Microphone {
+	int startMic(void *param);
+	int stopMic();
+	bool isHaveDevice();
+	bool isMicStarted();
+	bool isNeedInput();
+	u32 numNeedSamples();
+	u32 availableAudioBufSize();
+
+	int addAudioData(u8 *buf, u32 size);
+	u32 getAudioData(u8 *buf, u32 size);
+
+	std::vector<std::string> getDeviceList();
+	void onMicDeviceChange();
+}
+
+u32 __MicInputBlocking(u32 maxSamples, u32 sampleRate, u32 bufAddr);

UI/GameSettingsScreen.cpp

@@ -60,6 +60,7 @@
 #include "Core/TextureReplacer.h"
 #include "Core/WebServer.h"
 #include "Core/HLE/sceUsbCam.h"
+#include "Core/HLE/sceUsbMic.h"
 #include "GPU/Common/PostShader.h"
 #include "android/jni/TestRunner.h"
 #include "GPU/GPUInterface.h"
@@ -547,6 +548,13 @@ void GameSettingsScreen::CreateViews() {
 	}
 #endif
 
+	std::vector<std::string> micList = Microphone::getDeviceList();
+	if (micList.size() >= 1) {
+		audioSettings->Add(new ItemHeader(gr->T("Microphone")));
+		PopupMultiChoiceDynamic *MicChoice = audioSettings->Add(new PopupMultiChoiceDynamic(&g_Config.sMicDevice, gr->T("Microphone Device"), micList, nullptr, screenManager()));
+		MicChoice->OnChoice.Handle(this, &GameSettingsScreen::OnMicDeviceChange);
+	}
+
 #if defined(SDL)
 	std::vector<std::string> audioDeviceList;
 	SplitString(System_GetProperty(SYSPROP_AUDIO_DEVICE_LIST), '\0', audioDeviceList);
@@ -1309,6 +1317,11 @@ UI::EventReturn GameSettingsScreen::OnCameraDeviceChange(UI::EventParams& e) {
 	return UI::EVENT_DONE;
 }
 
+UI::EventReturn GameSettingsScreen::OnMicDeviceChange(UI::EventParams& e) {
+	Microphone::onMicDeviceChange();
+	return UI::EVENT_DONE;
+}
+
 UI::EventReturn GameSettingsScreen::OnAudioDevice(UI::EventParams &e) {
 	auto a = GetI18NCategory("Audio");
 	if (g_Config.sAudioDevice == a->T("Auto")) {

UI/GameSettingsScreen.h

@@ -103,6 +103,7 @@ private:
 	UI::EventReturn OnRenderingDevice(UI::EventParams &e);
 	UI::EventReturn OnInflightFramesChoice(UI::EventParams &e);
 	UI::EventReturn OnCameraDeviceChange(UI::EventParams& e);
+	UI::EventReturn OnMicDeviceChange(UI::EventParams& e);
 	UI::EventReturn OnAudioDevice(UI::EventParams &e);
 	UI::EventReturn OnJitAffectingSetting(UI::EventParams &e);
 #if PPSSPP_PLATFORM(ANDROID)

UI/NativeApp.cpp

@@ -894,6 +894,8 @@ bool NativeInitGraphics(GraphicsContext *graphicsContext) {
 	if (IsWin7OrHigher()) {
 		winCamera = new WindowsCaptureDevice(CAPTUREDEVIDE_TYPE::VIDEO);
 		winCamera->sendMessage({ CAPTUREDEVIDE_COMMAND::INITIALIZE, nullptr });
+		winMic = new WindowsCaptureDevice(CAPTUREDEVIDE_TYPE::AUDIO);
+		winMic->sendMessage({ CAPTUREDEVIDE_COMMAND::INITIALIZE, nullptr });
 	}
 #endif
 
@@ -926,6 +928,12 @@ void NativeShutdownGraphics() {
 		delete winCamera;
 		winCamera = nullptr;
 	}
+	if (winMic) {
+		winMic->sendMessage({ CAPTUREDEVIDE_COMMAND::SHUTDOWN, nullptr });
+		while (!winMic->isShutDown()) {};// Wait for shutting down.
+		delete winMic;
+		winMic = nullptr;
+	}
 #endif
 
 	UIBackgroundShutdown();

Windows/CaptureDevice.cpp

@@ -24,8 +24,6 @@
 #include "Core/HLE/sceUsbCam.h"
 #include "Core/Config.h"
 
-bool isDeviceChanged = false;
-
 namespace MFAPI {
 	HINSTANCE Mflib;
 	HINSTANCE Mfplatlib;
@@ -87,6 +85,7 @@ bool unRegisterCMPTMFApis() {
 }
 
 WindowsCaptureDevice *winCamera;
+WindowsCaptureDevice *winMic;
 
 // TODO: Add more formats, but need some tests.
 VideoFormatTransform g_VideoFormats[] =
@@ -97,19 +96,28 @@ VideoFormatTransform g_VideoFormats[] =
 	{ MFVideoFormat_NV12,  AV_PIX_FMT_NV12    }
 };
 
-const int g_cVideoFormats = 4;
+AudioFormatTransform g_AudioFormats[] = {
+	{ MFAudioFormat_PCM,    8, AV_SAMPLE_FMT_U8 },
+	{ MFAudioFormat_PCM,   16, AV_SAMPLE_FMT_S16 },
+	{ MFAudioFormat_PCM,   32, AV_SAMPLE_FMT_S32 },
+	{ MFAudioFormat_Float, 32, AV_SAMPLE_FMT_FLT }
+};
+
+const int g_cVideoFormats = ARRAYSIZE(g_VideoFormats);
+const int g_cAudioFormats = ARRAYSIZE(g_AudioFormats);
 
 MediaParam defaultVideoParam = { 640, 480, 0, MFVideoFormat_RGB24 };
-MediaParam defaultAudioParam = { 44100, 2, 0, MFAudioFormat_PCM };
+MediaParam defaultAudioParam = { 44100, 2, 2, MFAudioFormat_PCM };
 
 HRESULT GetDefaultStride(IMFMediaType *pType, LONG *plStride);
 
-ReaderCallback::ReaderCallback(WindowsCaptureDevice *device): img_convert_ctx(nullptr){
+ReaderCallback::ReaderCallback(WindowsCaptureDevice *device): img_convert_ctx(nullptr), resample_ctx(nullptr){
 	this->device = device;
 }
 
 ReaderCallback::~ReaderCallback() {
 	sws_freeContext(img_convert_ctx);
+	swr_free(&resample_ctx);
 }
 
 HRESULT ReaderCallback::QueryInterface(REFIID riid, void** ppv)
@@ -130,7 +138,6 @@ HRESULT ReaderCallback::OnReadSample(
 		IMFSample *pSample) {
 	HRESULT hr = S_OK;
 	IMFMediaBuffer *pBuffer = nullptr;
-	LONG lStride = 0;
 	std::lock_guard<std::mutex> lock(device->sdMutex);
 	if (device->isShutDown())
 		return hr;
@@ -151,6 +158,7 @@ HRESULT ReaderCallback::OnReadSample(
 			int imgJpegSize = device->imgJpegSize;
 			unsigned char* invertedSrcImg = nullptr;
 			LONG srcPadding = 0;
+			LONG lStride = 0;
 
 			UINT32 srcW = device->deviceParam.width;
 			UINT32 srcH = device->deviceParam.height;
@@ -207,14 +215,53 @@ HRESULT ReaderCallback::OnReadSample(
 					nullptr
 				);
 			}
-
 			delete videoBuffer;
 			break;
 		}
-		case CAPTUREDEVIDE_TYPE::AUDIO:
-			// TODO:
+		case CAPTUREDEVIDE_TYPE::AUDIO: {
+			BYTE *sampleBuf = nullptr;
+			DWORD length = 0;
+			u32 sizeAfterResample = 0;
+			// pSample can be null, in this case ReadSample still should be called to request next frame.
+			if (pSample && Microphone::isNeedInput()) {
+				pBuffer->Lock(&sampleBuf, nullptr, &length);
+				if (!device->rawAudioBuf) {
+					device->rawAudioBuf = new QueueBuf(length * 2); // Alloc enough space.
+				}
+				device->rawAudioBuf->push(sampleBuf, length);
+				if (device->needResample()) {
+					sizeAfterResample = device->rawAudioBuf->getAvailableSize() * device->targetMediaParam.sampleRate / device->deviceParam.sampleRate / device->deviceParam.channels;
+					// Wait until have enough audio data.
+					if (sizeAfterResample + Microphone::availableAudioBufSize() >= Microphone::numNeedSamples() * 2) {
+						u32 rawAudioBufSize = device->rawAudioBuf->getAvailableSize();
+						u8 *tempbuf = new u8[rawAudioBufSize];
+						device->rawAudioBuf->pop(tempbuf, rawAudioBufSize);
+						sizeAfterResample = doResample(
+							&device->resampleBuf, device->targetMediaParam.sampleRate, device->targetMediaParam.channels, &device->resampleBufSize,
+							tempbuf, device->deviceParam.sampleRate, device->deviceParam.channels, device->deviceParam.audioFormat, rawAudioBufSize, device->deviceParam.bitsPerSample);
+						delete[] tempbuf;
+						if (device->resampleBuf)
+							Microphone::addAudioData(device->resampleBuf, sizeAfterResample);
+					}		
+				} else {
+					Microphone::addAudioData(sampleBuf, length);
+				}
+				pBuffer->Unlock();
+			}
+			// Request the next frame.
+			if (SUCCEEDED(hr)) {
+				hr = device->m_pReader->ReadSample(
+					(DWORD)MF_SOURCE_READER_FIRST_AUDIO_STREAM,
+					0,
+					nullptr,
+					nullptr,
+					nullptr,
+					nullptr
+				);
+			}
 			break;
 		}
+		}
 	}
 
 	SafeRelease(&pBuffer);
@@ -229,6 +276,14 @@ AVPixelFormat ReaderCallback::getAVVideoFormatbyMFVideoFormat(const GUID &MFVide
 	return AV_PIX_FMT_RGB24;
 }
 
+AVSampleFormat ReaderCallback::getAVAudioFormatbyMFAudioFormat(const GUID &MFAudioFormat, const u32 &bitsPerSample) {
+	for (int i = 0; i < g_cAudioFormats; i++) {
+		if (MFAudioFormat == g_AudioFormats[i].MFAudioFormat && bitsPerSample == g_AudioFormats[i].bitsPerSample)
+			return g_AudioFormats[i].AVAudioFormat;
+	}
+	return AV_SAMPLE_FMT_S16;
+}
+
 void ReaderCallback::imgConvert(
 	unsigned char *dst, unsigned int &dstW, unsigned int &dstH, int dstLineSizes[4],
 	unsigned char *src, const unsigned int &srcW, const unsigned int &srcH, const GUID &srcFormat, 
@@ -237,9 +292,9 @@ void ReaderCallback::imgConvert(
 	unsigned char *pSrc[4];
 	unsigned char *pDst[4];
 
-	AVPixelFormat srcAvFormat = getAVVideoFormatbyMFVideoFormat(srcFormat);
+	AVPixelFormat srcAVFormat = getAVVideoFormatbyMFVideoFormat(srcFormat);
 
-	av_image_fill_linesizes(srcLineSizes, srcAvFormat, srcW);
+	av_image_fill_linesizes(srcLineSizes, srcAVFormat, srcW);
 
 	// Is this correct?
 	if (srcPadding != 0) {
@@ -249,16 +304,14 @@ void ReaderCallback::imgConvert(
 		}
 	}
 
-	av_image_fill_pointers(pSrc, srcAvFormat, srcH, src, srcLineSizes);
+	av_image_fill_pointers(pSrc, srcAVFormat, srcH, src, srcLineSizes);
 	av_image_fill_pointers(pDst, AV_PIX_FMT_RGB24, dstH, dst, dstLineSizes);
 
-
-
 	if (img_convert_ctx == nullptr) {
 		img_convert_ctx = sws_getContext(
 			srcW,
 			srcH,
-			srcAvFormat,
+			srcAVFormat,
 			dstW,
 			dstH,
 			AV_PIX_FMT_RGB24,
@@ -348,28 +401,68 @@ void ReaderCallback::imgInvertNV12(unsigned char *dst, int &dstStride, unsigned
 	}
 }
 
+u32 ReaderCallback::doResample(u8 **dst, u32 &dstSampleRate, u32 &dstChannels, u32 *dstSize, u8 *src, const u32 &srcSampleRate, const u32 &srcChannels, const GUID &srcFormat, const u32& srcSize, const u32& srcBitsPerSample) {
+	AVSampleFormat srcAVFormat = getAVAudioFormatbyMFAudioFormat(srcFormat, srcBitsPerSample);
+	int outSamplesCount = 0;
+	if (resample_ctx == nullptr) {
+		resample_ctx = swr_alloc_set_opts(nullptr,
+			av_get_default_channel_layout(dstChannels),
+			AV_SAMPLE_FMT_S16,
+			dstSampleRate,
+			av_get_default_channel_layout(srcChannels),
+			srcAVFormat,
+			srcSampleRate,
+			0,
+			nullptr);
+		if (resample_ctx == nullptr || swr_init(resample_ctx) < 0) {
+			swr_free(&resample_ctx);
+			return 0;
+		}
+	}
+	int srcSamplesCount = srcSize / srcChannels / av_get_bytes_per_sample(srcAVFormat); // per channel.
+	int outCount = srcSamplesCount * dstSampleRate / srcSampleRate + 256;
+	unsigned int outSize = av_samples_get_buffer_size(nullptr, dstChannels, outCount, AV_SAMPLE_FMT_S16, 0);
+	
+	if (!*dst) {
+		*dst = (u8 *)av_malloc(outSize);
+		*dstSize = outSize;
+	}
+	if (!*dst)
+		return 0;
+
+	if(*dstSize < outSize)
+		av_fast_malloc(dst, dstSize, outSize);
+
+	outSamplesCount = swr_convert(resample_ctx, dst, outCount, (const uint8_t **)&src, srcSamplesCount);
+	if (outSamplesCount < 0)
+		return 0;
+	return av_samples_get_buffer_size(nullptr, dstChannels, outSamplesCount, AV_SAMPLE_FMT_S16, 0);
+}
+
 WindowsCaptureDevice::WindowsCaptureDevice(CAPTUREDEVIDE_TYPE type) :
 	type(type),
 	m_pCallback(nullptr),
 	m_pSource(nullptr),
 	m_pReader(nullptr),
+	imageRGB(nullptr),
+	imageJpeg(nullptr),
+	imgJpegSize(0),
+	resampleBuf(nullptr),
+	resampleBufSize(0),
+	rawAudioBuf(nullptr),
 	error(CAPTUREDEVIDE_ERROR_NO_ERROR),
 	errorMessage(""),
+	isDeviceChanged(false),
 	state(CAPTUREDEVIDE_STATE::UNINITIALIZED) {
 	param = { 0 };
+	deviceParam = { 0 };
 
 	switch (type) {
 	case CAPTUREDEVIDE_TYPE::VIDEO:
 		targetMediaParam = defaultVideoParam;
-		imageRGB = (unsigned char*)av_malloc(av_image_get_buffer_size(AV_PIX_FMT_RGB24, targetMediaParam.width, targetMediaParam.height, 1));
-		imgJpegSize = av_image_get_buffer_size(AV_PIX_FMT_YUVJ411P, targetMediaParam.width, targetMediaParam.height, 1);
-		imageJpeg = (unsigned char*)av_malloc(imgJpegSize);
 		break;
 	case CAPTUREDEVIDE_TYPE::AUDIO:
-		// TODO:
 		targetMediaParam = defaultAudioParam;
-		imageRGB = nullptr;
-		imageJpeg = nullptr;
 		break;
 	}
 
@@ -380,11 +473,12 @@ WindowsCaptureDevice::WindowsCaptureDevice(CAPTUREDEVIDE_TYPE type) :
 WindowsCaptureDevice::~WindowsCaptureDevice() {
 	switch (type) {
 	case CAPTUREDEVIDE_TYPE::VIDEO:
-		av_free(imageRGB);
-		av_free(imageJpeg);
+		av_freep(&imageRGB);
+		av_freep(&imageJpeg);
 		break;
 	case CAPTUREDEVIDE_TYPE::AUDIO:
-		// TODO:
+		av_freep(&resampleBuf);
+		delete rawAudioBuf;
 		break;
 	}
 }
@@ -412,7 +506,7 @@ bool WindowsCaptureDevice::init() {
 	return true;
 }
 
-bool WindowsCaptureDevice::start(UINT32 width, UINT32 height) {
+bool WindowsCaptureDevice::start(void *startParam) {
 	HRESULT hr = S_OK;
 	IMFAttributes *pAttributes = nullptr;
 	IMFMediaType *pType = nullptr;
@@ -434,16 +528,14 @@ bool WindowsCaptureDevice::start(UINT32 width, UINT32 height) {
 		return false;
 	}
 
-	targetMediaParam.width = width;
-	targetMediaParam.height = height;
-	av_image_fill_linesizes(imgRGBLineSizes, AV_PIX_FMT_RGB24, targetMediaParam.width);
-
 	m_pCallback = new ReaderCallback(this);
 
+	std::string selectedDeviceName = type == CAPTUREDEVIDE_TYPE::VIDEO ? g_Config.sCameraDevice : g_Config.sMicDevice;
+
 	switch (state) {
 	case CAPTUREDEVIDE_STATE::STOPPED:
 		for (auto &name : deviceList) {
-			if (name == g_Config.sCameraDevice) {
+			if (name == selectedDeviceName) {
 				selection = count;
 				break;
 			}
@@ -477,7 +569,21 @@ bool WindowsCaptureDevice::start(UINT32 width, UINT32 height) {
 
 		if (SUCCEEDED(hr)) {
 			switch (type) {
-			case CAPTUREDEVIDE_TYPE::VIDEO:
+			case CAPTUREDEVIDE_TYPE::VIDEO: {
+				if (startParam) {
+					std::vector<int> *resolution = static_cast<std::vector<int>*>(startParam);
+					targetMediaParam.width = resolution->at(0);
+					targetMediaParam.height = resolution->at(1);
+					delete resolution;
+				}
+
+				av_freep(&imageRGB);
+				av_freep(&imageJpeg);
+				imageRGB = (unsigned char*)av_malloc(av_image_get_buffer_size(AV_PIX_FMT_RGB24, targetMediaParam.width, targetMediaParam.height, 1));
+				imgJpegSize = av_image_get_buffer_size(AV_PIX_FMT_YUVJ411P, targetMediaParam.width, targetMediaParam.height, 1);
+				imageJpeg = (unsigned char*)av_malloc(imgJpegSize);
+				av_image_fill_linesizes(imgRGBLineSizes, AV_PIX_FMT_RGB24, targetMediaParam.width);
+
 				for (DWORD i = 0; ; i++) {
 					hr = m_pReader->GetNativeMediaType(
 						(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,
@@ -512,13 +618,45 @@ bool WindowsCaptureDevice::start(UINT32 width, UINT32 height) {
 						nullptr
 					);
 				}
-
 				break;
+			}
 
-			case CAPTUREDEVIDE_TYPE::AUDIO:
-				// TODO:
+			case CAPTUREDEVIDE_TYPE::AUDIO: {
+				if (startParam) {
+					std::vector<u32> *micParam = static_cast<std::vector<u32>*>(startParam);
+					targetMediaParam.sampleRate = micParam->at(0);
+					targetMediaParam.channels = micParam->at(1);
+					delete micParam;
+				}
+
+				for (DWORD i = 0; ; i++) {
+					hr = m_pReader->GetNativeMediaType(
+						(DWORD)MF_SOURCE_READER_FIRST_AUDIO_STREAM,
+						i,
+						&pType
+					);
+
+					if (FAILED(hr)) { break; }
+
+					hr = setDeviceParam(pType);
+
+					if (SUCCEEDED(hr))
+						break;
+				}
+
+				if (SUCCEEDED(hr)) {
+					hr = m_pReader->ReadSample(
+						(DWORD)MF_SOURCE_READER_FIRST_AUDIO_STREAM,
+						0,
+						nullptr,
+						nullptr,
+						nullptr,
+						nullptr
+					);
+				}
 				break;
 			}
+			}
 		}
 
 		if (FAILED(hr)) {
@@ -670,7 +808,46 @@ HRESULT WindowsCaptureDevice::setDeviceParam(IMFMediaType *pType) {
 
 		break;
 	case CAPTUREDEVIDE_TYPE::AUDIO:
-		// TODO:
+		hr = pType->GetGUID(MF_MT_SUBTYPE, &subtype);
+		if (FAILED(hr))
+			break;
+
+		for (int i = 0; i < g_cVideoFormats; i++) {
+			if (subtype == g_AudioFormats[i].MFAudioFormat) {
+				deviceParam.audioFormat = subtype;
+				getFormat = true;
+				break;
+			}
+		}
+
+		if (!getFormat) {
+			for (int i = 0; i < g_cAudioFormats; i++) {
+				hr = pType->SetGUID(MF_MT_SUBTYPE, g_AudioFormats[i].MFAudioFormat);
+				if (FAILED(hr))
+					continue;
+
+				hr = m_pReader->SetCurrentMediaType(
+					(DWORD)MF_SOURCE_READER_FIRST_AUDIO_STREAM,
+					NULL,
+					pType
+				);
+
+				if (SUCCEEDED(hr)) {
+					deviceParam.audioFormat = g_AudioFormats[i].MFAudioFormat;
+					getFormat = true;
+					break;
+				}
+			}
+		}
+		if (SUCCEEDED(hr))
+			hr = pType->GetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, &deviceParam.sampleRate);
+
+		if (SUCCEEDED(hr))
+			hr = pType->GetUINT32(MF_MT_AUDIO_NUM_CHANNELS, &deviceParam.channels);
+
+		if (SUCCEEDED(hr))
+			hr = pType->GetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, (UINT32 *)&deviceParam.bitsPerSample);
+
 		break;
 	}
 
@@ -701,10 +878,11 @@ void WindowsCaptureDevice::messageHandler() {
 	CoInitializeEx(NULL, COINIT_MULTITHREADED);
 	MFStartup(MF_VERSION);
 	CAPTUREDEVIDE_MESSAGE message;
-	std::vector<int>* resolution;
 
 	if (type == CAPTUREDEVIDE_TYPE::VIDEO) {
 		setCurrentThreadName("Camera");
+	} else if (type == CAPTUREDEVIDE_TYPE::AUDIO) {
+		setCurrentThreadName("Microphone");
 	}
 
 	while ((message = getMessage()).command != CAPTUREDEVIDE_COMMAND::SHUTDOWN) {
@@ -713,8 +891,7 @@ void WindowsCaptureDevice::messageHandler() {
 			init();
 			break;
 		case CAPTUREDEVIDE_COMMAND::START:
-			resolution = static_cast<std::vector<int>*>(message.opacity);
-			start(resolution->at(0), resolution->at(1));
+			start(message.opacity);
 			break;
 		case CAPTUREDEVIDE_COMMAND::STOP:
 			stop();
@@ -781,6 +958,13 @@ HRESULT WindowsCaptureDevice::enumDevices() {
 	return hr;
 }
 
+bool WindowsCaptureDevice::needResample() {
+	return deviceParam.sampleRate    != targetMediaParam.sampleRate     || 
+		   deviceParam.channels      != targetMediaParam.channels       || 
+		   deviceParam.audioFormat   != targetMediaParam.audioFormat    ||
+		   deviceParam.bitsPerSample != targetMediaParam.bitsPerSample;
+}
+
 //-----------------------------------------------------------------------------
 // GetDefaultStride
 //

Windows/CaptureDevice.h

@@ -26,10 +26,13 @@
 #include <queue>
 #include <thread>
 
+#include "Core/HLE/sceUsbMic.h"
+
 #ifdef __cplusplus
 extern "C" {
 #include "libavformat/avformat.h"
 #include "libswscale/swscale.h"
+#include "libswresample/swresample.h"
 #include "libavutil/imgutils.h"
 }
 #endif // __cplusplus
@@ -39,6 +42,12 @@ struct VideoFormatTransform {
 	AVPixelFormat AVVideoFormat;
 };
 
+struct AudioFormatTransform {
+	GUID MFAudioFormat;
+	u32 bitsPerSample;
+	AVSampleFormat AVAudioFormat;
+};
+
 enum class CAPTUREDEVIDE_TYPE {
 	VIDEO,
 	AUDIO
@@ -90,8 +99,8 @@ union MediaParam {
 	struct  {
 		UINT32 sampleRate;
 		UINT32 channels;
-		LONG   padding;
-		GUID   audioFomat;
+		LONG bitsPerSample;
+		GUID   audioFormat;
 	};
 };
 
@@ -127,9 +136,10 @@ public:
 	STDMETHODIMP OnFlush(DWORD) { return S_OK; }
 
 	AVPixelFormat getAVVideoFormatbyMFVideoFormat(const GUID &MFVideoFormat);
+	AVSampleFormat getAVAudioFormatbyMFAudioFormat(const GUID &MFAudioFormat, const u32 &bitsPerSample);
 
 	/*
-	 * Always convet the image to RGB24
+	 * Always convert the image to RGB24
 	 * @param dst/src pointer to destination/source image
 	 * @param dstW/srcW, dstH/srcH destination/source image's width and height in pixels
 	 * @param dstLineSizes get the linesize of each plane by av_image_fill_linesizes()
@@ -149,10 +159,20 @@ public:
 	void imgInvertYUY2(unsigned char *dst, int &dstStride, unsigned char *src, const int &srcStride, const int &h);
 	void imgInvertNV12(unsigned char *dst, int &dstStride, unsigned char *src, const int &srcStride, const int &h);
 
+	/*
+	 * Always resample to uncompressed signed 16bits
+	 * @param dst pointer to pointer of dst buffer could be a nullptr, would be overwritten by a new pointer if space too small or a nullptr, should be freed by av_free/av_freep by caller
+	 * @param dstSize pointer to size of the dst buffer, could be modified after this func
+	 * @param srcFormat MF_MT_SUBTYPE attribute of the source audio data
+	 * @param srcSize size of valid data in the source buffer, in bytes
+	 * @return size of output in bytes 
+	 */
+	u32 doResample(u8 **dst, u32 &dstSampleRate, u32 &dstChannels, u32 *dstSize, u8 *src, const u32 &srcSampleRate, const u32 &srcChannels, const GUID &srcFormat, const u32 &srcSize, const u32& srcBitsPerSamples);
 
 protected:
 	WindowsCaptureDevice *device;
 	SwsContext *img_convert_ctx;
+	SwrContext *resample_ctx;
 };
 
 class WindowsCaptureDevice {
@@ -160,10 +180,10 @@ public:
 	WindowsCaptureDevice(CAPTUREDEVIDE_TYPE type);
 	~WindowsCaptureDevice();
 
-	static void CheckDevices();
+	void CheckDevices();
 
 	bool init();
-	bool start(UINT32 width, UINT32 height);
+	bool start(void *startParam);
 	bool stop();
 
 	CAPTUREDEVIDE_ERROR getError() const { return error; }
@@ -178,12 +198,15 @@ public:
 	HRESULT setDeviceParam(IMFMediaType *pType);
 
 	bool isShutDown() const { return state == CAPTUREDEVIDE_STATE::SHUTDOWN; }
+	bool isStarted() const { return state == CAPTUREDEVIDE_STATE::STARTED; }
 
 	void sendMessage(CAPTUREDEVIDE_MESSAGE message);
 	CAPTUREDEVIDE_MESSAGE getMessage();
 
 	HRESULT enumDevices();
 
+	bool needResample();
+
 	friend class ReaderCallback;
 
 protected:
@@ -199,6 +222,8 @@ protected:
 	CAPTUREDEVIDE_ERROR error;
 	std::string errorMessage;
 
+	bool isDeviceChanged;
+
 // MF interface.
 	ReaderCallback *m_pCallback;
 	IMFSourceReader *m_pReader;
@@ -220,6 +245,12 @@ protected:
 	int imgRGBLineSizes[4];
 	unsigned char *imageJpeg;
 	int imgJpegSize;
+
+//Microphone only
+	u8 *resampleBuf;
+	u32 resampleBufSize;
+	QueueBuf *rawAudioBuf;
 };
 
 extern WindowsCaptureDevice *winCamera;
+extern WindowsCaptureDevice *winMic;

Windows/MainWindow.cpp

@@ -849,7 +849,10 @@ namespace MainWindow
 #ifndef _M_ARM
 			DinputDevice::CheckDevices();
 #endif
-			WindowsCaptureDevice::CheckDevices();
+			if (winCamera)
+				winCamera->CheckDevices();
+			if (winMic)
+				winMic->CheckDevices();
 			return DefWindowProc(hWnd, message, wParam, lParam);
 
 		case WM_VERYSLEEPY_MSG: