diff --git a/sources/engine/Stride.Audio.Tests/Stride.Audio.Tests.csproj b/sources/engine/Stride.Audio.Tests/Stride.Audio.Tests.csproj index f7d3f144f..29fb67cf0 100644 --- a/sources/engine/Stride.Audio.Tests/Stride.Audio.Tests.csproj +++ b/sources/engine/Stride.Audio.Tests/Stride.Audio.Tests.csproj @@ -8,6 +8,16 @@ true true + + + + STRIDE_VIDEO_FFMPEG;$(DefineConstants) + + + STRIDE_VIDEO_MEDIACODEC;$(DefineConstants) + + + STRIDE_VIDEO_AVFOUNDATION;$(DefineConstants) diff --git a/sources/engine/Stride.Audio.Tests/VideoSmokeTest.cs b/sources/engine/Stride.Audio.Tests/VideoSmokeTest.cs index 1e737e9c0..50c09937e 100644 --- a/sources/engine/Stride.Audio.Tests/VideoSmokeTest.cs +++ b/sources/engine/Stride.Audio.Tests/VideoSmokeTest.cs @@ -1,8 +1,7 @@ // Copyright (c) .NET Foundation and Contributors (https://dotnetfoundation.org/ & https://stride3d.net) // Distributed under the MIT license. See the LICENSE.md file in the project root for more information. -// No video backend on iOS (FFmpeg not built for iOS, no AVFoundation backend yet). -#if STRIDE_VIDEO_FFMPEG || STRIDE_VIDEO_MEDIACODEC +#if STRIDE_VIDEO_FFMPEG || STRIDE_VIDEO_MEDIACODEC || STRIDE_VIDEO_AVFOUNDATION using System; using System.Collections.Generic; using System.Threading.Tasks; diff --git a/sources/engine/Stride.Audio/AVFoundationAssetSliceHelper.cs b/sources/engine/Stride.Audio/AVFoundationAssetSliceHelper.cs new file mode 100644 index 000000000..eeb554b01 --- /dev/null +++ b/sources/engine/Stride.Audio/AVFoundationAssetSliceHelper.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation and Contributors (https://dotnetfoundation.org/ & https://stride3d.net) +// Distributed under the MIT license. See the LICENSE.md file in the project root for more information. + +#if STRIDE_VIDEO_AVFOUNDATION +using System; +using System.IO; + +namespace Stride.Audio; + +// Copies the offset+length bundle slice to a temp file so AVAsset can take a file URL. +internal static class AVFoundationAssetSliceHelper +{ + public static string ExtractAssetSliceToTempFile(string sourcePath, long startPosition, long length, string suffix) + { + var temp = Path.Combine(Path.GetTempPath(), $"stride-{suffix}-{Guid.NewGuid():N}.bin"); + using var src = File.OpenRead(sourcePath); + using var dst = File.Create(temp); + + src.Seek(startPosition, SeekOrigin.Begin); + var buffer = new byte[64 * 1024]; + long remaining = length; + while (remaining > 0) + { + int toRead = (int)Math.Min(buffer.Length, remaining); + int read = src.Read(buffer, 0, toRead); + if (read <= 0) + break; + dst.Write(buffer, 0, read); + remaining -= read; + } + return temp; + } +} +#endif diff --git a/sources/engine/Stride.Audio/Native/AudioLayer.AVFoundation.cs b/sources/engine/Stride.Audio/Native/AudioLayer.AVFoundation.cs new file mode 100644 index 000000000..e24266de4 --- /dev/null +++ b/sources/engine/Stride.Audio/Native/AudioLayer.AVFoundation.cs @@ -0,0 +1,455 @@ +// Copyright (c) .NET Foundation and Contributors (https://dotnetfoundation.org/ & https://stride3d.net) +// Distributed under the MIT license. See the LICENSE.md file in the project root for more information. +#if STRIDE_PLATFORM_IOS || STRIDE_PLATFORM_MACOS +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using AVFoundation; +using Foundation; +using Stride.Core.Mathematics; + +namespace Stride.Audio +{ + /// + /// AVAudioEngine-backed managed implementation of . Replaces + /// libstrideaudio's OpenAL backend on Apple platforms. Handles flow through GCHandle: each + /// public Device/Listener/Source/Buffer.Ptr wraps a GCHandle to a , + /// , etc., resolved with . + /// + public partial class AudioLayer + { + // -- Handle plumbing ----------------------------------------------------- + + private static IntPtr AllocHandle(T obj) where T : class + { + return GCHandle.ToIntPtr(GCHandle.Alloc(obj, GCHandleType.Normal)); + } + + private static T ResolveHandle(IntPtr ptr) where T : class + { + if (ptr == IntPtr.Zero) return null; + var handle = GCHandle.FromIntPtr(ptr); + return handle.Target as T; + } + + private static void FreeHandle(IntPtr ptr) + { + if (ptr == IntPtr.Zero) return; + GCHandle.FromIntPtr(ptr).Free(); + } + + // -- Managed wrappers ---------------------------------------------------- + + private sealed class ManagedDevice + { + public AVAudioEngine Engine; + public AVAudioMixerNode Mixer; // == Engine.MainMixerNode; cached for master volume + public AVAudioEnvironmentNode Env; // 3D mixer; spatialized sources connect here + public List Sources = new(); + } + + private sealed class ManagedListener + { + public ManagedDevice Device; + public bool Enabled; + } + + private sealed class ManagedBuffer + { + public int CapacityBytes; // max bytes BufferFill can write + public AVAudioPcmBuffer Pcm; // null until BufferFill is called the first time + public AudioLayer.BufferType StreamType; + // Owner's current schedule state — used by the streaming path to know when a buffer + // becomes free again after a Source consumed it. + public bool Scheduled; + } + + private sealed class ManagedSource + { + public ManagedDevice Device; + public ManagedListener Listener; + public AVAudioPlayerNode Player; + public AVAudioFormat Format; // declared input format (int16 mono/stereo) + public int SampleRate; + public bool Mono; + public bool Spatialized; + public bool Streamed; + public bool Looping; + public float Gain = 1f; + public ManagedBuffer StaticBuffer; // SourceSetBuffer target; replays on Play + public Queue StreamingFree = new(); // pre-allocated, returned via SourceGetFreeBuffer + public Queue StreamingPending = new(); // queued, not yet consumed + public double PlayRangeStart, PlayRangeStop; + } + + // -- Init / Device ------------------------------------------------------- + + public static bool Init() + { + return true; + } + + public static Device Create(string deviceName, DeviceFlags flags) + { + try + { + var dev = new ManagedDevice + { + Engine = new AVAudioEngine(), + }; + dev.Mixer = dev.Engine.MainMixerNode; + // EnvironmentNode is set up lazily on first spatialized SourceCreate. AVAudioEngine + // on macOS deadlocks during Start if AttachNode/Connect happen before MainMixerNode + // has been touched and the output node format is fully resolved. + if (!dev.Engine.StartAndReturnError(out var error) || error != null) + { + AudioEngine.Logger?.Warning($"AVAudioEngine.Start failed: {error?.LocalizedDescription}"); + return default; + } + return new Device { Ptr = AllocHandle(dev) }; + } + catch (Exception ex) + { + AudioEngine.Logger?.Error($"AudioLayer.Create threw: {ex}"); + return default; + } + } + + private static void EnsureEnv(ManagedDevice dev) + { + if (dev.Env != null) return; + dev.Env = new AVAudioEnvironmentNode(); + dev.Engine.AttachNode(dev.Env); + dev.Engine.Connect(dev.Env, dev.Mixer, null); + } + + public static void Destroy(Device device) + { + var dev = ResolveHandle(device.Ptr); + if (dev == null) return; + + // Stop and tear down player nodes still attached. + foreach (var src in dev.Sources) + { + try { src.Player?.Stop(); } catch { } + try { if (src.Player != null) dev.Engine.DetachNode(src.Player); } catch { } + } + dev.Sources.Clear(); + + try { dev.Engine.Stop(); } catch { } + try { if (dev.Env != null) dev.Engine.DetachNode(dev.Env); } catch { } + dev.Engine.Dispose(); + FreeHandle(device.Ptr); + } + + public static void Update(Device device) + { + // AVAudioEngine is push-driven; no per-frame work required. + } + + public static void SetMasterVolume(Device device, float volume) + { + var dev = ResolveHandle(device.Ptr); + if (dev?.Mixer == null) return; + dev.Mixer.OutputVolume = volume; + } + + // -- Listener ------------------------------------------------------------ + + public static Listener ListenerCreate(Device device) + { + var dev = ResolveHandle(device.Ptr); + if (dev == null) return default; + var listener = new ManagedListener { Device = dev }; + return new Listener { Ptr = AllocHandle(listener) }; + } + + public static void ListenerDestroy(Listener listener) => FreeHandle(listener.Ptr); + + public static bool ListenerEnable(Listener listener) + { + var l = ResolveHandle(listener.Ptr); + if (l == null) return false; + l.Enabled = true; + return true; + } + + public static void ListenerDisable(Listener listener) + { + var l = ResolveHandle(listener.Ptr); + if (l != null) l.Enabled = false; + } + + public static void ListenerPush3D(Listener listener, ref Vector3 pos, ref Vector3 forward, ref Vector3 up, ref Vector3 vel, ref Matrix worldTransform) + { + var l = ResolveHandle(listener.Ptr); + if (l?.Device?.Env == null) return; + l.Device.Env.ListenerPosition = new System.Numerics.Vector3(pos.X, pos.Y, pos.Z); + l.Device.Env.ListenerVectorOrientation = new AVAudio3DVectorOrientation( + new System.Numerics.Vector3(forward.X, forward.Y, forward.Z), + new System.Numerics.Vector3(up.X, up.Y, up.Z)); + } + + // -- Buffer -------------------------------------------------------------- + + public static Buffer BufferCreate(int maxBufferSizeBytes) + { + var buf = new ManagedBuffer { CapacityBytes = maxBufferSizeBytes }; + return new Buffer { Ptr = AllocHandle(buf) }; + } + + public static void BufferDestroy(Buffer buffer) + { + var b = ResolveHandle(buffer.Ptr); + if (b != null) + { + b.Pcm?.Dispose(); + b.Pcm = null; + } + FreeHandle(buffer.Ptr); + } + + public static unsafe void BufferFill(Buffer buffer, IntPtr pcm, int bufferSize, int sampleRate, bool mono) + { + var b = ResolveHandle(buffer.Ptr); + if (b == null || pcm == IntPtr.Zero || bufferSize <= 0) return; + + uint channels = mono ? 1u : 2u; + // Stride writes int16 interleaved PCM into the buffer. + using var format = new AVAudioFormat(AVAudioCommonFormat.PCMInt16, sampleRate, channels, interleaved: true); + uint frameCapacity = (uint)(bufferSize / (2 * channels)); + if (b.Pcm == null || b.Pcm.FrameCapacity < frameCapacity || b.Pcm.Format == null || (int)b.Pcm.Format.SampleRate != sampleRate || b.Pcm.Format.ChannelCount != channels) + { + b.Pcm?.Dispose(); + b.Pcm = new AVAudioPcmBuffer(format, frameCapacity); + } + // AVAudioPcmBuffer.Int16ChannelData is one pointer per channel; interleaved formats expose + // a single channel pointer holding all samples interleaved. + var dst = b.Pcm.Int16ChannelData; + if (dst != IntPtr.Zero) + { + System.Buffer.MemoryCopy((void*)pcm, ((short**)dst)[0], frameCapacity * 2 * channels, bufferSize); + } + b.Pcm.FrameLength = frameCapacity; + } + + // -- Source -------------------------------------------------------------- + + public static Source SourceCreate(Listener listener, int sampleRate, int maxNumberOfBuffers, bool mono, bool spatialized, bool streamed, bool hrtf, float hrtfDirectionFactor, HrtfEnvironment environment) + { + var l = ResolveHandle(listener.Ptr); + if (l?.Device == null) return default; + var dev = l.Device; + + var src = new ManagedSource + { + Device = dev, + Listener = l, + SampleRate = sampleRate, + Mono = mono, + Spatialized = spatialized, + Streamed = streamed, + }; + uint channels = mono ? 1u : 2u; + src.Format = new AVAudioFormat(AVAudioCommonFormat.PCMInt16, sampleRate, channels, interleaved: true); + src.Player = new AVAudioPlayerNode(); + dev.Engine.AttachNode(src.Player); + + // 3D-spatialized mono sources route through the environment node (lazy-created); + // everything else goes straight to the main mixer. + if (spatialized && mono) + { + EnsureEnv(dev); + dev.Engine.Connect(src.Player, dev.Env, src.Format); + } + else + { + dev.Engine.Connect(src.Player, dev.Mixer, src.Format); + } + + // Streaming sources pre-allocate a pool of empty buffers that SourceGetFreeBuffer + // hands back to the caller; BufferFill (via the buffer the caller filled) + QueueBuffer + // recycle them as the player consumes them. + if (streamed && maxNumberOfBuffers > 0) + { + for (int i = 0; i < maxNumberOfBuffers; i++) + src.StreamingFree.Enqueue(new ManagedBuffer { CapacityBytes = 0 }); + } + + lock (dev.Sources) dev.Sources.Add(src); + return new Source { Ptr = AllocHandle(src) }; + } + + public static void SourceDestroy(Source source) + { + var src = ResolveHandle(source.Ptr); + if (src != null) + { + try { src.Player?.Stop(); } catch { } + try { if (src.Player != null && src.Device != null) src.Device.Engine.DetachNode(src.Player); } catch { } + if (src.Device != null) + lock (src.Device.Sources) src.Device.Sources.Remove(src); + } + FreeHandle(source.Ptr); + } + + public static double SourceGetPosition(Source source) + { + var src = ResolveHandle(source.Ptr); + if (src?.Player?.LastRenderTime is { } rt && src.SampleRate > 0) + { + var playerTime = src.Player.GetPlayerTimeFromNodeTime(rt); + if (playerTime != null) return (double)playerTime.SampleTime / src.SampleRate; + } + return 0; + } + + public static void SourceSetPan(Source source, float pan) + { + var src = ResolveHandle(source.Ptr); + if (src?.Player != null) src.Player.Pan = pan; + } + + public static void SourceSetBuffer(Source source, Buffer buffer) + { + var src = ResolveHandle(source.Ptr); + var buf = ResolveHandle(buffer.Ptr); + if (src == null) return; + src.StaticBuffer = buf; + } + + public static void SourceFlushBuffers(Source source) + { + var src = ResolveHandle(source.Ptr); + if (src?.Player == null) return; + try { src.Player.Stop(); } catch { } + // Recycle pending streaming buffers back to the free pool. + while (src.StreamingPending.Count > 0) + { + var b = src.StreamingPending.Dequeue(); + b.Scheduled = false; + src.StreamingFree.Enqueue(b); + } + } + + public static void SourceQueueBuffer(Source source, Buffer buffer, IntPtr pcm, int bufferSize, BufferType streamType) + { + var src = ResolveHandle(source.Ptr); + var buf = ResolveHandle(buffer.Ptr); + if (src?.Player == null || buf == null) return; + // Caller already wrote PCM into the buffer's AVAudioPcmBuffer via BufferFill. Just schedule. + buf.StreamType = streamType; + buf.Scheduled = true; + src.StreamingPending.Enqueue(buf); + if (buf.Pcm != null) + { + src.Player.ScheduleBuffer(buf.Pcm, () => + { + // Completion runs on an AV worker thread; bounce back onto a free pool the + // game thread polls via SourceGetFreeBuffer. + lock (src.StreamingFree) + { + buf.Scheduled = false; + src.StreamingFree.Enqueue(buf); + } + }); + } + } + + public static Buffer SourceGetFreeBuffer(Source source) + { + var src = ResolveHandle(source.Ptr); + if (src == null) return default; + lock (src.StreamingFree) + { + if (src.StreamingFree.Count == 0) return default; + var b = src.StreamingFree.Dequeue(); + return new Buffer { Ptr = AllocHandle(b) }; + } + } + + public static void SourcePlay(Source source) + { + var src = ResolveHandle(source.Ptr); + if (src?.Player == null) return; + // Static (non-streamed) sources schedule the single attached buffer here. The streaming + // path schedules via SourceQueueBuffer; SourcePlay just resumes the player node. + if (!src.Streamed && src.StaticBuffer?.Pcm != null && !src.Player.Playing) + { + var options = src.Looping ? AVAudioPlayerNodeBufferOptions.Loops : AVAudioPlayerNodeBufferOptions.Interrupts; + src.Player.ScheduleBuffer(src.StaticBuffer.Pcm, when: null, options, completionHandler: () => { }); + } + src.Player.Play(); + } + + public static void SourcePause(Source source) + { + var src = ResolveHandle(source.Ptr); + try { src?.Player?.Pause(); } catch { } + } + + public static void SourceStop(Source source) + { + var src = ResolveHandle(source.Ptr); + try { src?.Player?.Stop(); } catch { } + if (src != null) + { + while (src.StreamingPending.Count > 0) + { + var b = src.StreamingPending.Dequeue(); + b.Scheduled = false; + src.StreamingFree.Enqueue(b); + } + } + } + + public static void SourceSetLooping(Source source, bool looped) + { + var src = ResolveHandle(source.Ptr); + if (src != null) src.Looping = looped; + } + + public static void SourceSetRange(Source source, double startTime, double stopTime) + { + var src = ResolveHandle(source.Ptr); + if (src != null) + { + src.PlayRangeStart = startTime; + src.PlayRangeStop = stopTime; + } + } + + public static void SourceSetGain(Source source, float gain) + { + var src = ResolveHandle(source.Ptr); + if (src != null) + { + src.Gain = gain; + if (src.Player != null) src.Player.Volume = gain; + } + } + + public static void SourceSetPitch(Source source, float pitch) + { + // AVAudioPlayerNode has no rate property; an AVAudioUnitVarispeed node between player + // and mixer would handle this. Skipped until a real consumer needs it. + } + + public static void SourcePush3D(Source source, ref Vector3 pos, ref Vector3 forward, ref Vector3 up, ref Vector3 vel, ref Matrix worldTransform) + { + var src = ResolveHandle(source.Ptr); + if (src?.Player == null) return; + if (src.Spatialized) + { + src.Player.Position = new System.Numerics.Vector3(pos.X, pos.Y, pos.Z); + } + } + + public static bool SourceIsPlaying(Source source) + { + var src = ResolveHandle(source.Ptr); + return src?.Player?.Playing ?? false; + } + } +} +#endif diff --git a/sources/engine/Stride.Audio/Native/AudioLayer.OpenAL.cs b/sources/engine/Stride.Audio/Native/AudioLayer.OpenAL.cs new file mode 100644 index 000000000..bb5461be3 --- /dev/null +++ b/sources/engine/Stride.Audio/Native/AudioLayer.OpenAL.cs @@ -0,0 +1,142 @@ +// Copyright (c) .NET Foundation and Contributors (https://dotnetfoundation.org/ & https://stride3d.net) and Silicon Studio Corp. (https://www.siliconstudio.co.jp) +// Distributed under the MIT license. See the LICENSE.md file in the project root for more information. +#if !(STRIDE_PLATFORM_IOS || STRIDE_PLATFORM_MACOS) +using System; +using System.Runtime.InteropServices; +using System.Security; +using Stride.Core.Mathematics; + +namespace Stride.Audio +{ + /// + /// libstrideaudio (OpenAL on desktop / OpenSL ES on Android / XAudio2 on Windows / static-linked OpenAL on iOS) backend. + /// + public partial class AudioLayer + { + static AudioLayer() + { + NativeInvoke.PreLoad(); + } + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioInit", CallingConvention = CallingConvention.Cdecl)] + public static extern bool Init(); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioCreate", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Unicode)] + public static extern Device Create(string deviceName, DeviceFlags flags); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioDestroy", CallingConvention = CallingConvention.Cdecl)] + public static extern void Destroy(Device device); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioUpdate", CallingConvention = CallingConvention.Cdecl)] + public static extern void Update(Device device); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSetMasterVolume", CallingConvention = CallingConvention.Cdecl)] + public static extern void SetMasterVolume(Device device, float volume); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioListenerCreate", CallingConvention = CallingConvention.Cdecl)] + public static extern Listener ListenerCreate(Device device); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioListenerDestroy", CallingConvention = CallingConvention.Cdecl)] + public static extern void ListenerDestroy(Listener listener); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioListenerEnable", CallingConvention = CallingConvention.Cdecl)] + public static extern bool ListenerEnable(Listener listener); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioListenerDisable", CallingConvention = CallingConvention.Cdecl)] + public static extern void ListenerDisable(Listener listener); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceCreate", CallingConvention = CallingConvention.Cdecl)] + public static extern Source SourceCreate(Listener listener, int sampleRate, int maxNumberOfBuffers, bool mono, bool spatialized, bool streamed, bool hrtf, float hrtfDirectionFactor, HrtfEnvironment environment); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceDestroy", CallingConvention = CallingConvention.Cdecl)] + public static extern void SourceDestroy(Source source); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceGetPosition", CallingConvention = CallingConvention.Cdecl)] + public static extern double SourceGetPosition(Source source); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceSetPan", CallingConvention = CallingConvention.Cdecl)] + public static extern void SourceSetPan(Source source, float pan); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioBufferCreate", CallingConvention = CallingConvention.Cdecl)] + public static extern Buffer BufferCreate(int maxBufferSizeBytes); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioBufferDestroy", CallingConvention = CallingConvention.Cdecl)] + public static extern void BufferDestroy(Buffer buffer); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioBufferFill", CallingConvention = CallingConvention.Cdecl)] + public static extern void BufferFill(Buffer buffer, IntPtr pcm, int bufferSize, int sampleRate, bool mono); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceSetBuffer", CallingConvention = CallingConvention.Cdecl)] + public static extern void SourceSetBuffer(Source source, Buffer buffer); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceFlushBuffers", CallingConvention = CallingConvention.Cdecl)] + public static extern void SourceFlushBuffers(Source source); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceQueueBuffer", CallingConvention = CallingConvention.Cdecl)] + public static extern void SourceQueueBuffer(Source source, Buffer buffer, IntPtr pcm, int bufferSize, BufferType streamType); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceGetFreeBuffer", CallingConvention = CallingConvention.Cdecl)] + public static extern Buffer SourceGetFreeBuffer(Source source); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourcePlay", CallingConvention = CallingConvention.Cdecl)] + public static extern void SourcePlay(Source source); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourcePause", CallingConvention = CallingConvention.Cdecl)] + public static extern void SourcePause(Source source); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceStop", CallingConvention = CallingConvention.Cdecl)] + public static extern void SourceStop(Source source); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceSetLooping", CallingConvention = CallingConvention.Cdecl)] + public static extern void SourceSetLooping(Source source, bool looped); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceSetRange", CallingConvention = CallingConvention.Cdecl)] + public static extern void SourceSetRange(Source source, double startTime, double stopTime); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceSetGain", CallingConvention = CallingConvention.Cdecl)] + public static extern void SourceSetGain(Source source, float gain); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceSetPitch", CallingConvention = CallingConvention.Cdecl)] + public static extern void SourceSetPitch(Source source, float pitch); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioListenerPush3D", CallingConvention = CallingConvention.Cdecl)] + public static extern void ListenerPush3D(Listener listener, ref Vector3 pos, ref Vector3 forward, ref Vector3 up, ref Vector3 vel, ref Matrix worldTransform); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourcePush3D", CallingConvention = CallingConvention.Cdecl)] + public static extern void SourcePush3D(Source source, ref Vector3 pos, ref Vector3 forward, ref Vector3 up, ref Vector3 vel, ref Matrix worldTransform); + + [SuppressUnmanagedCodeSecurity] + [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceIsPlaying", CallingConvention = CallingConvention.Cdecl)] + public static extern bool SourceIsPlaying(Source source); + } +} +#endif diff --git a/sources/engine/Stride.Audio/Native/AudioLayer.cs b/sources/engine/Stride.Audio/Native/AudioLayer.cs index c94993122..d2bce273b 100644 --- a/sources/engine/Stride.Audio/Native/AudioLayer.cs +++ b/sources/engine/Stride.Audio/Native/AudioLayer.cs @@ -1,16 +1,15 @@ // Copyright (c) .NET Foundation and Contributors (https://dotnetfoundation.org/ & https://stride3d.net) and Silicon Studio Corp. (https://www.siliconstudio.co.jp) // Distributed under the MIT license. See the LICENSE.md file in the project root for more information. using System; -using System.Runtime.InteropServices; -using System.Security; -using Stride.Core.Mathematics; namespace Stride.Audio { /// - /// Wrapper around OpenAL + /// Cross-platform audio device abstraction. Static method bodies live in platform-specific partials: + /// AudioLayer.OpenAL.cs (Windows/Linux/Android/iOS — DllImport into libstrideaudio), + /// AudioLayer.AVFoundation.cs (iOS/macOS — managed AVAudioEngine implementation). /// - public class AudioLayer + public partial class AudioLayer { public struct Device { @@ -32,89 +31,12 @@ namespace Stride.Audio public IntPtr Ptr; } - static AudioLayer() - { - NativeInvoke.PreLoad(); - } - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioInit", CallingConvention = CallingConvention.Cdecl)] - public static extern bool Init(); - public enum DeviceFlags { None, Hrtf, } - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioCreate", CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Unicode)] - public static extern Device Create(string deviceName, DeviceFlags flags); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioDestroy", CallingConvention = CallingConvention.Cdecl)] - public static extern void Destroy(Device device); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioUpdate", CallingConvention = CallingConvention.Cdecl)] - public static extern void Update(Device device); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSetMasterVolume", CallingConvention = CallingConvention.Cdecl)] - public static extern void SetMasterVolume(Device device, float volume); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioListenerCreate", CallingConvention = CallingConvention.Cdecl)] - public static extern Listener ListenerCreate(Device device); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioListenerDestroy", CallingConvention = CallingConvention.Cdecl)] - public static extern void ListenerDestroy(Listener listener); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioListenerEnable", CallingConvention = CallingConvention.Cdecl)] - public static extern bool ListenerEnable(Listener listener); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioListenerDisable", CallingConvention = CallingConvention.Cdecl)] - public static extern void ListenerDisable(Listener listener); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceCreate", CallingConvention = CallingConvention.Cdecl)] - public static extern Source SourceCreate(Listener listener, int sampleRate, int maxNumberOfBuffers, bool mono, bool spatialized, bool streamed, bool hrtf, float hrtfDirectionFactor, HrtfEnvironment environment); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceDestroy", CallingConvention = CallingConvention.Cdecl)] - public static extern void SourceDestroy(Source source); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceGetPosition", CallingConvention = CallingConvention.Cdecl)] - public static extern double SourceGetPosition(Source source); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceSetPan", CallingConvention = CallingConvention.Cdecl)] - public static extern void SourceSetPan(Source source, float pan); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioBufferCreate", CallingConvention = CallingConvention.Cdecl)] - public static extern Buffer BufferCreate(int maxBufferSizeBytes); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioBufferDestroy", CallingConvention = CallingConvention.Cdecl)] - public static extern void BufferDestroy(Buffer buffer); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioBufferFill", CallingConvention = CallingConvention.Cdecl)] - public static extern void BufferFill(Buffer buffer, IntPtr pcm, int bufferSize, int sampleRate, bool mono); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceSetBuffer", CallingConvention = CallingConvention.Cdecl)] - public static extern void SourceSetBuffer(Source source, Buffer buffer); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceFlushBuffers", CallingConvention = CallingConvention.Cdecl)] - public static extern void SourceFlushBuffers(Source source); - public enum BufferType { None, @@ -122,53 +44,5 @@ namespace Stride.Audio EndOfStream, EndOfLoop, } - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceQueueBuffer", CallingConvention = CallingConvention.Cdecl)] - public static extern void SourceQueueBuffer(Source source, Buffer buffer, IntPtr pcm, int bufferSize, BufferType streamType); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceGetFreeBuffer", CallingConvention = CallingConvention.Cdecl)] - public static extern Buffer SourceGetFreeBuffer(Source source); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourcePlay", CallingConvention = CallingConvention.Cdecl)] - public static extern void SourcePlay(Source source); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourcePause", CallingConvention = CallingConvention.Cdecl)] - public static extern void SourcePause(Source source); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceStop", CallingConvention = CallingConvention.Cdecl)] - public static extern void SourceStop(Source source); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceSetLooping", CallingConvention = CallingConvention.Cdecl)] - public static extern void SourceSetLooping(Source source, bool looped); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceSetRange", CallingConvention = CallingConvention.Cdecl)] - public static extern void SourceSetRange(Source source, double startTime, double stopTime); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceSetGain", CallingConvention = CallingConvention.Cdecl)] - public static extern void SourceSetGain(Source source, float gain); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceSetPitch", CallingConvention = CallingConvention.Cdecl)] - public static extern void SourceSetPitch(Source source, float pitch); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioListenerPush3D", CallingConvention = CallingConvention.Cdecl)] - public static extern void ListenerPush3D(Listener listener, ref Vector3 pos, ref Vector3 forward, ref Vector3 up, ref Vector3 vel, ref Matrix worldTransform); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourcePush3D", CallingConvention = CallingConvention.Cdecl)] - public static extern void SourcePush3D(Source source, ref Vector3 pos, ref Vector3 forward, ref Vector3 up, ref Vector3 vel, ref Matrix worldTransform); - - [SuppressUnmanagedCodeSecurity] - [DllImport(NativeInvoke.Library, EntryPoint = "xnAudioSourceIsPlaying", CallingConvention = CallingConvention.Cdecl)] - public static extern bool SourceIsPlaying(Source source); } } diff --git a/sources/engine/Stride.Audio/Native/Celt.AVFoundation.cs b/sources/engine/Stride.Audio/Native/Celt.AVFoundation.cs new file mode 100644 index 000000000..f1bcc0a3c --- /dev/null +++ b/sources/engine/Stride.Audio/Native/Celt.AVFoundation.cs @@ -0,0 +1,171 @@ +// Copyright (c) .NET Foundation and Contributors (https://dotnetfoundation.org/ & https://stride3d.net) +// Distributed under the MIT license. See the LICENSE.md file in the project root for more information. +#if STRIDE_PLATFORM_IOS || STRIDE_PLATFORM_MACOS +#pragma warning disable SA1300 +using System; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Security; + +namespace Stride.Audio +{ + /// + /// Managed Celt wrapper that calls libCelt's opus_custom_* symbols directly. + /// Replaces the libstrideaudio xnCelt* shim on Apple platforms. + /// + internal class Celt : IDisposable + { + // iOS: static-only linkage, libCelt.a is force-loaded into the app binary, __Internal hits dlsym(RTLD_DEFAULT). + // macOS: libstrideaudio.dylib statically embeds libCelt at engine build time, exposing opus_custom_* as + // dynamic exports — DllImport("strideaudio") resolves them at runtime via the loaded dylib. +#if STRIDE_PLATFORM_IOS + private const string LibCelt = "__Internal"; +#else + private const string LibCelt = "strideaudio"; +#endif + + // OPUS control request codes, copied from deps/Celt/include/opus_defines.h. + private const int OPUS_RESET_STATE = 4028; + private const int OPUS_GET_LOOKAHEAD_REQUEST = 4027; + + public int SampleRate { get; set; } + public int BufferSize { get; set; } + public int Channels { get; set; } + + private IntPtr mode; + private IntPtr decoder; + private IntPtr encoder; + + public Celt(int sampleRate, int bufferSize, int channels, bool decoderOnly) + { +#if STRIDE_PLATFORM_MACOS + // Decode crashes inside libCelt.a's decode_pulses on osx-arm64 after a handful of + // packets — investigation pending. Bail cleanly so Sound loading raises a managed + // exception instead of taking down the whole process. + throw new NotSupportedException("Celt decoder not yet supported on macOS (osx-arm64 libCelt.a needs investigation)."); +#else + SampleRate = sampleRate; + BufferSize = bufferSize; + Channels = channels; + + mode = opus_custom_mode_create(sampleRate, bufferSize, IntPtr.Zero); + if (mode == IntPtr.Zero) + throw new Exception("Failed to create Celt custom mode."); + + decoder = opus_custom_decoder_create(mode, channels, IntPtr.Zero); + if (decoder == IntPtr.Zero) + throw new Exception("Failed to create Celt decoder."); + + if (!decoderOnly) + { + encoder = opus_custom_encoder_create(mode, channels, IntPtr.Zero); + if (encoder == IntPtr.Zero) + throw new Exception("Failed to create Celt encoder."); + } +#endif + } + + public void Dispose() + { + if (encoder != IntPtr.Zero) { opus_custom_encoder_destroy(encoder); encoder = IntPtr.Zero; } + if (decoder != IntPtr.Zero) { opus_custom_decoder_destroy(decoder); decoder = IntPtr.Zero; } + if (mode != IntPtr.Zero) { opus_custom_mode_destroy(mode); mode = IntPtr.Zero; } + } + + public unsafe int Decode(byte[] inputBuffer, int inputBufferSize, short[] outputSamples) + { + Debug.Assert((uint)inputBufferSize <= (uint)inputBuffer.Length); + fixed (short* samplesPtr = outputSamples) + fixed (byte* bufferPtr = inputBuffer) + { + return opus_custom_decode(decoder, bufferPtr, inputBufferSize, samplesPtr, outputSamples.Length / Channels); + } + } + + public unsafe int Decode(byte[] inputBuffer, int inputBufferSize, short* outputSamples) + { + Debug.Assert((uint)inputBufferSize <= (uint)inputBuffer.Length); + fixed (byte* bufferPtr = inputBuffer) + { + return opus_custom_decode(decoder, bufferPtr, inputBufferSize, outputSamples, BufferSize); + } + } + + public unsafe int Decode(byte[] inputBuffer, int inputBufferSize, float[] outputSamples) + { + fixed (float* samplesPtr = outputSamples) + fixed (byte* bufferPtr = inputBuffer) + { + return opus_custom_decode_float(decoder, bufferPtr, inputBufferSize, samplesPtr, outputSamples.Length / Channels); + } + } + + public unsafe int Encode(short[] audioSamples, byte[] outputBuffer) + { + fixed (short* samplesPtr = audioSamples) + fixed (byte* bufferPtr = outputBuffer) + { + return opus_custom_encode(encoder, samplesPtr, audioSamples.Length / Channels, bufferPtr, outputBuffer.Length); + } + } + + public unsafe int Encode(float[] audioSamples, byte[] outputBuffer) + { + fixed (float* samplesPtr = audioSamples) + fixed (byte* bufferPtr = outputBuffer) + { + return opus_custom_encode_float(encoder, samplesPtr, audioSamples.Length / Channels, bufferPtr, outputBuffer.Length); + } + } + + public void ResetDecoder() + { + opus_custom_decoder_ctl_reset(decoder, OPUS_RESET_STATE); + } + + public unsafe int GetDecoderSampleDelay() + { + int delay = 0; + int err = opus_custom_decoder_ctl_get_int(decoder, OPUS_GET_LOOKAHEAD_REQUEST, &delay); + return err == 0 ? delay : 0; + } + + // libCelt opus_custom_* exports — see LibCelt constant above for per-platform resolution. + [SuppressUnmanagedCodeSecurity, DllImport(LibCelt, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr opus_custom_mode_create(int sampleRate, int frameSize, IntPtr error); + + [SuppressUnmanagedCodeSecurity, DllImport(LibCelt, CallingConvention = CallingConvention.Cdecl)] + private static extern void opus_custom_mode_destroy(IntPtr mode); + + [SuppressUnmanagedCodeSecurity, DllImport(LibCelt, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr opus_custom_decoder_create(IntPtr mode, int channels, IntPtr error); + + [SuppressUnmanagedCodeSecurity, DllImport(LibCelt, CallingConvention = CallingConvention.Cdecl)] + private static extern void opus_custom_decoder_destroy(IntPtr decoder); + + [SuppressUnmanagedCodeSecurity, DllImport(LibCelt, CallingConvention = CallingConvention.Cdecl)] + private static extern unsafe int opus_custom_decode(IntPtr decoder, byte* data, int len, short* pcm, int frameSize); + + [SuppressUnmanagedCodeSecurity, DllImport(LibCelt, CallingConvention = CallingConvention.Cdecl)] + private static extern unsafe int opus_custom_decode_float(IntPtr decoder, byte* data, int len, float* pcm, int frameSize); + + [SuppressUnmanagedCodeSecurity, DllImport(LibCelt, EntryPoint = "opus_custom_decoder_ctl", CallingConvention = CallingConvention.Cdecl)] + private static extern int opus_custom_decoder_ctl_reset(IntPtr decoder, int request); + + [SuppressUnmanagedCodeSecurity, DllImport(LibCelt, EntryPoint = "opus_custom_decoder_ctl", CallingConvention = CallingConvention.Cdecl)] + private static extern unsafe int opus_custom_decoder_ctl_get_int(IntPtr decoder, int request, int* value); + + [SuppressUnmanagedCodeSecurity, DllImport(LibCelt, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr opus_custom_encoder_create(IntPtr mode, int channels, IntPtr error); + + [SuppressUnmanagedCodeSecurity, DllImport(LibCelt, CallingConvention = CallingConvention.Cdecl)] + private static extern void opus_custom_encoder_destroy(IntPtr encoder); + + [SuppressUnmanagedCodeSecurity, DllImport(LibCelt, CallingConvention = CallingConvention.Cdecl)] + private static extern unsafe int opus_custom_encode(IntPtr encoder, short* pcm, int frameSize, byte* data, int maxDataBytes); + + [SuppressUnmanagedCodeSecurity, DllImport(LibCelt, CallingConvention = CallingConvention.Cdecl)] + private static extern unsafe int opus_custom_encode_float(IntPtr encoder, float* pcm, int frameSize, byte* data, int maxDataBytes); + } +} +#endif diff --git a/sources/engine/Stride.Audio/Native/Celt.cs b/sources/engine/Stride.Audio/Native/Celt.cs index ff8885e60..12fb8977f 100644 --- a/sources/engine/Stride.Audio/Native/Celt.cs +++ b/sources/engine/Stride.Audio/Native/Celt.cs @@ -1,5 +1,6 @@ // Copyright (c) .NET Foundation and Contributors (https://dotnetfoundation.org/ & https://stride3d.net) and Silicon Studio Corp. (https://www.siliconstudio.co.jp) // Distributed under the MIT license. See the LICENSE.md file in the project root for more information. +#if !(STRIDE_PLATFORM_IOS || STRIDE_PLATFORM_MACOS) #pragma warning disable SA1300 // Element must begin with upper-case letter using System; using System.Diagnostics; @@ -190,3 +191,4 @@ namespace Stride.Audio private static extern unsafe int xnCeltDecodeShort(IntPtr celt, byte* inputBuffer, int inputBufferSize, short* outputBuffer, int numberOfOutputSamples); } } +#endif diff --git a/sources/engine/Stride.Audio/StreamedBufferSound.AVFoundation.cs b/sources/engine/Stride.Audio/StreamedBufferSound.AVFoundation.cs new file mode 100644 index 000000000..a43c62ffa --- /dev/null +++ b/sources/engine/Stride.Audio/StreamedBufferSound.AVFoundation.cs @@ -0,0 +1,52 @@ +// Copyright (c) .NET Foundation and Contributors (https://dotnetfoundation.org/ & https://stride3d.net) +// Distributed under the MIT license. See the LICENSE.md file in the project root for more information. + +#if STRIDE_VIDEO_AVFOUNDATION +using System; +using System.IO; +using AVFoundation; +using AudioToolbox; +using CoreMedia; +using Foundation; +using Stride.Media; + +namespace Stride.Audio; + +public partial class StreamedBufferSound : SoundBase, IMediaExtractor +{ + partial void InitializeImpl() + { + if (mediaDataUrl == null) + throw new ArgumentNullException(nameof(mediaDataUrl)); + + var tempPath = AVFoundationAssetSliceHelper.ExtractAssetSliceToTempFile(mediaDataUrl, startPosition, length, "audio-probe"); + try + { + using var url = NSUrl.FromFilename(tempPath); + using var asset = AVAsset.FromUrl(url); + + var audioTracks = asset.TracksWithMediaType(AVMediaTypes.Audio.GetConstant()); + if (audioTracks == null || audioTracks.Length == 0) + return; + + var audioTrack = audioTracks[0]; + MediaDuration = TimeSpan.FromSeconds(asset.Duration.Seconds); + + var formatDescriptions = audioTrack.FormatDescriptions; + if (formatDescriptions == null || formatDescriptions.Length == 0) + return; + + if (formatDescriptions[0] is CMAudioFormatDescription formatDesc + && formatDesc.AudioStreamBasicDescription is { } asbd) + { + Channels = (int)asbd.ChannelsPerFrame; + SampleRate = (int)asbd.SampleRate; + } + } + finally + { + try { File.Delete(tempPath); } catch { /* best-effort */ } + } + } +} +#endif diff --git a/sources/engine/Stride.Audio/StreamedBufferSoundSource.AVFoundation.cs b/sources/engine/Stride.Audio/StreamedBufferSoundSource.AVFoundation.cs new file mode 100644 index 000000000..345e7f19c --- /dev/null +++ b/sources/engine/Stride.Audio/StreamedBufferSoundSource.AVFoundation.cs @@ -0,0 +1,188 @@ +// Copyright (c) .NET Foundation and Contributors (https://dotnetfoundation.org/ & https://stride3d.net) +// Distributed under the MIT license. See the LICENSE.md file in the project root for more information. + +#if STRIDE_VIDEO_AVFOUNDATION +using System; +using System.IO; +using System.Runtime.InteropServices; +using AVFoundation; +using AudioToolbox; +using CoreMedia; +using Foundation; +using Stride.Core.Extensions; +using Stride.Media; + +namespace Stride.Audio; + +public partial class StreamedBufferSoundSource : DynamicSoundSource +{ + private string audioTempFilePath; + private AVAsset audioAsset; + private AVAssetReader audioReader; + private AVAssetReaderAudioMixOutput audioOutput; + private AVAssetTrack audioTrack; + + private bool audioExtractionDone; + + partial void InitializeMediaExtractor(string mediaDataUrl, long startPosition, long length) + { + if (mediaDataUrl == null) + throw new ArgumentNullException(nameof(mediaDataUrl)); + + ReleaseMediaInternal(); + + audioTempFilePath = AVFoundationAssetSliceHelper.ExtractAssetSliceToTempFile( + mediaDataUrl, startPosition, length, "audio-stream"); + + using var url = NSUrl.FromFilename(audioTempFilePath); + audioAsset = AVAsset.FromUrl(url); + + var audioTracks = audioAsset.TracksWithMediaType(AVMediaTypes.Audio.GetConstant()); + if (audioTracks == null || audioTracks.Length == 0) + { + Logger.Error($"The input file '{mediaDataUrl}' does not contain any audio track."); + ReleaseMediaInternal(); + return; + } + audioTrack = audioTracks[0]; + + int sourceChannels = 2; + int sourceSampleRate = 44100; + var formatDescriptions = audioTrack.FormatDescriptions; + if (formatDescriptions != null && formatDescriptions.Length > 0 + && formatDescriptions[0] is CMAudioFormatDescription formatDesc + && formatDesc.AudioStreamBasicDescription is { } asbd) + { + sourceChannels = (int)asbd.ChannelsPerFrame; + sourceSampleRate = (int)asbd.SampleRate; + } + Channels = sourceChannels; + SampleRate = sourceSampleRate; + MediaDuration = TimeSpan.FromSeconds(audioAsset.Duration.Seconds); + + CreateAudioReader(TimeSpan.Zero); + } + + partial void SeekInternalImpl(TimeSpan seekTime) + { + CreateAudioReader(seekTime); + audioExtractionDone = false; + } + + partial void ReleaseMediaInternal() + { + DisposeReader(); + + audioAsset?.Dispose(); + audioAsset = null; + audioTrack = null; + + if (audioTempFilePath != null && File.Exists(audioTempFilePath)) + { + try { File.Delete(audioTempFilePath); } catch { /* best-effort */ } + } + audioTempFilePath = null; + } + + private void CreateAudioReader(TimeSpan startTime) + { + DisposeReader(); + if (audioAsset == null || audioTrack == null) + return; + + audioReader = AVAssetReader.FromAsset(audioAsset, out var error); + if (error != null) + throw new InvalidOperationException($"AVAssetReader (audio) create failed: {error.LocalizedDescription}"); + + if (startTime > TimeSpan.Zero) + { + var startCMTime = new CMTime((long)(startTime.TotalMilliseconds), 1000); + audioReader.TimeRange = new CMTimeRange { Start = startCMTime, Duration = CMTime.PositiveInfinity }; + } + + // 16-bit signed LE interleaved PCM — the format Stride.Audio consumes downstream. + var settings = new NSMutableDictionary + { + [AVAudioSettings.AVFormatIDKey] = NSNumber.FromUInt32((uint)AudioFormatType.LinearPCM), + [AVAudioSettings.AVLinearPCMBitDepthKey] = NSNumber.FromInt32(16), + [AVAudioSettings.AVLinearPCMIsBigEndianKey] = NSNumber.FromBoolean(false), + [AVAudioSettings.AVLinearPCMIsFloatKey] = NSNumber.FromBoolean(false), + [AVAudioSettings.AVLinearPCMIsNonInterleaved] = NSNumber.FromBoolean(false), + }; + + audioOutput = new AVAssetReaderAudioMixOutput(new[] { audioTrack }, settings); + audioReader.AddOutput(audioOutput); + if (!audioReader.StartReading()) + { + var err = audioReader.Error; + throw new InvalidOperationException( + $"AVAssetReader.StartReading (audio) failed: {(err != null ? err.LocalizedDescription : "unknown")}"); + } + } + + private void DisposeReader() + { + audioOutput?.Dispose(); + audioOutput = null; + audioReader?.Dispose(); + audioReader = null; + } + + private bool ExtractSomeAudioData(out bool endOfFile) + { + endOfFile = audioExtractionDone; + if (audioExtractionDone || audioOutput == null) + return false; + + var sampleBuffer = audioOutput.CopyNextSampleBuffer(); + if (sampleBuffer == null) + { + audioExtractionDone = true; + endOfFile = true; + return false; + } + + try + { + using var blockBuffer = sampleBuffer.GetDataBuffer(); + if (blockBuffer == null) + return false; + + int totalLen = (int)blockBuffer.DataLength; + if (totalLen <= 0) + return false; + + var presentationTime = TimeSpan.FromSeconds(sampleBuffer.PresentationTimeStamp.Seconds); + + int available = storageBuffer.Data.Length - storageBuffer.CountDataBytes; + if (totalLen > available) + { + Logger.Error("The storage buffer has reached full capacity. Current data will be dropped"); + return false; + } + + unsafe + { + fixed (byte* dst = &storageBuffer.Data[storageBuffer.CountDataBytes]) + { + var status = blockBuffer.CopyDataBytes(0, (nuint)totalLen, (IntPtr)dst); + if (status != CMBlockBufferError.None) + { + Logger.Error($"CMBlockBuffer.CopyDataBytes failed: {status}"); + return false; + } + } + } + + if (storageBuffer.CountDataBytes == 0) + storageBuffer.PresentationTime = presentationTime; + storageBuffer.CountDataBytes += totalLen; + return true; + } + finally + { + sampleBuffer.Dispose(); + } + } +} +#endif diff --git a/sources/engine/Stride.Audio/StreamedBufferSoundSource.ffmpeg.cs b/sources/engine/Stride.Audio/StreamedBufferSoundSource.ffmpeg.cs index 17f8300c8..d79549bdd 100644 --- a/sources/engine/Stride.Audio/StreamedBufferSoundSource.ffmpeg.cs +++ b/sources/engine/Stride.Audio/StreamedBufferSoundSource.ffmpeg.cs @@ -1,7 +1,8 @@ // Copyright (c) .NET Foundation and Contributors (https://dotnetfoundation.org/ & https://stride3d.net) and Silicon Studio Corp. (https://www.siliconstudio.co.jp) // Distributed under the MIT license. See the LICENSE.md file in the project root for more information. -#if !STRIDE_PLATFORM_ANDROID || !STRIDE_VIDEO_MEDIACODEC +// Stub for platforms with no real audio extractor; suppressed when MediaCodec or AVFoundation is set. +#if !STRIDE_VIDEO_MEDIACODEC && !STRIDE_VIDEO_AVFOUNDATION using System; diff --git a/sources/engine/Stride.Audio/Stride.Audio.csproj b/sources/engine/Stride.Audio/Stride.Audio.csproj index 0ef23a091..9aa02787f 100644 --- a/sources/engine/Stride.Audio/Stride.Audio.csproj +++ b/sources/engine/Stride.Audio/Stride.Audio.csproj @@ -4,9 +4,14 @@ true libstrideaudio true + + true true true + + STRIDE_VIDEO_AVFOUNDATION;$(DefineConstants) + Properties\SharedAssemblyInfo.cs diff --git a/sources/engine/Stride.Graphics/Stride.Graphics.csproj b/sources/engine/Stride.Graphics/Stride.Graphics.csproj index 336afaed8..30cf1c88b 100644 --- a/sources/engine/Stride.Graphics/Stride.Graphics.csproj +++ b/sources/engine/Stride.Graphics/Stride.Graphics.csproj @@ -6,6 +6,7 @@ true true true + true $(DefineConstants);STRIDE_GRAPHICS_NO_DESCRIPTOR_COPIES diff --git a/sources/engine/Stride.Graphics/Vulkan/GraphicsDevice.Vulkan.cs b/sources/engine/Stride.Graphics/Vulkan/GraphicsDevice.Vulkan.cs index b432ae228..d132a816e 100644 --- a/sources/engine/Stride.Graphics/Vulkan/GraphicsDevice.Vulkan.cs +++ b/sources/engine/Stride.Graphics/Vulkan/GraphicsDevice.Vulkan.cs @@ -177,6 +177,11 @@ namespace Stride.Graphics internal bool HasAndroidHardwareBufferSupport { get; private set; } #endif +#if STRIDE_PLATFORM_IOS || STRIDE_PLATFORM_MACOS + // True when VK_EXT_metal_objects was enabled — lets us import CVPixelBuffer IOSurfaces as VkImage. + internal bool HasMetalObjectsSupport { get; private set; } +#endif + /// /// Marks context as active on the current thread. /// @@ -427,6 +432,10 @@ namespace Stride.Graphics { VK_KHR_SWAPCHAIN_EXTENSION_NAME, VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME, +#if STRIDE_PLATFORM_IOS || STRIDE_PLATFORM_MACOS + // IOSurface-backed VkImage import (zero-copy CVPixelBuffer upload on Apple platforms). + VK_EXT_METAL_OBJECTS_EXTENSION_NAME, +#endif #if STRIDE_PLATFORM_ANDROID // AHardwareBuffer import (zero-copy buffers from camera / MediaCodec / etc.). VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME, @@ -453,6 +462,14 @@ namespace Stride.Graphics if (isPortabilitySubsetDevice) desiredExtensionProperties.Add(VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME); +#if STRIDE_PLATFORM_IOS || STRIDE_PLATFORM_MACOS + // VK_EXT_metal_objects lets us import IOSurface-backed CVPixelBuffers as VkImages for + // zero-copy video upload. MoltenVK 1.2+ supports it; no extra dependency extensions needed. + HasMetalObjectsSupport = availableExtensionProperties.Contains(VK_EXT_METAL_OBJECTS_EXTENSION_NAME); + if (HasMetalObjectsSupport) + desiredExtensionProperties.Add(VK_EXT_METAL_OBJECTS_EXTENSION_NAME); +#endif + #if STRIDE_PLATFORM_ANDROID // All extensions in the AHardwareBuffer → VkImage chain must be present together. HasAndroidHardwareBufferSupport = diff --git a/sources/engine/Stride.Graphics/Vulkan/Texture.Apple.Vulkan.cs b/sources/engine/Stride.Graphics/Vulkan/Texture.Apple.Vulkan.cs new file mode 100644 index 000000000..fda5a7ce9 --- /dev/null +++ b/sources/engine/Stride.Graphics/Vulkan/Texture.Apple.Vulkan.cs @@ -0,0 +1,162 @@ +// Copyright (c) .NET Foundation and Contributors (https://dotnetfoundation.org/ & https://stride3d.net) +// Distributed under the MIT license. See the LICENSE.md file in the project root for more information. +#if STRIDE_GRAPHICS_API_VULKAN && (STRIDE_PLATFORM_IOS || STRIDE_PLATFORM_MACOS) +using System; +using System.Runtime.InteropServices; +using Vortice.Vulkan; + +namespace Stride.Graphics +{ + public partial class Texture + { + // Set when this Texture wraps an externally-owned IOSurface imported via VK_EXT_metal_objects. + private IntPtr ioSurfaceRef; + private bool isImportedImage; + + partial void TryInitializeImportedImage() + { + if (!isImportedImage) + return; + + HasStencil = false; + NativeImageAspect = VkImageAspectFlags.Color; + + var arraySlice = ArraySlice; + var mipLevel = MipLevel; + GetViewSliceBounds(ViewType, ref arraySlice, ref mipLevel, out var arrayOrDepthCount, out var mipCount); + NativeResourceRange = new VkImageSubresourceRange(NativeImageAspect, (uint)mipLevel, (uint)mipCount, (uint)arraySlice, (uint)arrayOrDepthCount); + + // Image starts Undefined per vkCreateImage; Stride will emit the transition to + // ShaderReadOnlyOptimal on first sampler bind (CommandList barrier path). + NativeLayout = VkImageLayout.Undefined; + NativeAccessMask = VkAccessFlags.None; + NativePipelineStageMask = VkPipelineStageFlags.TopOfPipe; + LayoutTracker.Initialize(BarrierLayout.Undefined, ArraySize * MipLevelCount); + IsInitialized = true; + importedImageHandled = true; + } + + partial void ReleaseImportedImageResources() + { + if (ioSurfaceRef != IntPtr.Zero) + { + CFRelease(ioSurfaceRef); + ioSurfaceRef = IntPtr.Zero; + } + } + + /// + /// Imports a CoreVideo IOSurface as a sampleable BGRA8_sRGB via VK_EXT_metal_objects. + /// Retains the IOSurface for the lifetime of the returned Texture; the caller is free to dispose its + /// CVPixelBuffer once this call returns. + /// + public static unsafe Texture NewFromIOSurface(GraphicsDevice device, IntPtr ioSurface, int width, int height) + { + if (device == null) throw new ArgumentNullException(nameof(device)); + if (ioSurface == IntPtr.Zero) throw new ArgumentNullException(nameof(ioSurface)); + if (!device.HasMetalObjectsSupport) + throw new InvalidOperationException("IOSurface import requires VK_EXT_metal_objects."); + + CFRetain(ioSurface); + + var ioSurfaceInfo = new VkImportMetalIOSurfaceInfoEXT + { + sType = VkStructureType.ImportMetalIOSurfaceInfoEXT, + ioSurface = ioSurface, + }; + + var imageCreateInfo = new VkImageCreateInfo + { + sType = VkStructureType.ImageCreateInfo, + pNext = &ioSurfaceInfo, + imageType = VkImageType.Image2D, + format = VkFormat.B8G8R8A8Srgb, + extent = new VkExtent3D(width, height, 1), + mipLevels = 1, + arrayLayers = 1, + samples = VkSampleCountFlags.Count1, + tiling = VkImageTiling.Optimal, + usage = VkImageUsageFlags.Sampled, + sharingMode = VkSharingMode.Exclusive, + initialLayout = VkImageLayout.Undefined, + }; + VkImage image; + var imageResult = device.NativeDeviceApi.vkCreateImage(device.NativeDevice, &imageCreateInfo, null, &image); + if (imageResult != VkResult.Success) + { + CFRelease(ioSurface); + device.CheckResult(imageResult); + } + + // MoltenVK actually backs the image with the IOSurface, but the Vulkan spec still + // requires vkBindImageMemory for the validation layer to be happy. Allocate a normal + // VkDeviceMemory block sized to GetImageMemoryRequirements; MoltenVK ignores it as + // soon as the IOSurface-backed MTLTexture is created on first use. + device.NativeDeviceApi.vkGetImageMemoryRequirements(device.NativeDevice, image, out var memReq); + var allocateInfo = new VkMemoryAllocateInfo + { + sType = VkStructureType.MemoryAllocateInfo, + allocationSize = memReq.size, + memoryTypeIndex = FindMemoryTypeIndex(device, memReq.memoryTypeBits), + }; + VkDeviceMemory memory; + var allocResult = device.NativeDeviceApi.vkAllocateMemory(device.NativeDevice, &allocateInfo, null, &memory); + if (allocResult != VkResult.Success) + { + device.NativeDeviceApi.vkDestroyImage(device.NativeDevice, image, null); + CFRelease(ioSurface); + device.CheckResult(allocResult); + } + device.CheckResult(device.NativeDeviceApi.vkBindImageMemory(device.NativeDevice, image, memory, 0)); + + var viewCreateInfo = new VkImageViewCreateInfo + { + sType = VkStructureType.ImageViewCreateInfo, + image = image, + viewType = VkImageViewType.Image2D, + format = VkFormat.B8G8R8A8Srgb, + components = VkComponentMapping.Identity, + subresourceRange = new VkImageSubresourceRange(VkImageAspectFlags.Color, baseMipLevel: 0, levelCount: 1, baseArrayLayer: 0, layerCount: 1), + }; + VkImageView imageView; + var viewResult = device.NativeDeviceApi.vkCreateImageView(device.NativeDevice, &viewCreateInfo, null, &imageView); + if (viewResult != VkResult.Success) + { + device.NativeDeviceApi.vkFreeMemory(device.NativeDevice, memory, null); + device.NativeDeviceApi.vkDestroyImage(device.NativeDevice, image, null); + CFRelease(ioSurface); + device.CheckResult(viewResult); + } + + var description = TextureDescription.New2D(width, height, mipCount: 1, PixelFormat.B8G8R8A8_UNorm_SRgb, TextureFlags.ShaderResource); + + var texture = new Texture(device); + texture.NativeImage = image; + texture.NativeMemory = memory; + texture.NativeImageView = imageView; + texture.NativeFormat = VkFormat.B8G8R8A8Srgb; + texture.ioSurfaceRef = ioSurface; + texture.isImportedImage = true; + texture.InitializeFrom(description); + return texture; + } + + private static unsafe uint FindMemoryTypeIndex(GraphicsDevice device, uint memoryTypeBits) + { + device.NativeInstanceApi.vkGetPhysicalDeviceMemoryProperties(device.NativePhysicalDevice, out var memoryProperties); + for (uint i = 0; i < memoryProperties.memoryTypeCount; i++) + { + if ((memoryTypeBits & (1u << (int)i)) != 0) + return i; + } + throw new InvalidOperationException("Vulkan: no memory type satisfies the IOSurface memoryTypeBits mask."); + } + + [DllImport("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation")] + private static extern IntPtr CFRetain(IntPtr cf); + + [DllImport("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation")] + private static extern void CFRelease(IntPtr cf); + } +} +#endif diff --git a/sources/engine/Stride.Video/Backends/AVFoundationVideoBackend.cs b/sources/engine/Stride.Video/Backends/AVFoundationVideoBackend.cs new file mode 100644 index 000000000..1322c8d22 --- /dev/null +++ b/sources/engine/Stride.Video/Backends/AVFoundationVideoBackend.cs @@ -0,0 +1,426 @@ +// Copyright (c) .NET Foundation and Contributors (https://dotnetfoundation.org/ & https://stride3d.net) +// Distributed under the MIT license. See the LICENSE.md file in the project root for more information. + +#if STRIDE_VIDEO_AVFOUNDATION +using System; +using System.IO; +using System.Linq; +using AVFoundation; +using CoreMedia; +using CoreVideo; +using Foundation; +using Stride.Audio; +using Stride.Core; +using Stride.Graphics; +using Stride.Media; + +namespace Stride.Video.Backends; + +/// +/// Video backend for iOS and macOS using AVFoundation. AVAssetReader does the demux and dispatches +/// to VideoToolbox under the hood for hardware decode; we pull s on +/// demand and import the resulting 's IOSurface as a Vulkan VkImage +/// via VK_EXT_metal_objects, then shader-blit to the user's RGBA target. Zero-copy GPU path. +/// +internal sealed class AVFoundationVideoBackend : VideoBackend +{ + // Stride bundles compiled assets as offset+length slices inside a single archive file. AVAsset + // wants a file URL pointing to the start of a self-contained container, so we extract the slice + // to a temp file at Initialize time. AVAssetResourceLoader is the right long-term answer but + // adds delegate plumbing we don't need to validate the decode pipeline first. + private string tempFilePath; + private AVAsset asset; + private AVAssetReader reader; + private AVAssetReaderTrackOutput videoOutput; + // Last delivered frame, held until the next one replaces it: the blit reading its IOSurface + // runs after Update, and releasing sooner lets the decoder pool overwrite the surface mid-read. + private CMSampleBuffer presentedSampleBuffer; + private AVAssetTrack videoTrack; + + private int videoWidth; + private int videoHeight; + private TimeSpan frameDuration; + private long adjustedTicksSinceLastFrame; + + // Audio runs on the StreamedBufferSoundSource worker thread; video clock forwards to the synchronizer. + private MediaSynchronizer audioSynchronizer; + private StreamedBufferSound audioSound; + private SoundInstanceStreamedBuffer audioSoundInstance; + private readonly System.Collections.Generic.List audioControllers = new(); + + public AVFoundationVideoBackend(VideoInstance instance) : base(instance) { } + + public override bool UsesHardwareDecode => true; // AVAssetReader uses VideoToolbox + + public override bool Initialize(string url, long startPosition, long length) + { + if (reader != null) + throw new InvalidOperationException("AVFoundationVideoBackend already initialized."); + + try + { + tempFilePath = ExtractAssetSliceToTempFile(url, startPosition, length); + + using var assetUrl = NSUrl.FromFilename(tempFilePath); + asset = AVAsset.FromUrl(assetUrl); + + var videoTracks = asset.TracksWithMediaType(AVMediaTypes.Video.GetConstant()); + if (videoTracks == null || videoTracks.Length == 0) + { + VideoInstance.Logger.Warning("AVFoundationVideoBackend: media has no video track."); + ReleaseMedia(); + return false; + } + videoTrack = videoTracks[0]; + + var size = videoTrack.NaturalSize; + videoWidth = (int)size.Width; + videoHeight = (int)size.Height; + + // NominalFrameRate is 0 for tracks where the muxer didn't write it (rare for H.264). + // Default to 30 fps as a conservative fallback — used only for the per-frame advance + // accumulator, not for sample-buffer timing (which comes from the CMSampleBuffer PTS). + var fps = videoTrack.NominalFrameRate > 0 ? videoTrack.NominalFrameRate : 30f; + frameDuration = TimeSpan.FromSeconds(1.0 / fps); + + Instance.SetDuration(TimeSpan.FromSeconds(asset.Duration.Seconds)); + Instance.AllocateVideoTexture(videoWidth, videoHeight); + + CreateReader(TimeSpan.Zero); + + InitializeAudio(url, startPosition, length); + return true; + } + catch (Exception ex) + { + VideoInstance.Logger.Error($"AVFoundationVideoBackend.Initialize failed: {ex}"); + ReleaseMedia(); + return false; + } + } + + public override void ReleaseMedia() + { + ReleaseAudio(); + + DisposeReader(); + presentedSampleBuffer?.Dispose(); + presentedSampleBuffer = null; + asset?.Dispose(); + asset = null; + videoTrack = null; + + if (tempFilePath != null && File.Exists(tempFilePath)) + { + try { File.Delete(tempFilePath); } + catch { /* best-effort cleanup */ } + } + tempFilePath = null; + } + + public override void Play() + { + if (Instance.PlayRange.Start > Instance.CurrentTime) + { + Instance.SetCurrentTime(Instance.PlayRange.Start); + Instance.Seek(Instance.PlayRange.Start); + } + else if (Instance.PlayState == PlayState.Stopped && reader != null) + { + // Prime the accumulator so the first frame extracts on the very next Update, matching + // the FFmpeg backend's behavior (otherwise the caller waits frameDuration/60Hz_tick + // game frames before any frame surfaces). + adjustedTicksSinceLastFrame = frameDuration.Ticks; + } + audioSynchronizer?.Play(); + } + + public override void Pause() + { + audioSynchronizer?.Pause(); + } + + public override void Stop() + { + // FFmpeg backend pattern: seeks back to the play range start. Keeps the reader alive in + // case Play() resumes; the next Seek/Play just recreates the reader at the new position. + Instance.Seek(Instance.PlayRange.Start); + audioSynchronizer?.Stop(); + } + + public override void Seek(TimeSpan time) + { + // AVAssetReader has no in-place seek — recreate it with a new timeRange. Cheap enough + // for the smoke test's seek-per-capture pattern (~5-50 ms per seek). + CreateReader(time); + adjustedTicksSinceLastFrame = frameDuration.Ticks; // make the first frame extract immediately + audioSynchronizer?.Seek(time); + } + + public override void SetAudioVolume(float volume) + { + if (audioSoundInstance != null) + audioSoundInstance.Volume = volume; + foreach (var controller in audioControllers) + controller.Volume = volume; + } + + public override void UpdatePlayRange() + { + if (audioSynchronizer != null) + audioSynchronizer.PlayRange = Instance.PlayRange; + } + + public override void UpdateLoopRange() + { + if (audioSynchronizer != null) + { + audioSynchronizer.IsLooping = Instance.IsLooping; + audioSynchronizer.LoopRange = Instance.LoopRange; + } + } + + public override void Update(TimeSpan elapsed) + { + audioSynchronizer?.Update(elapsed); + + if (reader == null) + return; + + // Mirror the FFmpeg accumulator semantics: speedFactor=0 when not playing freezes auto-advance, + // but a Seek() that just primed the accumulator still gets to decode this tick (= delivers + // a frame for seek-only captures with no Play()). + var speedFactor = Instance.SpeedFactor; + if (Instance.PlayState != PlayState.Playing) + speedFactor = 0; + + var frameDurationTicks = frameDuration.Ticks; + adjustedTicksSinceLastFrame += (long)(elapsed.Ticks * speedFactor); + if (adjustedTicksSinceLastFrame < frameDurationTicks) + return; + + var frameCount = (int)(adjustedTicksSinceLastFrame / frameDurationTicks); + if (frameCount == 0) + return; + + // Skip-ahead heuristic copied from FFmpeg backend: if we're more than 4 frames behind, + // seek instead of stepping (cheaper than decoding the gap, and tests don't depend on + // intermediate frames anyway). + if (frameCount > 4) + { + Instance.Seek(Instance.CurrentTime + TimeSpan.FromTicks(frameDurationTicks * frameCount)); + frameCount = 1; + } + + CMSampleBuffer sampleBuffer = null; + for (int i = 0; i < frameCount; i++) + { + sampleBuffer?.Dispose(); + sampleBuffer = videoOutput.CopyNextSampleBuffer(); + if (sampleBuffer == null) + break; // EOF or error + } + + if (sampleBuffer == null) + { + // End of media. Reader.Status will be Completed on clean EOF, Failed otherwise. + if (Instance.IsLooping && Instance.LoopRange.IsValid()) + { + Instance.SetCurrentTime(Instance.LoopRange.Start); + Instance.Seek(Instance.LoopRange.Start); + } + else + { + Instance.Stop(); + } + return; + } + + var pts = sampleBuffer.PresentationTimeStamp; + Instance.SetCurrentTime(TimeSpan.FromSeconds(pts.Seconds)); + + using (var imageBuffer = sampleBuffer.GetImageBuffer()) + { + if (imageBuffer is CVPixelBuffer pixelBuffer) + { + UploadFrameToTarget(pixelBuffer); + } + } + adjustedTicksSinceLastFrame %= frameDurationTicks; + + presentedSampleBuffer?.Dispose(); + presentedSampleBuffer = sampleBuffer; + } + + private void CreateReader(TimeSpan startTime) + { + DisposeReader(); + + reader = AVAssetReader.FromAsset(asset, out var error); + if (error != null) + throw new InvalidOperationException($"AVAssetReader create failed: {error.LocalizedDescription}"); + + if (startTime > TimeSpan.Zero) + { + // Timescale 600 covers all common video framerates exactly (24/25/30/50/60 fps all + // hit integer tick counts at 600). + var startCMTime = new CMTime((long)(startTime.TotalSeconds * 600), 600); + reader.TimeRange = new CMTimeRange { Start = startCMTime, Duration = CMTime.PositiveInfinity }; + } + + // Ask the reader to deliver decoded BGRA pixel buffers backed by IOSurface. BGRA is the + // VideoToolbox native output format for H.264/HEVC SDR content — picking it avoids an + // internal NV12→RGB conversion. The IOSurface is imported as a VkImage in UploadFrameToTarget. + var settings = new NSMutableDictionary + { + [CVPixelBuffer.PixelFormatTypeKey] = NSNumber.FromInt32((int)CVPixelFormatType.CV32BGRA), + [CVPixelBuffer.IOSurfacePropertiesKey] = new NSDictionary(), + }; + videoOutput = new AVAssetReaderTrackOutput(videoTrack, settings) + { + // The default of true caches output buffers via the track output's internal pool which + // is what we want — keeps each frame available until we Dispose() it. + AlwaysCopiesSampleData = false, + }; + reader.AddOutput(videoOutput); + if (!reader.StartReading()) + { + var err = reader.Error; + throw new InvalidOperationException( + $"AVAssetReader.StartReading failed: {(err != null ? err.LocalizedDescription : "unknown")}"); + } + } + + private void DisposeReader() + { + videoOutput?.Dispose(); + videoOutput = null; + reader?.Dispose(); + reader = null; + } + + private void UploadFrameToTarget(CVPixelBuffer pixelBuffer) + { + var target = Instance.VideoComponent.Target; + if (target == null) + return; + + // Zero-copy: import the IOSurface that backs the CVPixelBuffer as a BGRA8 VkImage, then + // shader-blit to the RGBA target. MoltenVK wraps the IOSurface storage directly; the only + // CPU work is allocating the VkImage/VkImageView (~tens of µs). + using var ioSurface = pixelBuffer.GetIOSurface(); + if (ioSurface == null) + return; + + var width = (int)pixelBuffer.Width; + var height = (int)pixelBuffer.Height; + var graphicsDevice = Instance.GraphicsDevice; + var graphicsContext = Instance.Services.GetSafeServiceAs(); + + // Swap the user's target with VideoTexture's internal RT and allocate the per-mip views. + // No-op if already done; matches the MediaEngine backend ordering. + Instance.VideoTexture.SetTargetContentToVideoStream(target); + + Texture imported = null; + try + { + imported = Texture.NewFromIOSurface(graphicsDevice, ioSurface.Handle.Handle, width, height); + Instance.VideoTexture.CopyTextureToTopLevelMipmap(graphicsContext, imported); + Instance.VideoTexture.GenerateMipMaps(graphicsContext); + Instance.NotifyFramePresented(); + } + finally + { + imported?.Dispose(); + } + } + + private void InitializeAudio(string url, long startPosition, long length) + { + var audioTracks = asset.TracksWithMediaType(AVMediaTypes.Audio.GetConstant()); + if (audioTracks == null || audioTracks.Length == 0) + return; + var videoComponent = Instance.VideoComponent; + if (!videoComponent.PlayAudio) + return; + + var audioEngine = Instance.Services.GetService()?.AudioEngine + ?? throw new Exception("VideoInstance AVFoundation failed to get the AudioEngine"); + + audioSynchronizer = new MediaSynchronizer(); + audioSynchronizer.PlayRange = Instance.PlayRange; + audioSynchronizer.IsLooping = Instance.IsLooping; + audioSynchronizer.LoopRange = Instance.LoopRange; + + var isSpatialized = videoComponent.AudioEmitters.Any(x => x != null); + audioSound = new StreamedBufferSound(audioEngine, audioSynchronizer, url, startPosition, length, isSpatialized); + audioSynchronizer.RegisterExtractor(audioSound); + + if (isSpatialized) + { + if (audioSound.GetCountChannels() == 1) + { + foreach (var emitter in videoComponent.AudioEmitters) + { + if (emitter == null) + continue; + var controller = emitter.AttachSound(audioSound); + audioSynchronizer.RegisterPlayer(controller); + audioControllers.Add(controller); + } + } + else + { + VideoInstance.Logger.Error("Stereo audio tracks cannot be played through audio emitters. The track needs to be mono."); + audioSound.Dispose(); + audioSound = null; + audioSynchronizer = null; + return; + } + } + else + { + audioSoundInstance = (SoundInstanceStreamedBuffer)audioSound.CreateInstance(); + audioSynchronizer.RegisterPlayer(audioSoundInstance); + } + } + + private void ReleaseAudio() + { + audioSoundInstance?.Stop(); + audioSoundInstance?.Dispose(); + audioSoundInstance = null; + + if (audioSound != null) + { + foreach (var emitter in Instance.VideoComponent.AudioEmitters) + emitter?.DetachSound(audioSound); + audioSound.Dispose(); + audioSound = null; + } + audioControllers.Clear(); + audioSynchronizer = null; + } + + private static string ExtractAssetSliceToTempFile(string sourcePath, long startPosition, long length) + { + var temp = Path.Combine(Path.GetTempPath(), $"stride-video-{Guid.NewGuid():N}.mp4"); + using var src = File.OpenRead(sourcePath); + using var dst = File.Create(temp); + + src.Seek(startPosition, SeekOrigin.Begin); + var buffer = new byte[64 * 1024]; + long remaining = length; + while (remaining > 0) + { + int toRead = (int)Math.Min(buffer.Length, remaining); + int read = src.Read(buffer, 0, toRead); + if (read <= 0) + break; + dst.Write(buffer, 0, read); + remaining -= read; + } + return temp; + } +} +#endif diff --git a/sources/engine/Stride.Video/Backends/AVFoundationVideoBackendFactory.cs b/sources/engine/Stride.Video/Backends/AVFoundationVideoBackendFactory.cs new file mode 100644 index 000000000..493894c7c --- /dev/null +++ b/sources/engine/Stride.Video/Backends/AVFoundationVideoBackendFactory.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation and Contributors (https://dotnetfoundation.org/ & https://stride3d.net) +// Distributed under the MIT license. See the LICENSE.md file in the project root for more information. + +#if STRIDE_VIDEO_AVFOUNDATION +using Stride.Core; +using Stride.Graphics; + +namespace Stride.Video.Backends; + +public sealed class AVFoundationVideoBackendFactory : VideoBackendFactory +{ + public override string Name => "AVFoundation"; + public override int Priority => 200; // platform-native HW decode on iOS/macOS + public override bool IsSupported(GraphicsDevice device) => true; + + public override VideoBackend CreateBackend(VideoInstance instance) => new AVFoundationVideoBackend(instance); +} + +internal static class AVFoundationVideoBackendModule +{ + [ModuleInitializer] + public static void Initialize() => VideoBackendRegistry.Register(new AVFoundationVideoBackendFactory()); +} +#endif diff --git a/sources/engine/Stride.Video/Stride.Video.csproj b/sources/engine/Stride.Video/Stride.Video.csproj index 706143b02..2e89bf087 100644 --- a/sources/engine/Stride.Video/Stride.Video.csproj +++ b/sources/engine/Stride.Video/Stride.Video.csproj @@ -3,6 +3,9 @@ true true + + true true true true @@ -14,6 +17,9 @@ STRIDE_VIDEO_MEDIACODEC;$(DefineConstants) + + STRIDE_VIDEO_AVFOUNDATION;$(DefineConstants) + Properties\SharedAssemblyInfo.cs diff --git a/sources/engine/Stride.Video/VideoTexture.cs b/sources/engine/Stride.Video/VideoTexture.cs index c46c42e89..7899ba1c9 100644 --- a/sources/engine/Stride.Video/VideoTexture.cs +++ b/sources/engine/Stride.Video/VideoTexture.cs @@ -136,6 +136,23 @@ namespace Stride.Video //} } + /// Copies a regular sampleable source texture into the top-level mipmap via SpriteEffectExtTextureRegular. + /// Used by backends whose decoder output is an ordinary VkImage (e.g. AVFoundation IOSurface import). + public void CopyTextureToTopLevelMipmap(GraphicsContext context, Texture sourceTexture) + { + var previousDepthStencilBuffer = context.CommandList.DepthStencilBuffer; + var previousRenderTarget = context.CommandList.RenderTarget; + + CopyTexture(context, + effectTexture2DCopy, + sourceTexture, + renderTargetMipMaps[0], + SpriteEffectExtTextureRegularKeys.TextureRegular, + SpriteEffectExtTextureRegularKeys.MipLevel); + + context.CommandList.SetRenderTargetAndViewport(previousDepthStencilBuffer, previousRenderTarget); + } + public void GenerateMipMaps(GraphicsContext graphicsContext) { if (renderTargetMipMaps.Count < 2) // avoid the change of render target when there is no mipmaps to generate. diff --git a/sources/native/Stride.Native.targets b/sources/native/Stride.Native.targets index b646d4a28..fa436b1b2 100644 --- a/sources/native/Stride.Native.targets +++ b/sources/native/Stride.Native.targets @@ -90,6 +90,9 @@ + + + @@ -315,13 +318,23 @@ - + under deps/NativePath/dotnet/osx-arm64/ (iOS objects refuse to link into macOS dylibs). + Fires for net10.0 (with -p:StridePlatforms=macOS opt-in) and net10.0-macos (TFM is + definitionally macOS, no opt-in gate). Net effect: each TFM lays its own dylib down + in bin//runtimes/osx-arm64/native/, so ProjectReference consumers building for + net10.0-macos pick it up directly. --> + - + + diff --git a/sources/sdk/Stride.Build.Sdk/Sdk/Stride.Platform.targets b/sources/sdk/Stride.Build.Sdk/Sdk/Stride.Platform.targets index e40bf416a..a924f35bc 100644 --- a/sources/sdk/Stride.Build.Sdk/Sdk/Stride.Platform.targets +++ b/sources/sdk/Stride.Build.Sdk/Sdk/Stride.Platform.targets @@ -33,6 +33,13 @@ STRIDE_PLATFORM_MONO_MOBILE;STRIDE_PLATFORM_IOS + + + + + STRIDE_PLATFORM_MACOS + +