Native AudioDecoder and VideoDecoder (MonoGame#8510)

This is a follow-up to MonoGame#8309 and part of the work on MonoGame#8194. This implements two new native APIs `AudioDecoder` and `VideoDecoder`. These native APIs live under the `Video` and `Song` classes. The design aims to let the native code handle the work of detecting the media format, selecting the correct decoder, process the media data, and return frames to C#. With `AudioDecoder` this means C# keeps a thread and pumps signed 16bit PCM data to a native `Voice`. With `VideoDecoder` a C# thread gathers `Texture`s and calls an `AudioDecoder` to process audio. There is no public implementation of audio or video decoders yet. The only implementation currently is for a native audio format of Nintendo Switch. The video decoder is totally experimental and untested, but i think it will generally work. I expect it be a good project for someone or should be made into bounties for different audio/video codecs. As always feedback is welcome!
SickheadGames · Oct 6, 2024 · 5e918e9 · 5e918e9
1 parent baac0a9
commit 5e918e9
Show file tree

Hide file tree

Showing 18 changed files with 664 additions and 410 deletions.
diff --git a/MonoGame.Framework/Media/Song.cs b/MonoGame.Framework/Media/Song.cs
@@ -12,8 +12,9 @@ namespace Microsoft.Xna.Framework.Media
     /// </summary>
     public sealed partial class Song : IEquatable<Song>, IDisposable
     {
-        private string _name;
-		private int _playCount = 0;
+        private string _name;
+        private string _filePath;
+        private int _playCount = 0;
         private TimeSpan _duration = TimeSpan.Zero;
         bool disposed;
         /// <summary>
@@ -61,8 +62,9 @@ internal Song(string fileName, int durationMS)
         }
 
 		internal Song(string fileName)
-		{			
-			_name = fileName;
+		{
+            _filePath = fileName;
+            _name = Path.GetFileNameWithoutExtension(fileName);
 
             PlatformInitialize(fileName);
         }
@@ -75,7 +77,7 @@ internal Song(string fileName)
 
         internal string FilePath
 		{
-			get { return _name; }
+			get { return _filePath; }
 		}
 
         /// <summary>
@@ -171,7 +173,7 @@ public override bool Equals(Object obj)
         /// </summary>
         public TimeSpan Duration
         {
-            get { return PlatformGetDuration(); }
+            get { return _duration; }
         }
 
         /// <summary>
@@ -195,7 +197,7 @@ public bool IsRated
         /// </summary>
         public string Name
         {
-            get { return PlatformGetName(); }
+            get { return _name; }
         }
 
         /// <summary>

diff --git a/MonoGame.Framework/Platform/Media/Song.Android.cs b/MonoGame.Framework/Platform/Media/Song.Android.cs
@@ -15,8 +15,6 @@ public sealed partial class Song : IEquatable<Song>, IDisposable
         private Album album;
         private Artist artist;
         private Genre genre;
-        private string name;
-        private TimeSpan duration;
         private TimeSpan position;
         private Android.Net.Uri assetUri;
 
@@ -36,9 +34,11 @@ internal Song(Album album, Artist artist, Genre genre, string name, TimeSpan dur
             this.album = album;
             this.artist = artist;
             this.genre = genre;
-            this.name = name;
-            this.duration = duration;
             this.assetUri = assetUri;
+            _duration = duration;
+
+            if (this.assetUri != null)
+                _name = name;
         }
 
         private void PlatformInitialize(string fileName)
@@ -168,11 +168,6 @@ private Genre PlatformGetGenre()
             return this.genre;
         }
 
-        private TimeSpan PlatformGetDuration()
-        {
-            return this.assetUri != null ? this.duration : _duration;
-        }
-
         private bool PlatformIsProtected()
         {
             return false;
@@ -183,11 +178,6 @@ private bool PlatformIsRated()
             return false;
         }
 
-        private string PlatformGetName()
-        {
-            return this.assetUri != null ? this.name : Path.GetFileNameWithoutExtension(_name);
-        }
-
         private int PlatformGetPlayCount()
         {
             return _playCount;

diff --git a/MonoGame.Framework/Platform/Media/Song.Default.cs b/MonoGame.Framework/Platform/Media/Song.Default.cs
@@ -141,11 +141,6 @@ private Genre PlatformGetGenre()
             return null;
         }
 
-        private TimeSpan PlatformGetDuration()
-        {
-            return _duration;
-        }
-
         private bool PlatformIsProtected()
         {
             return false;

diff --git a/MonoGame.Framework/Platform/Media/Song.NVorbis.cs b/MonoGame.Framework/Platform/Media/Song.NVorbis.cs
@@ -126,11 +126,6 @@ private Genre PlatformGetGenre()
             return null;
         }
 
-        private TimeSpan PlatformGetDuration()
-        {
-            return _duration;
-        }
-
         private bool PlatformIsProtected()
         {
             return false;
@@ -141,11 +136,6 @@ private bool PlatformIsRated()
             return false;
         }
 
-        private string PlatformGetName()
-        {
-            return Path.GetFileNameWithoutExtension(_name);
-        }
-
         private int PlatformGetPlayCount()
         {
             return _playCount;

diff --git a/MonoGame.Framework/Platform/Media/Song.WMS.cs b/MonoGame.Framework/Platform/Media/Song.WMS.cs
@@ -105,11 +105,6 @@ private Genre PlatformGetGenre()
             return null;
         }
 
-        private TimeSpan PlatformGetDuration()
-        {
-            return _duration;
-        }
-
         private bool PlatformIsProtected()
         {
             return false;
@@ -120,11 +115,6 @@ private bool PlatformIsRated()
             return false;
         }
 
-        private string PlatformGetName()
-        {
-            return Path.GetFileNameWithoutExtension(_name);
-        }
-
         private int PlatformGetPlayCount()
         {
             return _playCount;

diff --git a/MonoGame.Framework/Platform/Media/Song.iOS.cs b/MonoGame.Framework/Platform/Media/Song.iOS.cs
@@ -16,8 +16,6 @@ public sealed partial class Song
         private Album album;
         private Artist artist;
         private Genre genre;
-        private string title;
-        private TimeSpan duration;
         #if !TVOS
         private MPMediaItem mediaItem;
         #endif
@@ -40,12 +38,13 @@ internal Song(Album album, Artist artist, Genre genre, string title, TimeSpan du
             this.album = album;
             this.artist = artist;
             this.genre = genre;
-            this.title = title;
-            this.duration = duration;
             #if !TVOS
             this.mediaItem = mediaItem;
             #endif
             this.assetUrl = assetUrl;
+            _duration = duration;
+            if (!string.IsNullOrEmpty(title))
+                _name = title;
         }
 
         private void PlatformInitialize(string fileName)
@@ -193,15 +192,6 @@ private Genre PlatformGetGenre()
             return this.genre;
         }
 
-        private TimeSpan PlatformGetDuration()
-        {
-            #if !TVOS
-            if (this.mediaItem != null)
-                return this.duration;
-            #endif
-            return _duration;
-        }
-
         private bool PlatformIsProtected()
         {
             return false;
@@ -212,11 +202,6 @@ private bool PlatformIsRated()
             return false;
         }
 
-        private string PlatformGetName()
-        {
-            return this.title ?? Path.GetFileNameWithoutExtension(_name);
-        }
-
         private int PlatformGetPlayCount()
         {
             return _playCount;

diff --git a/MonoGame.Framework/Platform/Native/Audio.Interop.cs b/MonoGame.Framework/Platform/Native/Audio.Interop.cs
@@ -140,7 +140,7 @@ public static partial void Buffer_InitializeXact(
     #region Voice
 
     [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_Create", StringMarshalling = StringMarshalling.Utf8)]
-    public static partial MGA_Voice* Voice_Create(MGA_System* system);
+    public static partial MGA_Voice* Voice_Create(MGA_System* system, int sampleRate = 0, int channels = 0);
 
     [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_Destroy", StringMarshalling = StringMarshalling.Utf8)]
     public static partial void Voice_Destroy(MGA_Voice* voice);
@@ -152,7 +152,7 @@ public static partial void Buffer_InitializeXact(
     public static partial void Voice_SetBuffer(MGA_Voice* voice, MGA_Buffer* buffer);
 
     [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_AppendBuffer", StringMarshalling = StringMarshalling.Utf8)]
-    public static partial void Voice_AppendBuffer(MGA_Voice* voice, byte[] buffer, int offset, int count, [MarshalAs(UnmanagedType.U1)] bool clear);
+    public static partial void Voice_AppendBuffer(MGA_Voice* voice, byte* buffer, uint size);
 
     [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_Play", StringMarshalling = StringMarshalling.Utf8)]
     public static partial void Voice_Play(MGA_Voice* voice, [MarshalAs(UnmanagedType.U1)] bool looped);
@@ -169,6 +169,9 @@ public static partial void Buffer_InitializeXact(
     [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_GetState", StringMarshalling = StringMarshalling.Utf8)]
     public static partial SoundState Voice_GetState(MGA_Voice* voice);
 
+    [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_GetPosition", StringMarshalling = StringMarshalling.Utf8)]
+    public static partial ulong Voice_GetPosition(MGA_Voice* voice);
+
     [LibraryImport(MGP.MonoGameNativeDLL, EntryPoint = "MGA_Voice_SetPan", StringMarshalling = StringMarshalling.Utf8)]
     public static partial void Voice_SetPan(MGA_Voice* voice, float pan);
 

diff --git a/MonoGame.Framework/Platform/Native/DynamicSoundEffectInstance.Native.cs b/MonoGame.Framework/Platform/Native/DynamicSoundEffectInstance.Native.cs
@@ -11,7 +11,7 @@ public sealed partial class DynamicSoundEffectInstance : SoundEffectInstance
 {
     private unsafe void PlatformCreate()
     {
-        Voice = MGA.Voice_Create(SoundEffect.System);
+        Voice = MGA.Voice_Create(SoundEffect.System, _sampleRate, (int)_channels);
     }
 
     private unsafe int PlatformGetPendingBufferCount()
@@ -49,7 +49,10 @@ private unsafe void PlatformStop()
     private unsafe void PlatformSubmitBuffer(byte[] buffer, int offset, int count)
     {
         if (Voice != null)
-            MGA.Voice_AppendBuffer(Voice, buffer, offset, count, false);
+        {
+            fixed (byte* ptr = buffer)
+                MGA.Voice_AppendBuffer(Voice, ptr + offset, (uint)count);
+        }
     }
 
     private unsafe void PlatformDispose(bool disposing)