Audio bugfixes and improvements.

src/Discord.Net.Core/Audio/IAudioClient.cs

@@ -22,31 +22,13 @@ namespace Discord.Audio
 
         Task StopAsync();
 
-        /// <summary>
+        /// <summary>Creates a new outgoing stream accepting Opus-encoded data.</summary>
-        /// Creates a new outgoing stream accepting Opus-encoded data.
+        AudioOutStream CreateOpusStream(int bufferMillis = 1000);
-        /// </summary>
+        /// <summary>Creates a new outgoing stream accepting Opus-encoded data. This is a direct stream with no internal timer.</summary>
-        /// <param name="samplesPerFrame">Samples per frame. Must be 120, 240, 480, 960, 1920 or 2880, representing 2.5, 5, 10, 20, 40 or 60 milliseconds respectively.</param>
+        AudioOutStream CreateDirectOpusStream();
-        /// <returns></returns>
+        /// <summary>Creates a new outgoing stream accepting PCM (raw) data.</summary>
-        AudioOutStream CreateOpusStream(int samplesPerFrame, int bufferMillis = 1000);
+        AudioOutStream CreatePCMStream(AudioApplication application, int? bitrate = null, int bufferMillis = 1000);
-        /// <summary>
+        /// <summary>Creates a new direct outgoing stream accepting PCM (raw) data. This is a direct stream with no internal timer.</summary>
-        /// Creates a new outgoing stream accepting Opus-encoded data. This is a direct stream with no internal timer.
+        AudioOutStream CreateDirectPCMStream(AudioApplication application, int? bitrate = null);
-        /// </summary>
-        /// <param name="samplesPerFrame">Samples per frame. Must be 120, 240, 480, 960, 1920 or 2880, representing 2.5, 5, 10, 20, 40 or 60 milliseconds respectively.</param>
-        /// <returns></returns>
-        AudioOutStream CreateDirectOpusStream(int samplesPerFrame);
-        /// <summary>
-        /// Creates a new outgoing stream accepting PCM (raw) data.
-        /// </summary>
-        /// <param name="samplesPerFrame">Samples per frame. Must be 120, 240, 480, 960, 1920 or 2880, representing 2.5, 5, 10, 20, 40 or 60 milliseconds respectively.</param>
-        /// <param name="bitrate"></param>
-        /// <returns></returns>
-        AudioOutStream CreatePCMStream(AudioApplication application, int samplesPerFrame, int channels = 2, int? bitrate = null, int bufferMillis = 1000);
-        /// <summary>
-        /// Creates a new direct outgoing stream accepting PCM (raw) data. This is a direct stream with no internal timer.
-        /// </summary>
-        /// <param name="samplesPerFrame">Samples per frame. Must be 120, 240, 480, 960, 1920 or 2880, representing 2.5, 5, 10, 20, 40 or 60 milliseconds respectively.</param>
-        /// <param name="bitrate"></param>
-        /// <returns></returns>
-        AudioOutStream CreateDirectPCMStream(AudioApplication application, int samplesPerFrame, int channels = 2, int? bitrate = null);
     }
 }

src/Discord.Net.WebSocket/Audio/AudioClient.cs

@@ -139,43 +139,33 @@ namespace Discord.Audio
             await Discord.ApiClient.SendVoiceStateUpdateAsync(Guild.Id, null, false, false).ConfigureAwait(false);
         }
 
-        public AudioOutStream CreateOpusStream(int samplesPerFrame, int bufferMillis)
+        public AudioOutStream CreateOpusStream(int bufferMillis)
         {
-            CheckSamplesPerFrame(samplesPerFrame);
             var outputStream = new OutputStream(ApiClient);
             var sodiumEncrypter = new SodiumEncryptStream( outputStream, this);
-            var rtpWriter = new RTPWriteStream(sodiumEncrypter, samplesPerFrame, _ssrc);
+            var rtpWriter = new RTPWriteStream(sodiumEncrypter, _ssrc);
-            return new BufferedWriteStream(rtpWriter, this, samplesPerFrame, bufferMillis, _connection.CancelToken, _audioLogger);
+            return new BufferedWriteStream(rtpWriter, this, bufferMillis, _connection.CancelToken, _audioLogger);
         }
-        public AudioOutStream CreateDirectOpusStream(int samplesPerFrame)
+        public AudioOutStream CreateDirectOpusStream()
         {
-            CheckSamplesPerFrame(samplesPerFrame);
             var outputStream = new OutputStream(ApiClient);
             var sodiumEncrypter = new SodiumEncryptStream(outputStream, this);
-            return new RTPWriteStream(sodiumEncrypter, samplesPerFrame, _ssrc);
+            return new RTPWriteStream(sodiumEncrypter, _ssrc);
         }
-        public AudioOutStream CreatePCMStream(AudioApplication application, int samplesPerFrame, int channels, int? bitrate, int bufferMillis)
+        public AudioOutStream CreatePCMStream(AudioApplication application, int? bitrate, int bufferMillis)
         {
-            CheckSamplesPerFrame(samplesPerFrame);
             var outputStream = new OutputStream(ApiClient);
             var sodiumEncrypter = new SodiumEncryptStream(outputStream, this);
-            var rtpWriter = new RTPWriteStream(sodiumEncrypter, samplesPerFrame, _ssrc);
+            var rtpWriter = new RTPWriteStream(sodiumEncrypter, _ssrc);
-            var bufferedStream = new BufferedWriteStream(rtpWriter, this, samplesPerFrame, bufferMillis, _connection.CancelToken, _audioLogger);
+            var bufferedStream = new BufferedWriteStream(rtpWriter, this, bufferMillis, _connection.CancelToken, _audioLogger);
-            return new OpusEncodeStream(bufferedStream, channels, samplesPerFrame, bitrate ?? (96 * 1024), application);
+            return new OpusEncodeStream(bufferedStream, bitrate ?? (96 * 1024), application);
         }
-        public AudioOutStream CreateDirectPCMStream(AudioApplication application, int samplesPerFrame, int channels, int? bitrate)
+        public AudioOutStream CreateDirectPCMStream(AudioApplication application, int? bitrate)
         {
-            CheckSamplesPerFrame(samplesPerFrame);
             var outputStream = new OutputStream(ApiClient);
             var sodiumEncrypter = new SodiumEncryptStream(outputStream, this);
-            var rtpWriter = new RTPWriteStream(sodiumEncrypter, samplesPerFrame, _ssrc);
+            var rtpWriter = new RTPWriteStream(sodiumEncrypter, _ssrc);
-            return new OpusEncodeStream(rtpWriter, channels, samplesPerFrame, bitrate ?? (96 * 1024), application);
+            return new OpusEncodeStream(rtpWriter, bitrate ?? (96 * 1024), application);
-        }
-        private void CheckSamplesPerFrame(int samplesPerFrame)
-        {
-            if (samplesPerFrame != 120 && samplesPerFrame != 240 && samplesPerFrame != 480 &&
-                samplesPerFrame != 960 && samplesPerFrame != 1920 && samplesPerFrame != 2880)
-                throw new ArgumentException("Value must be 120, 240, 480, 960, 1920 or 2880", nameof(samplesPerFrame));
         }
 
         internal async Task CreateInputStreamAsync(ulong userId)

src/Discord.Net.WebSocket/Audio/Opus/OpusConverter.cs

@@ -6,37 +6,22 @@ namespace Discord.Audio
     {
         protected IntPtr _ptr;
 
-        /// <summary> Gets the bit rate of this converter. </summary>
+        public const int SamplingRate = 48000;
-        public const int BitsPerSample = sizeof(short) * 8;
+        public const int Channels = 2;
-        /// <summary> Gets the bytes per sample. </summary>
+        public const int FrameMillis = 20;
-        public const int SampleSize = (BitsPerSample / 8) * MaxChannels;
-        /// <summary> Gets the maximum amount of channels this encoder supports. </summary>
-        public const int MaxChannels = 2;
 
-        /// <summary> Gets the input sampling rate of this converter. </summary>
+        public const int SampleBytes = sizeof(short) * Channels;
-        public int SamplingRate { get; }
-        /// <summary> Gets the number of samples per second for this stream. </summary>
-        public int Channels { get; }
 
-        protected OpusConverter(int samplingRate, int channels)
+        public const int FrameSamples = SamplingRate / 1000 * FrameMillis;
-        {
+        public const int FrameSamplesPerChannel = SamplingRate / 1000 * FrameMillis;
-            if (samplingRate != 8000 && samplingRate != 12000 &&
+        public const int FrameBytes = FrameSamples * SampleBytes;
-                samplingRate != 16000 && samplingRate != 24000 &&
-                samplingRate != 48000)
-                throw new ArgumentOutOfRangeException(nameof(samplingRate));
-            if (channels != 1 && channels != 2)
-                throw new ArgumentOutOfRangeException(nameof(channels));
         
-            SamplingRate = samplingRate;
+        protected bool _isDisposed = false;
-            Channels = channels;
-        }
-        
-        private bool disposedValue = false; // To detect redundant calls
 
         protected virtual void Dispose(bool disposing)
         {
-            if (!disposedValue)
+            if (!_isDisposed)
-                disposedValue = true;
+                _isDisposed = true;
         }
         ~OpusConverter()
         {
@@ -47,5 +32,16 @@ namespace Discord.Audio
             Dispose(true);
             GC.SuppressFinalize(this);
         }
+        
+        protected static void CheckError(int result)
+        {
+            if (result < 0)
+                throw new Exception($"Opus Error: {(OpusError)result}");
+        }
+        protected static void CheckError(OpusError error)
+        {
+            if ((int)error < 0)
+                throw new Exception($"Opus Error: {error}");
+        }
     }
 }

src/Discord.Net.WebSocket/Audio/Opus/OpusDecoder.cs

@@ -14,37 +14,29 @@ namespace Discord.Audio
         [DllImport("opus", EntryPoint = "opus_decoder_ctl", CallingConvention = CallingConvention.Cdecl)]
         private static extern int DecoderCtl(IntPtr st, OpusCtl request, int value);
 
-        public OpusDecoder(int samplingRate, int channels)
+        public OpusDecoder()
-            : base(samplingRate, channels)
         {
-            OpusError error;
+            _ptr = CreateDecoder(SamplingRate, Channels, out var error);
-            _ptr = CreateDecoder(samplingRate, channels, out error);
+            CheckError(error);
-            if (error != OpusError.OK)
-                throw new Exception($"Opus Error: {error}");
         }
         
-        /// <summary> Produces PCM samples from Opus-encoded audio. </summary>
-        /// <param name="input">PCM samples to decode.</param>
-        /// <param name="inputOffset">Offset of the frame in input.</param>
-        /// <param name="output">Buffer to store the decoded frame.</param>
         public unsafe int DecodeFrame(byte[] input, int inputOffset, int inputCount, byte[] output, int outputOffset)
         {
             int result = 0;
             fixed (byte* inPtr = input)
             fixed (byte* outPtr = output)
-                result = Decode(_ptr, inPtr + inputOffset, inputCount, outPtr + outputOffset, (output.Length - outputOffset) / SampleSize, 0); //TODO: Enable FEC
+                result = Decode(_ptr, inPtr + inputOffset, inputCount, outPtr + outputOffset, FrameBytes / SampleBytes, 1);
-
+            CheckError(result);
-            if (result < 0)
+            return FrameBytes;
-                throw new Exception($"Opus Error: {(OpusError)result}");
-            return result * SampleSize;
         }
 
         protected override void Dispose(bool disposing)
         {
-            if (_ptr != IntPtr.Zero)
+            if (!_isDisposed)
             {
-                DestroyDecoder(_ptr);
+                if (_ptr != IntPtr.Zero)
-                _ptr = IntPtr.Zero;
+                    DestroyDecoder(_ptr);
+                base.Dispose(disposing);
             }
         }
     }

src/Discord.Net.WebSocket/Audio/Opus/OpusEncoder.cs

@@ -12,14 +12,12 @@ namespace Discord.Audio
         [DllImport("opus", EntryPoint = "opus_encode", CallingConvention = CallingConvention.Cdecl)]
         private static extern int Encode(IntPtr st, byte* pcm, int frame_size, byte* data, int max_data_bytes);
         [DllImport("opus", EntryPoint = "opus_encoder_ctl", CallingConvention = CallingConvention.Cdecl)]
-        private static extern int EncoderCtl(IntPtr st, OpusCtl request, int value);
+        private static extern OpusError EncoderCtl(IntPtr st, OpusCtl request, int value);
         
-        /// <summary> Gets the coding mode of the encoder. </summary>
         public AudioApplication Application { get; }
         public int BitRate { get;}
 
-        public OpusEncoder(int samplingRate, int channels, int bitrate, AudioApplication application)
+        public OpusEncoder(int bitrate, AudioApplication application)
-            : base(samplingRate, channels)
         {
             if (bitrate < 1 || bitrate > DiscordVoiceAPIClient.MaxBitrate)
                 throw new ArgumentOutOfRangeException(nameof(bitrate));
@@ -47,57 +45,31 @@ namespace Discord.Audio
                     throw new ArgumentOutOfRangeException(nameof(application));
             }
 
-            OpusError error;
+            _ptr = CreateEncoder(SamplingRate, Channels, (int)opusApplication, out var error);
-            _ptr = CreateEncoder(samplingRate, channels, (int)opusApplication, out error);
+            CheckError(error);
-            if (error != OpusError.OK)
+            CheckError(EncoderCtl(_ptr, OpusCtl.SetSignal, (int)opusSignal));
-                throw new Exception($"Opus Error: {error}");
+            CheckError(EncoderCtl(_ptr, OpusCtl.SetPacketLossPercent, 30)); //%
-
+            CheckError(EncoderCtl(_ptr, OpusCtl.SetInbandFEC, 1)); //True
-            var result = EncoderCtl(_ptr, OpusCtl.SetSignal, (int)opusSignal);
+            CheckError(EncoderCtl(_ptr, OpusCtl.SetBitrate, bitrate));
-            if (result < 0)
-                throw new Exception($"Opus Error: {(OpusError)result}");
-
-            result = EncoderCtl(_ptr, OpusCtl.SetPacketLossPercent, 30); //%%
-            if (result < 0)
-                throw new Exception($"Opus Error: {(OpusError)result}");
-
-            result = EncoderCtl(_ptr, OpusCtl.SetInbandFEC, 1); //True
-            if (result < 0)
-                throw new Exception($"Opus Error: {(OpusError)result}");
-
-            result = EncoderCtl(_ptr, OpusCtl.SetBitrate, bitrate);
-            if (result < 0)
-                throw new Exception($"Opus Error: {(OpusError)result}");
-
-            /*if (application == AudioApplication.Music)
-            {
-                result = EncoderCtl(_ptr, OpusCtl.SetBandwidth, 1105);
-                if (result < 0)
-                    throw new Exception($"Opus Error: {(OpusError)result}");
-            }*/
         }
 
-        /// <summary> Produces Opus encoded audio from PCM samples. </summary>
+        public unsafe int EncodeFrame(byte[] input, int inputOffset, byte[] output, int outputOffset)
-        /// <param name="input">PCM samples to encode.</param>
-        /// <param name="output">Buffer to store the encoded frame.</param>
-        /// <returns>Length of the frame contained in outputBuffer.</returns>
-        public unsafe int EncodeFrame(byte[] input, int inputOffset, int inputCount, byte[] output, int outputOffset)
         {
             int result = 0;
             fixed (byte* inPtr = input)
             fixed (byte* outPtr = output)
-                result = Encode(_ptr, inPtr + inputOffset, inputCount / SampleSize, outPtr + outputOffset, output.Length - outputOffset);
+                result = Encode(_ptr, inPtr + inputOffset, FrameSamplesPerChannel, outPtr + outputOffset, output.Length - outputOffset);
-
+            CheckError(result);
-            if (result < 0)
-                throw new Exception($"Opus Error: {(OpusError)result}");
             return result;
         }
 
         protected override void Dispose(bool disposing)
         {
-            if (_ptr != IntPtr.Zero)
+            if (!_isDisposed)
             {
-                DestroyEncoder(_ptr);
+                if (_ptr != IntPtr.Zero)
-                _ptr = IntPtr.Zero;
+                    DestroyEncoder(_ptr);
+                base.Dispose(disposing);
             }
         }
     }

src/Discord.Net.WebSocket/Audio/Streams/BufferedWriteStream.cs

@@ -38,14 +38,14 @@ namespace Discord.Audio.Streams
         private bool _isPreloaded;
         private int _silenceFrames;
 
-        public BufferedWriteStream(AudioStream next, IAudioClient client, int samplesPerFrame, int bufferMillis, CancellationToken cancelToken, int maxFrameSize = 1500)
+        public BufferedWriteStream(AudioStream next, IAudioClient client, int bufferMillis, CancellationToken cancelToken, int maxFrameSize = 1500)
-            : this(next, client as AudioClient, samplesPerFrame, bufferMillis, cancelToken, null, maxFrameSize) { }
+            : this(next, client as AudioClient, bufferMillis, cancelToken, null, maxFrameSize) { }
-        internal BufferedWriteStream(AudioStream next, AudioClient client, int samplesPerFrame, int bufferMillis, CancellationToken cancelToken, Logger logger, int maxFrameSize = 1500)
+        internal BufferedWriteStream(AudioStream next, AudioClient client, int bufferMillis, CancellationToken cancelToken, Logger logger, int maxFrameSize = 1500)
         {
             //maxFrameSize = 1275 was too limiting at 128kbps,2ch,60ms
             _next = next;
             _client = client;
-            _ticksPerFrame = samplesPerFrame / 48;
+            _ticksPerFrame = OpusEncoder.FrameSamples / 48;
             _logger = logger;
             _queueLength = (bufferMillis + (_ticksPerFrame - 1)) / _ticksPerFrame; //Round up
 

src/Discord.Net.WebSocket/Audio/Streams/OpusDecodeStream.cs

@@ -9,14 +9,14 @@ namespace Discord.Audio.Streams
         public const int SampleRate = OpusEncodeStream.SampleRate;
 
         private readonly AudioStream _next;
-        private readonly byte[] _buffer;
         private readonly OpusDecoder _decoder;
+        private readonly byte[] _buffer;
 
-        public OpusDecodeStream(AudioStream next, int channels = OpusConverter.MaxChannels, int bufferSize = 5760 * 2 * sizeof(short))
+        public OpusDecodeStream(AudioStream next)
         {
             _next = next;
-            _buffer = new byte[bufferSize];
+            _buffer = new byte[OpusConverter.FrameBytes];
-            _decoder = new OpusDecoder(SampleRate, channels);
+            _decoder = new OpusDecoder();
         }
 
         public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)

src/Discord.Net.WebSocket/Audio/Streams/OpusEncodeStream.cs

@@ -12,18 +12,13 @@ namespace Discord.Audio.Streams
         private readonly AudioStream _next;
         private readonly OpusEncoder _encoder;
         private readonly byte[] _buffer;
-
-        private int _frameSize;
-        private byte[] _partialFrameBuffer;
         private int _partialFramePos;
 
-        public OpusEncodeStream(AudioStream next, int channels, int samplesPerFrame, int bitrate, AudioApplication application, int bufferSize = 4000)
+        public OpusEncodeStream(AudioStream next, int bitrate, AudioApplication application)
         {
             _next = next;
-            _encoder = new OpusEncoder(SampleRate, channels, bitrate, application);
+            _encoder = new OpusEncoder(bitrate, application);
-            _frameSize = samplesPerFrame * channels * 2;
+            _buffer = new byte[OpusConverter.FrameBytes];
-            _buffer = new byte[bufferSize];
-            _partialFrameBuffer = new byte[_frameSize];
         }
 
         public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
@@ -31,20 +26,31 @@ namespace Discord.Audio.Streams
             //Assume threadsafe
             while (count > 0)
             {
-                if (_partialFramePos + count >= _frameSize)
+                if (_partialFramePos == 0 && count >= OpusConverter.FrameBytes)
                 {
-                    int partialSize = _frameSize - _partialFramePos;
+                    //We have enough data and no partial frames. Pass the buffer directly to the encoder
-                    Buffer.BlockCopy(buffer, offset, _partialFrameBuffer, _partialFramePos, partialSize);
+                    int encFrameSize = _encoder.EncodeFrame(buffer, offset, _buffer, 0);
+                    await _next.WriteAsync(_buffer, 0, encFrameSize, cancellationToken).ConfigureAwait(false);
+
+                    offset += OpusConverter.FrameBytes;
+                    count -= OpusConverter.FrameBytes;
+                }
+                else if (_partialFramePos + count >= OpusConverter.FrameBytes)
+                {
+                    //We have enough data to complete a previous partial frame.
+                    int partialSize = OpusConverter.FrameBytes - _partialFramePos;
+                    Buffer.BlockCopy(buffer, offset, _buffer, _partialFramePos, partialSize);
+                    int encFrameSize = _encoder.EncodeFrame(_buffer, 0, _buffer, 0);
+                    await _next.WriteAsync(_buffer, 0, encFrameSize, cancellationToken).ConfigureAwait(false);
+
                     offset += partialSize;
                     count -= partialSize;
                     _partialFramePos = 0;
-
-                    int encFrameSize = _encoder.EncodeFrame(_partialFrameBuffer, 0, _frameSize, _buffer, 0);
-                    await _next.WriteAsync(_buffer, 0, encFrameSize, cancellationToken).ConfigureAwait(false);
                 }
                 else
                 {
-                    Buffer.BlockCopy(buffer, offset, _partialFrameBuffer, _partialFramePos, count);
+                    //Not enough data to build a complete frame, store this part for later
+                    Buffer.BlockCopy(buffer, offset, _buffer, _partialFramePos, count);
                     _partialFramePos += count;
                     break;
                 }

src/Discord.Net.WebSocket/Audio/Streams/RTPWriteStream.cs

@@ -9,15 +9,12 @@ namespace Discord.Audio.Streams
     {
         private readonly AudioStream _next;
         private readonly byte[] _header;
-        private int _samplesPerFrame;
+        protected readonly byte[] _buffer;
         private uint _ssrc, _timestamp = 0;
 
-        protected readonly byte[] _buffer;
+        public RTPWriteStream(AudioStream next, uint ssrc, int bufferSize = 4000)
-
-        public RTPWriteStream(AudioStream next, int samplesPerFrame, uint ssrc, int bufferSize = 4000)
         {
             _next = next;
-            _samplesPerFrame = samplesPerFrame;
             _ssrc = ssrc;
             _buffer = new byte[bufferSize];
             _header = new byte[24];
@@ -38,7 +35,7 @@ namespace Discord.Audio.Streams
                 if (_header[3]++ == byte.MaxValue)
                     _header[2]++;
 
-                _timestamp += (uint)_samplesPerFrame;
+                _timestamp += (uint)OpusEncoder.FrameSamples;
                 _header[4] = (byte)(_timestamp >> 24);
                 _header[5] = (byte)(_timestamp >> 16);
                 _header[6] = (byte)(_timestamp >> 8);