github/Discord.Net
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Audio bugfixes and improvements.

Browse Source
This commit is contained in:
RogueException
2017-04-08 17:12:10 -03:00
parent 424216b793
commit e92cfd20ac
9 changed files with 98 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/Discord.Net.Core/Audio/IAudioClient.cs
View File

@@ -22,31 +22,13 @@ namespace Discord.Audio
Task StopAsync();
/// <summary>
/// Creates a new outgoing stream accepting Opus-encoded data.
/// </summary>
/// <param name="samplesPerFrame">Samples per frame. Must be 120, 240, 480, 960, 1920 or 2880, representing 2.5, 5, 10, 20, 40 or 60 milliseconds respectively.</param>
/// <returns></returns>
AudioOutStream CreateOpusStream(int samplesPerFrame, int bufferMillis = 1000);
/// <summary>
/// Creates a new outgoing stream accepting Opus-encoded data. This is a direct stream with no internal timer.
/// </summary>
/// <param name="samplesPerFrame">Samples per frame. Must be 120, 240, 480, 960, 1920 or 2880, representing 2.5, 5, 10, 20, 40 or 60 milliseconds respectively.</param>
/// <returns></returns>
AudioOutStream CreateDirectOpusStream(int samplesPerFrame);
/// <summary>
/// Creates a new outgoing stream accepting PCM (raw) data.
/// </summary>
/// <param name="samplesPerFrame">Samples per frame. Must be 120, 240, 480, 960, 1920 or 2880, representing 2.5, 5, 10, 20, 40 or 60 milliseconds respectively.</param>
/// <param name="bitrate"></param>
/// <returns></returns>
AudioOutStream CreatePCMStream(AudioApplication application, int samplesPerFrame, int channels = 2, int? bitrate = null, int bufferMillis = 1000);
/// <summary>
/// Creates a new direct outgoing stream accepting PCM (raw) data. This is a direct stream with no internal timer.
/// </summary>
/// <param name="samplesPerFrame">Samples per frame. Must be 120, 240, 480, 960, 1920 or 2880, representing 2.5, 5, 10, 20, 40 or 60 milliseconds respectively.</param>
/// <param name="bitrate"></param>
/// <returns></returns>
AudioOutStream CreateDirectPCMStream(AudioApplication application, int samplesPerFrame, int channels = 2, int? bitrate = null);
/// <summary>Creates a new outgoing stream accepting Opus-encoded data.</summary>
AudioOutStream CreateOpusStream(int bufferMillis = 1000);
/// <summary>Creates a new outgoing stream accepting Opus-encoded data. This is a direct stream with no internal timer.</summary>
AudioOutStream CreateDirectOpusStream();
/// <summary>Creates a new outgoing stream accepting PCM (raw) data.</summary>
AudioOutStream CreatePCMStream(AudioApplication application, int? bitrate = null, int bufferMillis = 1000);
/// <summary>Creates a new direct outgoing stream accepting PCM (raw) data. This is a direct stream with no internal timer.</summary>
AudioOutStream CreateDirectPCMStream(AudioApplication application, int? bitrate = null);
}
}

34
src/Discord.Net.WebSocket/Audio/AudioClient.cs
View File

@@ -139,43 +139,33 @@ namespace Discord.Audio
await Discord.ApiClient.SendVoiceStateUpdateAsync(Guild.Id, null, false, false).ConfigureAwait(false);
}
public AudioOutStream CreateOpusStream(int samplesPerFrame, int bufferMillis)
public AudioOutStream CreateOpusStream(int bufferMillis)
{
CheckSamplesPerFrame(samplesPerFrame);
var outputStream = new OutputStream(ApiClient);
var sodiumEncrypter = new SodiumEncryptStream( outputStream, this);
var rtpWriter = new RTPWriteStream(sodiumEncrypter, samplesPerFrame, _ssrc);
return new BufferedWriteStream(rtpWriter, this, samplesPerFrame, bufferMillis, _connection.CancelToken, _audioLogger);
var rtpWriter = new RTPWriteStream(sodiumEncrypter, _ssrc);
return new BufferedWriteStream(rtpWriter, this, bufferMillis, _connection.CancelToken, _audioLogger);
}
public AudioOutStream CreateDirectOpusStream(int samplesPerFrame)
public AudioOutStream CreateDirectOpusStream()
{
CheckSamplesPerFrame(samplesPerFrame);
var outputStream = new OutputStream(ApiClient);
var sodiumEncrypter = new SodiumEncryptStream(outputStream, this);
return new RTPWriteStream(sodiumEncrypter, samplesPerFrame, _ssrc);
return new RTPWriteStream(sodiumEncrypter, _ssrc);
}
public AudioOutStream CreatePCMStream(AudioApplication application, int samplesPerFrame, int channels, int? bitrate, int bufferMillis)
public AudioOutStream CreatePCMStream(AudioApplication application, int? bitrate, int bufferMillis)
{
CheckSamplesPerFrame(samplesPerFrame);
var outputStream = new OutputStream(ApiClient);
var sodiumEncrypter = new SodiumEncryptStream(outputStream, this);
var rtpWriter = new RTPWriteStream(sodiumEncrypter, samplesPerFrame, _ssrc);
var bufferedStream = new BufferedWriteStream(rtpWriter, this, samplesPerFrame, bufferMillis, _connection.CancelToken, _audioLogger);
return new OpusEncodeStream(bufferedStream, channels, samplesPerFrame, bitrate ?? (96 * 1024), application);
var rtpWriter = new RTPWriteStream(sodiumEncrypter, _ssrc);
var bufferedStream = new BufferedWriteStream(rtpWriter, this, bufferMillis, _connection.CancelToken, _audioLogger);
return new OpusEncodeStream(bufferedStream, bitrate ?? (96 * 1024), application);
}
public AudioOutStream CreateDirectPCMStream(AudioApplication application, int samplesPerFrame, int channels, int? bitrate)
public AudioOutStream CreateDirectPCMStream(AudioApplication application, int? bitrate)
{
CheckSamplesPerFrame(samplesPerFrame);
var outputStream = new OutputStream(ApiClient);
var sodiumEncrypter = new SodiumEncryptStream(outputStream, this);
var rtpWriter = new RTPWriteStream(sodiumEncrypter, samplesPerFrame, _ssrc);
return new OpusEncodeStream(rtpWriter, channels, samplesPerFrame, bitrate ?? (96 * 1024), application);
}
private void CheckSamplesPerFrame(int samplesPerFrame)
{
if (samplesPerFrame != 120 && samplesPerFrame != 240 && samplesPerFrame != 480 &&
samplesPerFrame != 960 && samplesPerFrame != 1920 && samplesPerFrame != 2880)
throw new ArgumentException("Value must be 120, 240, 480, 960, 1920 or 2880", nameof(samplesPerFrame));
var rtpWriter = new RTPWriteStream(sodiumEncrypter, _ssrc);
return new OpusEncodeStream(rtpWriter, bitrate ?? (96 * 1024), application);
}
internal async Task CreateInputStreamAsync(ulong userId)

46
src/Discord.Net.WebSocket/Audio/Opus/OpusConverter.cs
View File

@@ -6,37 +6,22 @@ namespace Discord.Audio
{
protected IntPtr _ptr;
/// <summary> Gets the bit rate of this converter. </summary>
public const int BitsPerSample = sizeof(short) * 8;
/// <summary> Gets the bytes per sample. </summary>
public const int SampleSize = (BitsPerSample / 8) * MaxChannels;
/// <summary> Gets the maximum amount of channels this encoder supports. </summary>
public const int MaxChannels = 2;
public const int SamplingRate = 48000;
public const int Channels = 2;
public const int FrameMillis = 20;
/// <summary> Gets the input sampling rate of this converter. </summary>
public int SamplingRate { get; }
/// <summary> Gets the number of samples per second for this stream. </summary>
public int Channels { get; }
public const int SampleBytes = sizeof(short) * Channels;
protected OpusConverter(int samplingRate, int channels)
{
if (samplingRate != 8000 && samplingRate != 12000 &&
samplingRate != 16000 && samplingRate != 24000 &&
samplingRate != 48000)
throw new ArgumentOutOfRangeException(nameof(samplingRate));
if (channels != 1 && channels != 2)
throw new ArgumentOutOfRangeException(nameof(channels));
SamplingRate = samplingRate;
Channels = channels;
}
public const int FrameSamples = SamplingRate / 1000 * FrameMillis;
public const int FrameSamplesPerChannel = SamplingRate / 1000 * FrameMillis;
public const int FrameBytes = FrameSamples * SampleBytes;
private bool disposedValue = false; // To detect redundant calls
protected bool _isDisposed = false;
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
disposedValue = true;
if (!_isDisposed)
_isDisposed = true;
}
~OpusConverter()
{
@@ -47,5 +32,16 @@ namespace Discord.Audio
Dispose(true);
GC.SuppressFinalize(this);
}
protected static void CheckError(int result)
{
if (result < 0)
throw new Exception($"Opus Error: {(OpusError)result}");
}
protected static void CheckError(OpusError error)
{
if ((int)error < 0)
throw new Exception($"Opus Error: {error}");
}
}
}

28
src/Discord.Net.WebSocket/Audio/Opus/OpusDecoder.cs
View File

@@ -14,37 +14,29 @@ namespace Discord.Audio
[DllImport("opus", EntryPoint = "opus_decoder_ctl", CallingConvention = CallingConvention.Cdecl)]
private static extern int DecoderCtl(IntPtr st, OpusCtl request, int value);
public OpusDecoder(int samplingRate, int channels)
: base(samplingRate, channels)
public OpusDecoder()
{
OpusError error;
_ptr = CreateDecoder(samplingRate, channels, out error);
if (error != OpusError.OK)
throw new Exception($"Opus Error: {error}");
_ptr = CreateDecoder(SamplingRate, Channels, out var error);
CheckError(error);
}
/// <summary> Produces PCM samples from Opus-encoded audio. </summary>
/// <param name="input">PCM samples to decode.</param>
/// <param name="inputOffset">Offset of the frame in input.</param>
/// <param name="output">Buffer to store the decoded frame.</param>
public unsafe int DecodeFrame(byte[] input, int inputOffset, int inputCount, byte[] output, int outputOffset)
{
int result = 0;
fixed (byte* inPtr = input)
fixed (byte* outPtr = output)
result = Decode(_ptr, inPtr + inputOffset, inputCount, outPtr + outputOffset, (output.Length - outputOffset) / SampleSize, 0); //TODO: Enable FEC
if (result < 0)
throw new Exception($"Opus Error: {(OpusError)result}");
return result * SampleSize;
result = Decode(_ptr, inPtr + inputOffset, inputCount, outPtr + outputOffset, FrameBytes / SampleBytes, 1);
CheckError(result);
return FrameBytes;
}
protected override void Dispose(bool disposing)
{
if (_ptr != IntPtr.Zero)
if (!_isDisposed)
{
DestroyDecoder(_ptr);
_ptr = IntPtr.Zero;
if (_ptr != IntPtr.Zero)
DestroyDecoder(_ptr);
base.Dispose(disposing);
}
}
}

58
src/Discord.Net.WebSocket/Audio/Opus/OpusEncoder.cs
View File

@@ -12,14 +12,12 @@ namespace Discord.Audio
[DllImport("opus", EntryPoint = "opus_encode", CallingConvention = CallingConvention.Cdecl)]
private static extern int Encode(IntPtr st, byte* pcm, int frame_size, byte* data, int max_data_bytes);
[DllImport("opus", EntryPoint = "opus_encoder_ctl", CallingConvention = CallingConvention.Cdecl)]
private static extern int EncoderCtl(IntPtr st, OpusCtl request, int value);
private static extern OpusError EncoderCtl(IntPtr st, OpusCtl request, int value);
/// <summary> Gets the coding mode of the encoder. </summary>
public AudioApplication Application { get; }
public int BitRate { get;}
public OpusEncoder(int samplingRate, int channels, int bitrate, AudioApplication application)
: base(samplingRate, channels)
public OpusEncoder(int bitrate, AudioApplication application)
{
if (bitrate < 1 || bitrate > DiscordVoiceAPIClient.MaxBitrate)
throw new ArgumentOutOfRangeException(nameof(bitrate));
@@ -47,57 +45,31 @@ namespace Discord.Audio
throw new ArgumentOutOfRangeException(nameof(application));
}
OpusError error;
_ptr = CreateEncoder(samplingRate, channels, (int)opusApplication, out error);
if (error != OpusError.OK)
throw new Exception($"Opus Error: {error}");
var result = EncoderCtl(_ptr, OpusCtl.SetSignal, (int)opusSignal);
if (result < 0)
throw new Exception($"Opus Error: {(OpusError)result}");
result = EncoderCtl(_ptr, OpusCtl.SetPacketLossPercent, 30); //%%
if (result < 0)
throw new Exception($"Opus Error: {(OpusError)result}");
result = EncoderCtl(_ptr, OpusCtl.SetInbandFEC, 1); //True
if (result < 0)
throw new Exception($"Opus Error: {(OpusError)result}");
result = EncoderCtl(_ptr, OpusCtl.SetBitrate, bitrate);
if (result < 0)
throw new Exception($"Opus Error: {(OpusError)result}");
/*if (application == AudioApplication.Music)
{
result = EncoderCtl(_ptr, OpusCtl.SetBandwidth, 1105);
if (result < 0)
throw new Exception($"Opus Error: {(OpusError)result}");
}*/
_ptr = CreateEncoder(SamplingRate, Channels, (int)opusApplication, out var error);
CheckError(error);
CheckError(EncoderCtl(_ptr, OpusCtl.SetSignal, (int)opusSignal));
CheckError(EncoderCtl(_ptr, OpusCtl.SetPacketLossPercent, 30)); //%
CheckError(EncoderCtl(_ptr, OpusCtl.SetInbandFEC, 1)); //True
CheckError(EncoderCtl(_ptr, OpusCtl.SetBitrate, bitrate));
}
/// <summary> Produces Opus encoded audio from PCM samples. </summary>
/// <param name="input">PCM samples to encode.</param>
/// <param name="output">Buffer to store the encoded frame.</param>
/// <returns>Length of the frame contained in outputBuffer.</returns>
public unsafe int EncodeFrame(byte[] input, int inputOffset, int inputCount, byte[] output, int outputOffset)
public unsafe int EncodeFrame(byte[] input, int inputOffset, byte[] output, int outputOffset)
{
int result = 0;
fixed (byte* inPtr = input)
fixed (byte* outPtr = output)
result = Encode(_ptr, inPtr + inputOffset, inputCount / SampleSize, outPtr + outputOffset, output.Length - outputOffset);
if (result < 0)
throw new Exception($"Opus Error: {(OpusError)result}");
result = Encode(_ptr, inPtr + inputOffset, FrameSamplesPerChannel, outPtr + outputOffset, output.Length - outputOffset);
CheckError(result);
return result;
}
protected override void Dispose(bool disposing)
{
if (_ptr != IntPtr.Zero)
if (!_isDisposed)
{
DestroyEncoder(_ptr);
_ptr = IntPtr.Zero;
if (_ptr != IntPtr.Zero)
DestroyEncoder(_ptr);
base.Dispose(disposing);
}
}
}

8
src/Discord.Net.WebSocket/Audio/Streams/BufferedWriteStream.cs
View File

@@ -38,14 +38,14 @@ namespace Discord.Audio.Streams
private bool _isPreloaded;
private int _silenceFrames;
public BufferedWriteStream(AudioStream next, IAudioClient client, int samplesPerFrame, int bufferMillis, CancellationToken cancelToken, int maxFrameSize = 1500)
: this(next, client as AudioClient, samplesPerFrame, bufferMillis, cancelToken, null, maxFrameSize) { }
internal BufferedWriteStream(AudioStream next, AudioClient client, int samplesPerFrame, int bufferMillis, CancellationToken cancelToken, Logger logger, int maxFrameSize = 1500)
public BufferedWriteStream(AudioStream next, IAudioClient client, int bufferMillis, CancellationToken cancelToken, int maxFrameSize = 1500)
: this(next, client as AudioClient, bufferMillis, cancelToken, null, maxFrameSize) { }
internal BufferedWriteStream(AudioStream next, AudioClient client, int bufferMillis, CancellationToken cancelToken, Logger logger, int maxFrameSize = 1500)
{
//maxFrameSize = 1275 was too limiting at 128kbps,2ch,60ms
_next = next;
_client = client;
_ticksPerFrame = samplesPerFrame / 48;
_ticksPerFrame = OpusEncoder.FrameSamples / 48;
_logger = logger;
_queueLength = (bufferMillis + (_ticksPerFrame - 1)) / _ticksPerFrame; //Round up

8
src/Discord.Net.WebSocket/Audio/Streams/OpusDecodeStream.cs
View File

@@ -9,14 +9,14 @@ namespace Discord.Audio.Streams
public const int SampleRate = OpusEncodeStream.SampleRate;
private readonly AudioStream _next;
private readonly byte[] _buffer;
private readonly OpusDecoder _decoder;
private readonly byte[] _buffer;
public OpusDecodeStream(AudioStream next, int channels = OpusConverter.MaxChannels, int bufferSize = 5760 * 2 * sizeof(short))
public OpusDecodeStream(AudioStream next)
{
_next = next;
_buffer = new byte[bufferSize];
_decoder = new OpusDecoder(SampleRate, channels);
_buffer = new byte[OpusConverter.FrameBytes];
_decoder = new OpusDecoder();
}
public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)

36
src/Discord.Net.WebSocket/Audio/Streams/OpusEncodeStream.cs
View File

@@ -12,18 +12,13 @@ namespace Discord.Audio.Streams
private readonly AudioStream _next;
private readonly OpusEncoder _encoder;
private readonly byte[] _buffer;
private int _frameSize;
private byte[] _partialFrameBuffer;
private int _partialFramePos;
public OpusEncodeStream(AudioStream next, int channels, int samplesPerFrame, int bitrate, AudioApplication application, int bufferSize = 4000)
public OpusEncodeStream(AudioStream next, int bitrate, AudioApplication application)
{
_next = next;
_encoder = new OpusEncoder(SampleRate, channels, bitrate, application);
_frameSize = samplesPerFrame * channels * 2;
_buffer = new byte[bufferSize];
_partialFrameBuffer = new byte[_frameSize];
_encoder = new OpusEncoder(bitrate, application);
_buffer = new byte[OpusConverter.FrameBytes];
}
public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
@@ -31,20 +26,31 @@ namespace Discord.Audio.Streams
//Assume threadsafe
while (count > 0)
{
if (_partialFramePos + count >= _frameSize)
if (_partialFramePos == 0 && count >= OpusConverter.FrameBytes)
{
int partialSize = _frameSize - _partialFramePos;
Buffer.BlockCopy(buffer, offset, _partialFrameBuffer, _partialFramePos, partialSize);
//We have enough data and no partial frames. Pass the buffer directly to the encoder
int encFrameSize = _encoder.EncodeFrame(buffer, offset, _buffer, 0);
await _next.WriteAsync(_buffer, 0, encFrameSize, cancellationToken).ConfigureAwait(false);
offset += OpusConverter.FrameBytes;
count -= OpusConverter.FrameBytes;
}
else if (_partialFramePos + count >= OpusConverter.FrameBytes)
{
//We have enough data to complete a previous partial frame.
int partialSize = OpusConverter.FrameBytes - _partialFramePos;
Buffer.BlockCopy(buffer, offset, _buffer, _partialFramePos, partialSize);
int encFrameSize = _encoder.EncodeFrame(_buffer, 0, _buffer, 0);
await _next.WriteAsync(_buffer, 0, encFrameSize, cancellationToken).ConfigureAwait(false);
offset += partialSize;
count -= partialSize;
_partialFramePos = 0;
int encFrameSize = _encoder.EncodeFrame(_partialFrameBuffer, 0, _frameSize, _buffer, 0);
await _next.WriteAsync(_buffer, 0, encFrameSize, cancellationToken).ConfigureAwait(false);
}
else
{
Buffer.BlockCopy(buffer, offset, _partialFrameBuffer, _partialFramePos, count);
//Not enough data to build a complete frame, store this part for later
Buffer.BlockCopy(buffer, offset, _buffer, _partialFramePos, count);
_partialFramePos += count;
break;
}

9
src/Discord.Net.WebSocket/Audio/Streams/RTPWriteStream.cs
View File

@@ -9,15 +9,12 @@ namespace Discord.Audio.Streams
{
private readonly AudioStream _next;
private readonly byte[] _header;
private int _samplesPerFrame;
protected readonly byte[] _buffer;
private uint _ssrc, _timestamp = 0;
protected readonly byte[] _buffer;
public RTPWriteStream(AudioStream next, int samplesPerFrame, uint ssrc, int bufferSize = 4000)
public RTPWriteStream(AudioStream next, uint ssrc, int bufferSize = 4000)
{
_next = next;
_samplesPerFrame = samplesPerFrame;
_ssrc = ssrc;
_buffer = new byte[bufferSize];
_header = new byte[24];
@@ -38,7 +35,7 @@ namespace Discord.Audio.Streams
if (_header[3]++ == byte.MaxValue)
_header[2]++;
_timestamp += (uint)_samplesPerFrame;
_timestamp += (uint)OpusEncoder.FrameSamples;
_header[4] = (byte)(_timestamp >> 24);
_header[5] = (byte)(_timestamp >> 16);
_header[6] = (byte)(_timestamp >> 8);