feature: build overrides (#2212)

* add build overrides

* override docs

* add server submodule

* add overrides to build step

* remove testing api url

Co-Authored-By: Quahu <quahuu@gmail.com>

Co-authored-by: Quahu <quahuu@gmail.com>

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "overrides/Discord.Net.BuildOverrides"]
+	path = overrides/Discord.Net.BuildOverrides
+	url = https://github.com/discord-net/Discord.Net.BuildOverrides

Discord.Net.sln

@@ -34,7 +34,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "_InteractionFramework", "sa
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "_WebhookClient", "samples\WebhookClient\_WebhookClient.csproj", "{B61AAE66-15CC-40E4-873A-C23E697C3411}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IDN", "samples\idn\idn.csproj", "{4A03840B-9EBE-47E3-89AB-E0914DF21AFB}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "idn", "samples\idn\idn.csproj", "{4A03840B-9EBE-47E3-89AB-E0914DF21AFB}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{C7CF5621-7D36-433B-B337-5A2E3C101A71}"
 EndProject
@@ -44,6 +44,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Extensions", "Extensions",
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Samples", "Samples", "{BB59D5B5-E7B0-4BF4-8F82-D14431B2799B}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Discord.Net.BuildOverrides", "experiment\Discord.Net.BuildOverrides\Discord.Net.BuildOverrides.csproj", "{115F4921-B44D-4F69-996B-69796959C99D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -258,6 +260,18 @@ Global
 		{4A03840B-9EBE-47E3-89AB-E0914DF21AFB}.Release|x64.Build.0 = Release|Any CPU
 		{4A03840B-9EBE-47E3-89AB-E0914DF21AFB}.Release|x86.ActiveCfg = Release|Any CPU
 		{4A03840B-9EBE-47E3-89AB-E0914DF21AFB}.Release|x86.Build.0 = Release|Any CPU
+		{115F4921-B44D-4F69-996B-69796959C99D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{115F4921-B44D-4F69-996B-69796959C99D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{115F4921-B44D-4F69-996B-69796959C99D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{115F4921-B44D-4F69-996B-69796959C99D}.Debug|x64.Build.0 = Debug|Any CPU
+		{115F4921-B44D-4F69-996B-69796959C99D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{115F4921-B44D-4F69-996B-69796959C99D}.Debug|x86.Build.0 = Debug|Any CPU
+		{115F4921-B44D-4F69-996B-69796959C99D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{115F4921-B44D-4F69-996B-69796959C99D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{115F4921-B44D-4F69-996B-69796959C99D}.Release|x64.ActiveCfg = Release|Any CPU
+		{115F4921-B44D-4F69-996B-69796959C99D}.Release|x64.Build.0 = Release|Any CPU
+		{115F4921-B44D-4F69-996B-69796959C99D}.Release|x86.ActiveCfg = Release|Any CPU
+		{115F4921-B44D-4F69-996B-69796959C99D}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -279,6 +293,7 @@ Global
 		{A23E46D2-1610-4AE5-820F-422D34810887} = {BB59D5B5-E7B0-4BF4-8F82-D14431B2799B}
 		{B61AAE66-15CC-40E4-873A-C23E697C3411} = {BB59D5B5-E7B0-4BF4-8F82-D14431B2799B}
 		{4A03840B-9EBE-47E3-89AB-E0914DF21AFB} = {BB59D5B5-E7B0-4BF4-8F82-D14431B2799B}
+		{115F4921-B44D-4F69-996B-69796959C99D} = {CC3D4B1C-9DE0-448B-8AE7-F3F1F3EC5C3A}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {D2404771-EEC8-45F2-9D71-F3373F6C1495}

azure/deploy.yml

@@ -8,6 +8,7 @@ steps:
     dotnet pack "src\Discord.Net.Providers.WS4Net\Discord.Net.Providers.WS4Net.csproj" --no-restore --no-build -v minimal -c $(buildConfiguration) -o "$(Build.ArtifactStagingDirectory)" /p:BuildNumber=$(buildNumber) /p:IsTagBuild=$(buildTag)
     dotnet pack "src\Discord.Net.Analyzers\Discord.Net.Analyzers.csproj" --no-restore --no-build -v minimal -c $(buildConfiguration) -o "$(Build.ArtifactStagingDirectory)" /p:BuildNumber=$(buildNumber) /p:IsTagBuild=$(buildTag)
     dotnet pack "src\Discord.Net.Interactions\Discord.Net.Interactions.csproj" --no-restore --no-build -v minimal -c $(buildConfiguration) -o "$(Build.ArtifactStagingDirectory)" /p:BuildNumber=$(buildNumber) /p:IsTagBuild=$(buildTag)
+    dotnet pack "experiment\Discord.Net.BuildOverrides\Discord.Net.BuildOverrides.csproj" --no-restore --no-build -v minimal -c $(buildConfiguration) -o "$(Build.ArtifactStagingDirectory)" /p:BuildNumber=$(buildNumber) /p:IsTagBuild=$(buildTag)
   displayName: Pack projects
 
 - task: NuGetCommand@2

docs/faq/build_overrides/what-are-they.md

@@ -0,0 +1,41 @@
+---
+uid: FAQ.BuildOverrides.WhatAreThey
+title: Build Overrides, What are they?
+---
+
+# Build Overrides
+
+Build overrides are a way for library developers to override the default behavior of the library on the fly. Adding them to your code is really simple.
+
+## Installing the package
+
+The build override package can be installed on nuget [here](TODO) or by using the package manager
+
+```
+PM> Install-Package Discord.Net.BuildOverrides
+```
+
+## Adding an override
+
+```cs
+public async Task MainAsync()
+{
+  // hook into the log function
+  BuildOverrides.Log += (buildOverride, message) => 
+  {
+    Console.WriteLine($"{buildOverride.Name}: {message}");
+    return Task.CompletedTask;
+  };
+
+  // add your overrides
+  await BuildOverrides.AddOverrideAsync("example-override-name");
+}
+
+```
+
+Overrides are normally built for specific problems, for example if someone is having an issue and we think we might have a fix then we can create a build override for them to test out the fix.
+
+## Security and Transparency
+
+Overrides can only be created and updated by library developers, you should only apply an override if a library developer askes you to.
+Code for the overrides server and the overrides themselves can be found [here](https://github.com/discord-net/Discord.Net.BuildOverrides).

docs/faq/toc.yml

@@ -22,3 +22,5 @@
       topicUid: FAQ.TextCommands.General
 - name: Legacy or Upgrade
   topicUid: FAQ.Legacy
+- name: Build Overrides
+  topicUid: FAQ.BuildOverrides.WhatAreThey

experiment/Discord.Net.BuildOverrides/BuildOverrides.cs

@@ -0,0 +1,278 @@
+using Discord.Overrides;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Reflection;
+using System.Runtime.Loader;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Discord
+{
+    /// <summary>
+    ///     Represents an override that can be loaded.
+    /// </summary>
+    public sealed class Override
+    {
+        /// <summary>
+        ///     Gets the ID of the override.
+        /// </summary>
+        public Guid Id { get; internal set; }
+
+        /// <summary>
+        ///     Gets the name of the override.
+        /// </summary>
+        public string Name { get; internal set; }
+
+        /// <summary>
+        ///     Gets the description of the override.
+        /// </summary>
+        public string Description { get; internal set; }
+
+        /// <summary>
+        ///     Gets the date this override was created.
+        /// </summary>
+        public DateTimeOffset CreatedAt { get; internal set; }
+
+        /// <summary>
+        ///     Gets the date the override was last modified.
+        /// </summary>
+        public DateTimeOffset LastUpdated { get; internal set; }
+
+        internal static Override FromJson(string json)
+        {
+            var result = new Override();
+
+            using(var textReader = new StringReader(json))
+            using(var reader = new JsonTextReader(textReader))
+            {
+                var obj = JObject.ReadFrom(reader);
+                result.Id = obj["id"].ToObject<Guid>();
+                result.Name = obj["name"].ToObject<string>();
+                result.Description = obj["description"].ToObject<string>();
+                result.CreatedAt = obj["created_at"].ToObject<DateTimeOffset>();
+                result.LastUpdated = obj["last_updated"].ToObject<DateTimeOffset>();
+            }
+
+            return result;
+        }
+    }
+
+    /// <summary>
+    ///     Represents a loaded override instance.
+    /// </summary>
+    public sealed class LoadedOverride
+    {
+        /// <summary>
+        ///     Gets the aseembly containing the overrides definition.
+        /// </summary>
+        public Assembly Assembly { get; internal set; }
+
+        /// <summary>
+        ///     Gets an instance of the override.
+        /// </summary>
+        public IOverride Instance { get; internal set; }
+
+        /// <summary>
+        ///     Gets the overrides type.
+        /// </summary>
+        public Type Type { get; internal set; }
+    }
+
+    public sealed class BuildOverrides
+    {
+        /// <summary>
+        ///     Fired when an override logs a message.
+        /// </summary>
+        public static event Func<Override, string, Task> Log
+        {
+            add => _logEvents.Add(value);
+            remove => _logEvents.Remove(value);
+
+        }
+
+        /// <summary>
+        ///     Gets a read-only dictionary containing the currently loaded overrides.
+        /// </summary>
+        public IReadOnlyDictionary<Override, IReadOnlyCollection<LoadedOverride>> LoadedOverrides
+            => _loadedOverrides.Select(x => new KeyValuePair<Override, IReadOnlyCollection<LoadedOverride>> (x.Key, x.Value)).ToDictionary(x => x.Key, x => x.Value);
+
+        private static AssemblyLoadContext _overrideDomain;
+        private static List<Func<Override, string, Task>> _logEvents = new();
+        private static ConcurrentDictionary<Override, List<LoadedOverride>> _loadedOverrides = new ConcurrentDictionary<Override, List<LoadedOverride>>();
+
+        private const string ApiUrl = "https://overrides.discordnet.dev";
+        
+        static BuildOverrides()
+        {
+            _overrideDomain = new AssemblyLoadContext("Discord.Net.Overrides.Runtime");
+
+            _overrideDomain.Resolving += _overrideDomain_Resolving;
+        }
+
+        /// <summary>
+        ///     Gets details about a specific override.
+        /// </summary>
+        /// <remarks>
+        ///     <b>Note:</b> This method does not load an override, it simply retrives the info about it.
+        /// </remarks>
+        /// <param name="name">The name of the override to get.</param>
+        /// <returns>
+        ///     A task representing the asynchronous get operation. The tasks result is an <see cref="Override"/>
+        ///     if it exists; otherwise <see langword="null"/>.
+        /// </returns>
+        public static async Task<Override> GetOverrideAsync(string name)
+        {
+            using (var client = new HttpClient())
+            {
+                var result = await client.GetAsync($"{ApiUrl}/override/{name}");
+
+                if (result.IsSuccessStatusCode)
+                {
+                    var content = await result.Content.ReadAsStringAsync();
+
+                    return Override.FromJson(content);
+                }
+                else
+                    return null;
+            }
+        }
+
+        /// <summary>
+        ///     Adds an override to the current Discord.Net instance.
+        /// </summary>
+        /// <remarks>
+        ///     The override initialization is non-blocking, any errors that occor within
+        ///     the overrides initialization procedure will be sent in the <see cref="Log"/> event.
+        /// </remarks>
+        /// <param name="name">The name of the override to add.</param>
+        /// <returns>
+        ///     A task representing the asynchronous add operaton. The tasks result is a boolean
+        ///     determining if the add operation was successful.
+        /// </returns>
+        public static async Task<bool> AddOverrideAsync(string name)
+        {
+            var ovrride = await GetOverrideAsync(name);
+
+            if (ovrride == null)
+                return false;
+
+            return await AddOverrideAsync(ovrride);
+        }
+
+        /// <summary>
+        ///     Adds an override to the current Discord.Net instance.
+        /// </summary>
+        /// <remarks>
+        ///     The override initialization is non-blocking, any errors that occor within
+        ///     the overrides initialization procedure will be sent in the <see cref="Log"/> event.
+        /// </remarks>
+        /// <param name="ovrride">The override to add.</param>
+        /// <returns>
+        ///     A task representing the asynchronous add operaton. The tasks result is a boolean
+        ///     determining if the add operation was successful.
+        /// </returns>
+        public static async Task<bool> AddOverrideAsync(Override ovrride)
+        {
+            // download it
+            var ms = new MemoryStream();
+
+            using (var client = new HttpClient())
+            {
+                var result = await client.GetAsync($"{ApiUrl}/override/download/{ovrride.Id}");
+
+                if (!result.IsSuccessStatusCode)
+                    return false;
+
+                await (await result.Content.ReadAsStreamAsync()).CopyToAsync(ms);
+            }
+
+            ms.Position = 0;
+
+            // load the assembly
+            //var test = Assembly.Load(ms.ToArray());
+            var asm = _overrideDomain.LoadFromStream(ms);
+
+            // find out IOverride
+            var overrides = asm.GetTypes().Where(x => x.GetInterfaces().Any(x => x == typeof(IOverride)));
+
+            List<LoadedOverride> loaded = new();
+
+            var context = new OverrideContext((m) => HandleLog(ovrride, m), ovrride);
+
+            foreach (var ovr in overrides)
+            {
+                var inst = (IOverride)Activator.CreateInstance(ovr);
+
+                inst.RegisterPackageLookupHandler((s) =>
+                {
+                    return GetDependencyAsync(ovrride.Id, s);
+                });
+
+                _ = Task.Run(async () =>
+                {
+                    try
+                    {
+                        await inst.InitializeAsync(context);
+                    }
+                    catch (Exception x)
+                    {
+                        HandleLog(ovrride, $"Failed to initialize build override: {x}");
+                    }
+                });
+
+                loaded.Add(new LoadedOverride()
+                {
+                    Assembly = asm,
+                    Instance = inst,
+                    Type = ovr
+                });
+            }
+
+            return _loadedOverrides.AddOrUpdate(ovrride, loaded, (_, __) => loaded) != null;
+        }
+
+        internal static void HandleLog(Override ovr, string msg)
+        {
+            _ = Task.Run(async () =>
+            {
+                foreach (var item in _logEvents)
+                {
+                    await item.Invoke(ovr, msg).ConfigureAwait(false);
+                }
+            });
+        }
+
+        private static Assembly _overrideDomain_Resolving(AssemblyLoadContext arg1, AssemblyName arg2)
+        {
+            // resolve the override id
+            var v = _loadedOverrides.FirstOrDefault(x => x.Value.Any(x => x.Assembly.FullName == arg1.Assemblies.FirstOrDefault().FullName));
+
+            return GetDependencyAsync(v.Key.Id, $"{arg2}").GetAwaiter().GetResult();
+        }
+
+        private static async Task<Assembly> GetDependencyAsync(Guid id, string name)
+        {
+            using(var client = new HttpClient())
+            {
+                var result = await client.PostAsync($"{ApiUrl}/override/{id}/dependency", new StringContent($"{{ \"info\": \"{name}\"}}", Encoding.UTF8, "application/json"));
+
+                if (!result.IsSuccessStatusCode)
+                    throw new Exception("Failed to get dependency");
+
+                using(var ms = new MemoryStream())
+                {
+                    var innerStream = await result.Content.ReadAsStreamAsync();
+                    await innerStream.CopyToAsync(ms);
+                    ms.Position = 0;
+                    return _overrideDomain.LoadFromStream(ms);
+                }
+            }
+        }
+    }
+}

experiment/Discord.Net.BuildOverrides/Discord.Net.BuildOverrides.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <LangVersion>9.0</LangVersion>
+    <AssemblyName>Discord.Net.BuildOverrides</AssemblyName>
+    <RootNamespace>Discord.BuildOverrides</RootNamespace>
+    <Description>A Discord.Net extension adding a way to add build overrides for testing.</Description>
+    <TargetFrameworks Condition=" '$(OS)' == 'Windows_NT' ">net6.0;net5.0;</TargetFrameworks>
+    <TargetFrameworks Condition=" '$(OS)' != 'Windows_NT' ">net6.0;net5.0;</TargetFrameworks>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
+  </ItemGroup>
+
+  <ItemGroup Condition=" '$(TargetFramework)' == 'net461' ">
+    <Reference Include="System.Net.Http" />
+  </ItemGroup>
+
+</Project>

experiment/Discord.Net.BuildOverrides/IOverride.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Discord.Overrides
+{
+    /// <summary>
+    ///     Represents a generic build override for Discord.Net
+    /// </summary>
+    public interface IOverride
+    {
+        /// <summary>
+        ///     Initializes the override.
+        /// </summary>
+        /// <remarks>
+        ///     This method is called by the <see cref="BuildOverrides"/> class
+        ///     and should not be called externally from it.
+        /// </remarks>
+        /// <param name="context">Context used by an override to initialize.</param>
+        /// <returns>
+        ///     A task representing the asynchronous initialization operation.
+        /// </returns>
+        Task InitializeAsync(OverrideContext context);
+
+        /// <summary>
+        ///     Registers a callback to load a dependency for this override.
+        /// </summary>
+        /// <param name="func">The callback to load an external dependency.</param>
+        void RegisterPackageLookupHandler(Func<string, Task<Assembly>> func);
+    }
+}

experiment/Discord.Net.BuildOverrides/OverrideContext.cs

@@ -0,0 +1,30 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Discord.Overrides
+{
+    /// <summary>
+    ///     Represents context thats passed to an override in the initialization step.
+    /// </summary>
+    public sealed class OverrideContext
+    {
+        /// <summary>
+        ///     A callback used to log messages.
+        /// </summary>
+        public Action<string> Log { get; private set; }
+
+        /// <summary>
+        ///     The info about the override.
+        /// </summary>
+        public Override Info { get; private set; }
+
+        internal OverrideContext(Action<string> log, Override info)
+        {
+            Log = log;
+            Info = info;
+        }
+    }
+}