feat: experiment.UseCustomOrdering()

README.md

@@ -29,7 +29,7 @@ public bool CanAccess(IUser user)
 Wrap a `Use` block around the code's original behavior, and wrap `Try` around the new behavior. Invoking `Scientist.Science<T>` will always return whatever the `Use` block returns, but it does a bunch of stuff behind the scenes:
 
 * It decides whether or not to run the `Try` block,
-* Randomizes the order in which `Use` and `Try` blocks are run,
+* By default randomises the order in which `Use` and `Try` blocks are run,
 * Measures the durations of all behaviors,
 * Compares the result of `Try` to the result of `Use`,
 * Swallows (but records) any exceptions raised in the `Try` block, and
@@ -341,6 +341,41 @@ public bool CanAccess(IUser user)
 }
 ```
 
+### Changing the order
+
+By default the library randomises the order in which behaviors (control & candidates) are ran however, under certain cirumstances you might want to run your own ordering so we provide a handy method `UseCustomOrdering`. 
+```csharp
+scientist.Experiment<int>(experimentName, experiment =>
+{
+    experiment.UseCustomOrdering(Ordering.ControlFirst);
+    // ...
+});
+```
+There are some pre-written alogrithms
+```csharp
+Ordering.Random
+Ordering.ControlFirst
+Ordering.ControlLast
+```
+But if you need to rock your own then you can write something like below
+```csharp
+ private static int _seed = 123;
+
+ public static IReadOnlyList<INamedBehavior<T>> SeededExperimentOrderer<T>(IReadOnlyList<INamedBehavior<T>> behaviors)
+ {
+     var random = new Random(_seed);
+     return behaviors.OrderBy(_ => random.Next()).ToList();
+ }
+
+// ...
+
+scientist.Experiment<int>(experimentName, experiment =>
+{
+    experiment.UseCustomOrdering(SeededExperimentOrderer);
+    // ...
+});
+```
+
 ## Alternatives
 
 Here are other implementations of Scientist available in different languages.

src/Scientist/IExperiment.cs

@@ -1,4 +1,6 @@
-using System;
+using Github.Ordering;
+using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 
 namespace GitHub.Internals
@@ -78,8 +80,14 @@ public interface IExperiment<T> : IExperiment
         /// <summary>
         /// Defines the check to run to determine if mismatches should be ignored.
         /// </summary>
-        /// <param name="block">The delegate to execute</param>
+        /// <param name="block">The delegate to execute.</param>
         void Ignore(Func<T, T, bool> block);
+
+        /// <summary>
+        /// Defines the custom ordering to run on the behaviors
+        /// </summary>
+        /// <param name="ordering">The delgate to execute.</param>
+        void UseCustomOrdering(Func<IReadOnlyList<INamedBehavior<T>>, IReadOnlyList<INamedBehavior<T>>> ordering);
     }
 
     /// <summary>
@@ -136,8 +144,14 @@ public interface IExperimentAsync<T> : IExperiment
         /// <summary>
         /// Defines the check to run to determine if mismatches should be ignored.
         /// </summary>
-        /// <param name="block">The delegate to execute</param>
+        /// <param name="block">The delegate to execute.</param>
         void Ignore(Func<T, T, Task<bool>> block);
+
+        /// <summary>
+        /// Defines the custom ordering to run on the behaviors
+        /// </summary>
+        /// <param name="customOrdering">The delgate to execute.</param>
+        void UseCustomOrdering(CustomOrderer<T> customOrdering);
     }
 
     /// <summary>

src/Scientist/Internals/Experiment.cs

@@ -1,4 +1,5 @@
-using System;
+using Github.Ordering;
+using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 
@@ -28,6 +29,8 @@ private static readonly Action<Operation, Exception> _alwaysThrow
         private readonly Dictionary<string, dynamic> _contexts = new Dictionary<string, dynamic>();
         private readonly IResultPublisher _resultPublisher;
 
+        private CustomOrderer<T> _customOrderer = behaviors => Task.FromResult(Ordering.Random(behaviors));
+
         public Experiment(string name, Func<Task<bool>> enabled, int concurrentTasks, IResultPublisher resultPublisher)
         {
             if (concurrentTasks <= 0)
@@ -130,7 +133,8 @@ internal ExperimentInstance<T, TClean> Build() =>
                 RunIf = _runIf,
                 Thrown = _thrown,
                 ThrowOnMismatches = ThrowOnMismatches,
-                ResultPublisher = _resultPublisher
+                ResultPublisher = _resultPublisher,
+                CustomOrderer = _customOrderer
             });
 
         public void Compare(Func<T, T, bool> comparison)
@@ -156,5 +160,16 @@ public void BeforeRun(Func<Task> action)
         {
             _beforeRun = action;
         }
+
+
+        public void UseCustomOrdering(Func<IReadOnlyList<INamedBehavior<T>>, IReadOnlyList<INamedBehavior<T>>> customOrdering)
+        {
+            _customOrderer = list => Task.FromResult(customOrdering(list));
+        }
+
+        public void UseCustomOrdering(CustomOrderer<T> customOrdering)
+        {
+            _customOrderer = customOrdering;
+        }
     }
 }

src/Scientist/Internals/ExperimentInstance.cs

@@ -1,4 +1,5 @@
-using System;
+using Github.Ordering;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
@@ -16,7 +17,7 @@ internal class ExperimentInstance<T, TClean>
 
         internal readonly string Name;
         internal readonly int ConcurrentTasks;
-        internal readonly List<NamedBehavior> Behaviors;
+        internal readonly List<NamedBehavior<T>> Behaviors;
         internal readonly Func<T, TClean> Cleaner;
         internal readonly Func<T, T, bool> Comparator;
         internal readonly Func<Task> BeforeRun;
@@ -27,19 +28,18 @@ internal class ExperimentInstance<T, TClean>
         internal readonly Action<Operation, Exception> Thrown;
         internal readonly bool ThrowOnMismatches;
         internal readonly IResultPublisher ResultPublisher;
-
-        static Random _random = new Random(DateTimeOffset.UtcNow.Millisecond);
-
+        internal readonly CustomOrderer<T> CustomOrderer;
+
         public ExperimentInstance(ExperimentSettings<T, TClean> settings)
         {
             Name = settings.Name;
 
-            Behaviors = new List<NamedBehavior>
+            Behaviors = new List<NamedBehavior<T>>
             {
-                new NamedBehavior(ControlExperimentName, settings.Control),
+                new NamedBehavior<T>(ControlExperimentName, settings.Control),
             };
             Behaviors.AddRange(
-                settings.Candidates.Select(c => new NamedBehavior(c.Key, c.Value)));
+                settings.Candidates.Select(c => new NamedBehavior<T>(c.Key, c.Value)));
 
             BeforeRun = settings.BeforeRun;
             Cleaner = settings.Cleaner;
@@ -52,6 +52,7 @@ public ExperimentInstance(ExperimentSettings<T, TClean> settings)
             Thrown = settings.Thrown;
             ThrowOnMismatches = settings.ThrowOnMismatches;
             ResultPublisher = settings.ResultPublisher;
+            CustomOrderer = settings.CustomOrderer;
         }
 
         public async Task<T> Run()
@@ -68,12 +69,7 @@ public async Task<T> Run()
                 await BeforeRun().ConfigureAwait(false);
             }
 
-            // Randomize ordering...
-            NamedBehavior[] orderedBehaviors;
-            lock (_random)
-            {
-                orderedBehaviors = Behaviors.OrderBy(b => _random.Next()).ToArray();
-            }
+            var orderedBehaviors = await CustomOrderer(Behaviors).ConfigureAwait(false);
 
             // Break tasks into batches of "ConcurrentTasks" size
             var observations = new List<Observation<T, TClean>>();
@@ -95,7 +91,7 @@ public async Task<T> Run()
             }
 
             var controlObservation = observations.FirstOrDefault(o => o.Name == ControlExperimentName);
-            
+
             var result = new Result<T, TClean>(this, observations, controlObservation, Contexts);
 
             try
@@ -115,7 +111,7 @@ public async Task<T> Run()
             if (controlObservation.Thrown) throw controlObservation.Exception;
             return controlObservation.Value;
         }
-        
+
         /// <summary>
         /// Does <see cref="RunIf"/> allow the experiment to run?
         /// </summary>
@@ -149,7 +145,7 @@ public async Task<bool> IgnoreMismatchedObservation(Observation<T, TClean> contr
                 return false;
             }
         }
-        
+
         /// <summary>
         /// Determine whether or not the experiment should run.
         /// </summary>
@@ -167,29 +163,5 @@ async Task<bool> ShouldExperimentRun()
                 return false;
             }
         }
-
-        internal class NamedBehavior
-        {
-            public NamedBehavior(string name, Func<T> behavior)
-                : this(name, () => Task.FromResult(behavior()))
-            {
-            }
-
-            public NamedBehavior(string name, Func<Task<T>> behavior)
-            {
-                Behavior = behavior;
-                Name = name;
-            }
-
-            /// <summary>
-            /// Gets the behavior to execute during an experiment.
-            /// </summary>
-            public Func<Task<T>> Behavior { get; }
-
-            /// <summary>
-            /// Gets the name of the behavior.
-            /// </summary>
-            public string Name { get; }
-        }
     }
 }

src/Scientist/Internals/ExperimentSettings.cs

@@ -1,4 +1,5 @@
-using System;
+using Github.Ordering;
+using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 
@@ -26,5 +27,6 @@ internal class ExperimentSettings<T, TClean>
         public bool ThrowOnMismatches { get; set; }
         public Action<Operation, Exception> Thrown { get; set; }
         public IResultPublisher ResultPublisher { get; set; }
+        public CustomOrderer<T> CustomOrderer { get; set;  }
     }
 }

src/Scientist/NamedBehavior.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Threading.Tasks;
+
+namespace GitHub
+{
+    public interface INamedBehavior<T>
+    {
+        string Name { get; }
+        Func<Task<T>> Behavior { get; }
+    }
+    public class NamedBehavior<T> : INamedBehavior<T>
+    {
+        public NamedBehavior(string name, Func<T> method)
+                : this(name, () => Task.FromResult(method()))
+        {
+        }
+
+        public NamedBehavior(string name, Func<Task<T>> method)
+        {
+            Behavior = method;
+            Name = name;
+        }
+
+        /// <summary>
+        /// Gets the name of the behavior.
+        /// </summary>
+        public string Name { get; }
+
+        /// <summary>
+        /// Gets the behavior to execute during an experiment.
+        /// </summary>
+        public Func<Task<T>> Behavior { get; }
+    }
+}

src/Scientist/Ordering/Ordering.cs

@@ -0,0 +1,34 @@
+using GitHub;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace Github.Ordering
+{
+    public delegate Task<IReadOnlyList<INamedBehavior<T>>> CustomOrderer<T>(IReadOnlyList<INamedBehavior<T>> namedBehaviors);
+
+    public class Ordering
+    {
+        public static IReadOnlyList<INamedBehavior<T>> ControlFirst<T>(IReadOnlyList<INamedBehavior<T>> namedBehaviors)
+        {
+            return namedBehaviors.OrderByDescending(namedBehavior => namedBehavior.Name == "control").ToList();
+        }
+
+        public static IReadOnlyList<INamedBehavior<T>> ControlLast<T>(IReadOnlyList<INamedBehavior<T>> namedBehaviors)
+        {
+            return ControlFirst(namedBehaviors).Reverse().ToList();
+        }
+
+        static Random _random = new Random(DateTimeOffset.UtcNow.Millisecond);
+
+        public static IReadOnlyList<INamedBehavior<T>> Random<T>(IReadOnlyList<INamedBehavior<T>> namedBehaviors)
+        {
+            lock (_random)
+            {
+                return namedBehaviors.OrderBy(b => _random.Next()).ToArray();
+            }
+        }
+    }
+
+}

src/Scientist/Scientist.csproj

@@ -31,7 +31,7 @@
     <None Include="..\..\LICENSE.txt" Pack="true" PackagePath="" />
     <None Include="..\..\README.md" Pack="true" PackagePath="" />
     <None Include="..\..\ReleaseNotes.md" Pack="true" PackagePath="" />
-    <None Include="..\..\assets\scientist-logo-64x64.png" Pack="true" PackagePath=""/>
+    <None Include="..\..\assets\scientist-logo-64x64.png" Pack="true" PackagePath="" />
   </ItemGroup>
 
   <ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">

src/Scientist/packages.lock.json

@@ -2,15 +2,6 @@
   "version": 1,
   "dependencies": {
     ".NETFramework,Version=v4.5.1": {
-      "Microsoft.NETFramework.ReferenceAssemblies": {
-        "type": "Direct",
-        "requested": "[1.0.3, )",
-        "resolved": "1.0.3",
-        "contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg==",
-        "dependencies": {
-          "Microsoft.NETFramework.ReferenceAssemblies.net451": "1.0.3"
-        }
-      },
       "Microsoft.SourceLink.GitHub": {
         "type": "Direct",
         "requested": "[8.0.0, )",
@@ -26,11 +17,6 @@
         "resolved": "8.0.0",
         "contentHash": "bZKfSIKJRXLTuSzLudMFte/8CempWjVamNUR5eHJizsy+iuOuO/k2gnh7W0dHJmYY0tBf+gUErfluCv5mySAOQ=="
       },
-      "Microsoft.NETFramework.ReferenceAssemblies.net451": {
-        "type": "Transitive",
-        "resolved": "1.0.3",
-        "contentHash": "vVPinxdLrwoX81ApbNIHDBI6qymQEy8eSOxDNBgKJtc2+cifnF0oT1U2d3EFx+V5O68yaqna2myZJNsgKCpVkA=="
-      },
       "Microsoft.SourceLink.Common": {
         "type": "Transitive",
         "resolved": "8.0.0",