feat: Asynchronous experiment cancellation

README.md

@@ -263,6 +263,54 @@ await Scientist.ScienceAsync<int>(
 
 As always when using async/await, don't forget to call `.ConfigureAwait(false)` where appropriate.
 
+#### Asynchronous cancellation
+Experiments can be cancelled part way through running by using any of the three ways below
+
+1. Via the global `WithCancellationToken` method
+```csharp
+var cancellationTokenSource = new CancellationTokenSource();
+
+await scientist.ExperimentAsync<int>(experimentName, experiment =>
+{
+    experiment.WithCancellationToken(cancellationTokenSource.Token);
+    experiment.Use(async () => await StartRunningSomething(myData));
+    experiment.Try("candidate", async () => await AnAsyncMethod(myData));
+});
+```
+
+2. Per method
+```csharp
+var controlCancellationTokenSource = new CancellationTokenSource();
+var candidateCancellationTokenSource = new CancellationTokenSource();
+
+await scientist.ExperimentAsync<int>(experimentName, experiment =>
+{
+    experiment.Use(async () => await StartRunningSomething(myData), controlCancellationTokenSource.Token);
+    experiment.Try(
+        "candidate",
+        async () => await AnAsyncMethod(myData),
+        candidateCancellationTokenSource.Token
+        );
+});
+```
+
+3. Or overriding the global token on the `Try` and/or `Catch`
+```csharp
+var cancellationTokenSource = new CancellationTokenSource();
+var candidateCancellationTokenSource = new CancellationTokenSource();
+
+await scientist.ExperimentAsync<int>(experimentName, experiment =>
+{
+    experiment.WithCancellationToken(cancellationTokenSource.Token);
+    experiment.Use(async () => await StartRunningSomething(myData));
+    experiment.Try(
+        "candidate",
+        async () => await AnAsyncMethod(myData),
+        candidateCancellationTokenSource.Token
+        );
+});
+```
+
 ### Testing
 
 When running your test suite, it's helpful to know that the experimental results always match. To help with testing, Scientist has a `ThrowOnMismatches` property that can be set to `true`. Only do this in your test suite!

Scientist.net.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 15
-VisualStudioVersion = 15.0.26730.16
+# Visual Studio Version 18
+VisualStudioVersion = 18.0.11217.181 d18.0
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Meta", "Meta", "{0FF544BE-E75C-4EF7-AEB3-A534ED5D7BB5}"
 	ProjectSection(SolutionItems) = preProject

src/Scientist/Candidate.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace GitHub
+{
+    public class Candidate<T>
+    {
+        public Candidate(Func<Task<T>> behavior, CancellationToken cancellationToken = default)
+        {
+            Behavior = behavior;
+            CancellationToken = cancellationToken;
+        }
+        public Func<Task<T>> Behavior { get; }
+        public CancellationToken CancellationToken { get; }
+    }
+}

src/Scientist/IExperiment.cs

@@ -1,6 +1,7 @@
 using Github.Ordering;
 using System;
 using System.Collections.Generic;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace GitHub.Internals
@@ -121,20 +122,20 @@ public interface IExperimentAsync<T> : IExperiment
         /// Defines the operation to try.
         /// </summary>
         /// <param name="candidate">The delegate to execute.</param>
-        void Try(Func<Task<T>> candidate);
+        void Try(Func<Task<T>> candidate, CancellationToken cancellationToken = default);
 
         /// <summary>
         /// Defines the operation to try.
         /// </summary>
         /// <param name="name"></param>
         /// <param name="candidate">The delegate to execute.</param>
-        void Try(string name, Func<Task<T>> candidate);
+        void Try(string name, Func<Task<T>> candidate, CancellationToken cancellationToken = default);
 
         /// <summary>
         /// Defines the operation to actually use.
         /// </summary>
         /// <param name="control">The delegate to execute.</param>
-        void Use(Func<Task<T>> control);
+        void Use(Func<Task<T>> control, CancellationToken cancellationToken = default   );
 
         /// <summary>
         /// Defines a func used to compare results.
@@ -152,6 +153,12 @@ public interface IExperimentAsync<T> : IExperiment
         /// </summary>
         /// <param name="customOrdering">The delgate to execute.</param>
         void UseCustomOrdering(CustomOrderer<T> customOrdering);
+
+        /// <summary>
+        /// Defines the global cancellation token to use
+        /// </summary>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        void WithCancellationToken(CancellationToken cancellationToken = default);
     }
 
     /// <summary>

src/Scientist/Internals/Experiment.cs

@@ -1,6 +1,7 @@
 using Github.Ordering;
 using System;
 using System.Collections.Generic;
+using System.Threading;
 using System.Threading.Tasks;
 
 namespace GitHub.Internals
@@ -16,9 +17,9 @@ private static readonly Action<Operation, Exception> _alwaysThrow
         private string _name;
         private int _concurrentTasks;
 
-        private Func<Task<T>> _control;
+        private Candidate<T> _control;
 
-        private readonly Dictionary<string, Func<Task<T>>> _candidates;
+        private readonly Dictionary<string, Candidate<T>> _candidates;
         private Func<T, TClean> _cleaner;
         private Func<T, T, bool> _comparison = DefaultComparison;
         private Func<Task> _beforeRun;
@@ -31,6 +32,8 @@ private static readonly Action<Operation, Exception> _alwaysThrow
 
         private CustomOrderer<T> _customOrderer = behaviors => Task.FromResult(Ordering.Random(behaviors));
 
+        private CancellationToken _cancellationToken = default;
+
         public Experiment(string name, Func<Task<bool>> enabled, int concurrentTasks, IResultPublisher resultPublisher)
         {
             if (concurrentTasks <= 0)
@@ -40,7 +43,7 @@ public Experiment(string name, Func<Task<bool>> enabled, int concurrentTasks, IR
                 throw new ArgumentNullException("A result publisher must be specified", nameof(resultPublisher));
 
             _name = name;
-            _candidates = new Dictionary<string, Func<Task<T>>>();
+            _candidates = new Dictionary<string, Candidate<T>>();
             _enabled = enabled;
             _concurrentTasks = concurrentTasks;
             _resultPublisher = resultPublisher;
@@ -60,20 +63,30 @@ public void RunIf(Func<bool> block) =>
         public void Thrown(Action<Operation, Exception> block) =>
             _thrown = block;
 
-        public void Use(Func<Task<T>> control) =>
-            _control = control;
+        public void Use(Func<Task<T>> control, CancellationToken cancellationToken = default)
+        {
+            var tokenToUse = cancellationToken == default
+              ? _cancellationToken
+              : cancellationToken;
+            _control = new Candidate<T>(control, tokenToUse);
+        }
 
         public void Use(Func<T> control) =>
-            _control = () => Task.FromResult(control());
+            _control = new Candidate<T>(() => Task.FromResult(control()));
 
-        public void Try(Func<Task<T>> candidate)
+        public void Try(Func<Task<T>> candidate, CancellationToken cancellationToken = default)
         {
             if (_candidates.ContainsKey(CandidateExperimentName))
             {
                 throw new InvalidOperationException(
                     "You have already added a default try. Give this candidate a new name with the Try(string, Func<Task<T>>) overload");
             }
-            _candidates.Add(CandidateExperimentName, candidate);
+
+            var tokenToUse = cancellationToken == default
+                ? _cancellationToken
+                : cancellationToken;
+
+            _candidates.Add(CandidateExperimentName, new Candidate<T>(candidate, tokenToUse));
         }
 
         public void Try(Func<T> candidate)
@@ -83,17 +96,22 @@ public void Try(Func<T> candidate)
                 throw new InvalidOperationException(
                     "You have already added a default try. Give this candidate a new name with the Try(string, Func<Task<T>>) overload");
             }
-            _candidates.Add(CandidateExperimentName, () => Task.FromResult(candidate()));
+            _candidates.Add(CandidateExperimentName, new Candidate<T>(() => Task.FromResult(candidate())));
         }
 
-        public void Try(string name, Func<Task<T>> candidate)
+        public void Try(string name, Func<Task<T>> candidate, CancellationToken cancellationToken = default)
         {
             if (_candidates.ContainsKey(name))
             {
                 throw new InvalidOperationException(
                     $"You already have a candidate named {name}. Provide a different name for this test.");
             }
-            _candidates.Add(name, candidate);
+
+            var tokenToUse = cancellationToken == default
+                ? _cancellationToken
+                : cancellationToken;
+
+            _candidates.Add(name, new Candidate<T>(candidate, tokenToUse));
         }
 
         public void Try(string name, Func<T> candidate)
@@ -103,7 +121,7 @@ public void Try(string name, Func<T> candidate)
                 throw new InvalidOperationException(
                     $"You already have a candidate named {name}. Provide a different name for this test.");
             }
-            _candidates.Add(name, () => Task.FromResult(candidate()));
+            _candidates.Add(name, new Candidate<T>(() => Task.FromResult(candidate())));
         }
 
         public void Ignore(Func<T, T, bool> block) =>
@@ -134,7 +152,8 @@ internal ExperimentInstance<T, TClean> Build() =>
                 Thrown = _thrown,
                 ThrowOnMismatches = ThrowOnMismatches,
                 ResultPublisher = _resultPublisher,
-                CustomOrderer = _customOrderer
+                CustomOrderer = _customOrderer,
+                CancellationToken = _cancellationToken,
             });
 
         public void Compare(Func<T, T, bool> comparison)
@@ -161,7 +180,6 @@ public void BeforeRun(Func<Task> action)
             _beforeRun = action;
         }
 
-
         public void UseCustomOrdering(Func<IReadOnlyList<INamedBehavior<T>>, IReadOnlyList<INamedBehavior<T>>> customOrdering)
         {
             _customOrderer = list => Task.FromResult(customOrdering(list));
@@ -171,5 +189,10 @@ public void UseCustomOrdering(CustomOrderer<T> customOrdering)
         {
             _customOrderer = customOrdering;
         }
+
+        public void WithCancellationToken(CancellationToken cancellationToken)
+        {
+            _cancellationToken = cancellationToken;
+        }
     }
 }