Merge pull request #930 from SciML/record_broken_tests

[v14 - Ready] Go through broken tests

Author: TorkelE

Diff for: test/dsl/dsl_options.jl

@@ -488,12 +488,12 @@ let
     @test sol[:Y][end] ≈ 3.0
 
     # Tests that observables can be used for plot indexing.
-    @test_broken false # plot(sol; idxs=X).series_list[1].plotattributes[:y][end] ≈ 10.0
+    plot(sol; idxs=X).series_list[1].plotattributes[:y][end] ≈ 10.0
     @test plot(sol; idxs=rn.X).series_list[1].plotattributes[:y][end] ≈ 10.0
     @test plot(sol; idxs=:X).series_list[1].plotattributes[:y][end] ≈ 10.0
     @test plot(sol; idxs=[X, Y]).series_list[2].plotattributes[:y][end] ≈ 3.0
     @test plot(sol; idxs=[rn.X, rn.Y]).series_list[2].plotattributes[:y][end] ≈ 3.0
-    @test_broken false # plot(sol; idxs=[:X, :Y]).series_list[2].plotattributes[:y][end] ≈ 3.0
+    @test_broken plot(sol; idxs=[:X, :Y]).series_list[2].plotattributes[:y][end] ≈ 3.0 # (https://github.com/SciML/ModelingToolkit.jl/issues/2778)
 end
 
 # Compares programmatic and DSL system with observables.

Diff for: test/reactionsystem_core/coupled_equation_crn_systems.jl

@@ -293,7 +293,7 @@ end
 # Checks for both differential and algebraic equations.
 # Checks for problems, integrators, and solutions yielded by coupled systems.
 # Checks that metadata, types, and default values are carried through correctly.
-@test_broken let # SDEs are currently broken with structural simplify.
+@test_broken let # SDEs are currently broken with structural simplify (https://github.com/SciML/ModelingToolkit.jl/issues/2614).
     # Creates the model
     @parameters a1 [description="Parameter a1"] a2::Rational{Int64} a3=0.3 a4::Rational{Int64}=4//10 [description="Parameter a4"]
     @parameters b1 [description="Parameter b1"] b2::Int64 b3 = 3 b4::Int64=4 [description="Parameter b4"]
@@ -557,15 +557,12 @@ let
     @test osol[B][end] ≈ 1.0
 
     # Checks that SteadyState simulation of the system achieves the correct steady state.
-    # Currently broken due to MTK.
-    @test_broken begin
-        ssprob = SteadyStateProblem(coupled_rs, u0, ps; structural_simplify = true)
-        sssol = solve(oprob, DynamicSS(Vern7()); abstol = 1e-8, reltol = 1e-8)
-        @test osol[X][end] ≈ 2.0
-        @test osol[A][end] ≈ 0.0 atol = 1e-8
-        @test osol[D(A)][end] ≈ 0.0 atol = 1e-8
-        @test osol[B][end] ≈ 1.0
-    end
+    ssprob = SteadyStateProblem(coupled_rs, u0, ps; structural_simplify = true)
+    sssol = solve(ssprob, DynamicSS(Vern7()); abstol = 1e-8, reltol = 1e-8)
+    @test sssol[X][end] ≈ 2.0
+    @test sssol[A][end] ≈ 0.0 atol = 1e-8
+    @test sssol[D(A)][end] ≈ 0.0 atol = 1e-8
+    @test sssol[B][end] ≈ 1.0
 
     # Checks that the steady state can be found by solving a nonlinear problem.
     # Here `B => 0.1` has to be provided as well (and it shouldn't for the 2nd order ODE), hence the

Diff for: test/reactionsystem_core/events.jl

@@ -158,7 +158,7 @@ let
     ]
 
     # Declares various misformatted events .
-    # Relevant MTK issue regarding misformatted events not throwing an early error https://github.com/SciML/ModelingToolkit.jl/issues/2612.
+    @test_broken false # Some misformatted tests should throw error at this stage, but does not (https://github.com/SciML/ModelingToolkit.jl/issues/2612).
     continuous_events_bad = [
         X ~ 1.0 => [X ~ 0.5],       # Scalar condition.
         [X ~ 1.0] => X ~ 0.5,       # Scalar affect.
@@ -362,9 +362,9 @@ let
     sol = solve(jprob, SSAStepper(); seed)
 
     # Checks that all `e` parameters have been updated properly.
-    # Note that periodic discrete events are currently broken for jump processes.
+    # Note that periodic discrete events are currently broken for jump processes (and unlikely to be fixed soon due to periodic callbacks using the internals of ODE integrator and Datastructures heap implementations).
     @test sol.ps[:e1] == 1
-    @test_broken sol.ps[:e2] == 1
+    @test_broken sol.ps[:e2] == 1 # (https://github.com/SciML/JumpProcesses.jl/issues/417)
     @test sol.ps[:e3] == 1
 end
 
@@ -424,21 +424,22 @@ let
     osol_events = solve(oprob_events, Tsit5())
     @test osol == osol_events
 
-    # Checks for SDE simulations.
+    # Checks for SDE simulations (note, non-seed dependant test should be created instead).
     sprob = SDEProblem(rn, u0, tspan, ps)
     sprob_events = SDEProblem(rn_events, u0, tspan, ps)
     ssol = solve(sprob, ImplicitEM(); seed, callback)
     ssol_events = solve(sprob_events, ImplicitEM(); seed)
     @test ssol == ssol_events
 
-    # Checks for Jump simulations.
+    # Checks for Jump simulations. (note, non-seed dependant test should be created instead)
+    # Note that periodic discrete events are currently broken for jump processes (and unlikely to be fixed soon due to have events are implemented).
     callback = CallbackSet(cb_disc_1, cb_disc_2, cb_disc_3)
     dprob = DiscreteProblem(rn, u0, tspan, ps)
     dprob_events = DiscreteProblem(rn_dics_events, u0, tspan, ps)
     jprob = JumpProblem(rn, dprob, Direct(); rng)
     jprob_events = JumpProblem(rn_dics_events, dprob_events, Direct(); rng)
     sol = solve(jprob, SSAStepper(); seed, callback)
-    @test_broken let # Broken due to. Even if fixed, seeding might not work due to events.
+    @test_broken let # (https://github.com/SciML/JumpProcesses.jl/issues/417)
         sol_events = solve(jprob_events, SSAStepper(); seed)
         @test sol == sol_events
     end

Diff for: test/reactionsystem_core/parameter_type_designation.jl

@@ -88,7 +88,7 @@ let
     nsol = solve(nprob, NewtonRaphson())
 
     # Checks all stored parameters.
-    for mtk_struct in [oprob, sprob, dprob, jprob, nprob, oinit, sinit, jinit, osol, ssol, jsol, nsol]
+    for mtk_struct in [oprob, sprob, dprob, jprob, nprob, oinit, sinit, jinit, ninit, osol, ssol, jsol, nsol]
         # Checks that all parameters have the correct type.
         @test unwrap(mtk_struct.ps[p1]) isa Float64
         @test unwrap(mtk_struct.ps[d1]) isa Float64
@@ -114,8 +114,8 @@ let
         @test unwrap(mtk_struct.ps[d5]) == Float32(1.5)
     end
 
-    # Checks all stored variables.
-    for mtk_struct in [oprob, sprob, dprob, jprob, nprob, oinit, sinit, jinit]
+    # Checks all stored variables (these should always be `Float64`).
+    for mtk_struct in [oprob, sprob, dprob, jprob, nprob, oinit, sinit, jinit, ninit]
         # Checks that all variables have the correct type.
         @test unwrap(mtk_struct[X1]) isa Float64
         @test unwrap(mtk_struct[X2]) isa Float64
@@ -130,10 +130,4 @@ let
         @test unwrap(mtk_struct[X4]) == 0.4
         @test unwrap(mtk_struct[X5]) == 0.5
     end
-
-    # This test started working now, probably due to a MTK fix. Need to look at where to put it
-    # back into the test properly though.
-    @test_broken false 
-    # Indexing currently broken for NonlinearSystem integrators (MTK intend to support this though).
-    @test unwrap(ninit.ps[p1]) isa Float64
 end

Diff for: test/reactionsystem_core/reactionsystem.jl

@@ -750,5 +750,5 @@ end
 # there are several places in the code where the `reactionsystem_uptodate` function is called, here
 # the code might need adaptation to take the updated reaction system into account.
 let
-    @test_nowarn  Catalyst.reactionsystem_uptodate_check()
+    @test_nowarn Catalyst.reactionsystem_uptodate_check()
 end

Diff for: test/simulation_and_solving/simulate_SDEs.jl

@@ -224,13 +224,14 @@ let
     @species X1(t) X2(t)
     p_syms = @parameters $(η_stored) k1 k2
 
-    r1 = Reaction(k1,[X1],[X2],[1],[1]; metadata = [:noise_scaling => η_stored])
-    r2 = Reaction(k2,[X2],[X1],[1],[1]; metadata = [:noise_scaling => η_stored])
+    r1 = Reaction(k1, [X1], [X2], [1], [1]; metadata = [:noise_scaling => p_syms[1]])
+    r2 = Reaction(k2, [X2], [X1], [1], [1]; metadata = [:noise_scaling => p_syms[1]])
     @named noise_scaling_network = ReactionSystem([r1, r2], t, [X1, X2], [k1, k2, p_syms[1]])
+    noise_scaling_network = complete(noise_scaling_network)
 
     u0 = [:X1 => 1100.0, :X2 => 3900.0]
     p = [:k1 => 2.0, :k2 => 0.5, :η => 0.0]
-    @test_broken SDEProblem(noise_scaling_network, u0, (0.0, 1000.0), p).ps[:η] == 0.0 # Broken due to SII/MTK stuff.
+    @test SDEProblem(noise_scaling_network, u0, (0.0, 1000.0), p).ps[:η] == 0.0
 end
 
 # Complicated test with many combinations of options.

Diff for: test/spatial_modelling/lattice_reaction_systems.jl

@@ -280,6 +280,7 @@ end
 # Currently not supported. Won't be until the LatticeReactionSystem internal update is merged.
 
 # Checks that parameter types designated in the non-spatial `ReactionSystem` is handled correctly.
+# Broken lattice tests have local branches that fixes them.
 @test_broken let
     # Declares LatticeReactionSystem with designated parameter types.
     rs = @reaction_network begin

Diff for: test/upstream/mtk_problem_inputs.jl

@@ -172,6 +172,11 @@ let
     @species X3(t)
     @parameters k3
 
+    # Creates systems (so these are not recreated in each problem call).
+    osys = convert(ODESystem, rn)
+    ssys = convert(SDESystem, rn)
+    nsys = convert(NonlinearSystem, rn)
+
     # Declares valid initial conditions and parameter values
     u0_valid = [X1 => 1, X2 => 2]
     ps_valid = [k1 => 0.5, k2 => 0.1]
@@ -218,25 +223,25 @@ let
     ]
 
     # Loops through all potential parameter sets, checking their inputs yield errors.
+    # Broken tests are due to this issue: https://github.com/SciML/ModelingToolkit.jl/issues/2779
     for ps in [ps_valid; ps_invalid], u0 in [u0_valid; u0s_invalid]
         # Handles problems with/without tspan separately. Special check ensuring that valid inputs passes.
-        for XProblem in [ODEProblem, SDEProblem, DiscreteProblem]
+        for (xsys, XProblem) in zip([osys, ssys, rn], [ODEProblem, SDEProblem, DiscreteProblem])
             if (ps == ps_valid) && (u0 == u0_valid)
-                XProblem(rn, u0, (0.0, 1.0), ps); @test true;
+                XProblem(xsys, u0, (0.0, 1.0), ps); @test true;
             else
-                # Several of these cases do not throw errors (https://github.com/SciML/ModelingToolkit.jl/issues/2624).
                 @test_broken false
                 continue
-                @test_throws Exception XProblem(rn, u0, (0.0, 1.0), ps)
+                @test_throws Exception XProblem(xsys, u0, (0.0, 1.0), ps)
             end
         end
-        for XProblem in [NonlinearProblem, SteadyStateProblem]
+        for (xsys, XProblem) in zip([nsys, osys], [NonlinearProblem, SteadyStateProblem])
             if (ps == ps_valid) && (u0 == u0_valid)
-                XProblem(rn, u0, ps); @test true;
+                XProblem(xsys, u0, ps); @test true;
             else
                 @test_broken false
                 continue
-                @test_throws Exception XProblem(rn, u0, ps)
+                @test_throws Exception XProblem(xsys, u0, ps)
             end
         end
     end

Diff for: test/upstream/mtk_structure_indexing.jl

@@ -50,7 +50,7 @@ begin
     sint = init(sprob, ImplicitEM(); save_everystep=false)
     jint = init(jprob, SSAStepper())
     nint = init(nprob, NewtonRaphson(); save_everystep=false)
-    @test_broken ssint = init(ssprob, DynamicSS(Tsit5()); save_everystep=false) # https://github.com/SciML/SciMLBase.jl/issues/660
+    @test_broken ssint = init(ssprob, DynamicSS(Tsit5()); save_everystep=false) # https://github.com/SciML/SteadyStateDiffEq.jl/issues/79
     integrators = [oint, sint, jint, nint]
 
     # Creates solutions.
@@ -64,14 +64,12 @@ end
 
 # Tests problem indexing and updating.
 let 
-    @test_broken false # A few cases fails for SteadyStateProblem: https://github.com/SciML/SciMLBase.jl/issues/660
-    @test_broken false # Most cases broken for Ensemble problems: https://github.com/SciML/SciMLBase.jl/issues/661
-    for prob in deepcopy(problems[1:end-1])
+    for prob in [deepcopy(problems); deepcopy(eproblems)]
         # Get u values (including observables).
         @test prob[X] == prob[model.X] == prob[:X] == 4
         @test prob[XY] == prob[model.XY] == prob[:XY] == 9
         @test prob[[XY,Y]] == prob[[model.XY,model.Y]] == prob[[:XY,:Y]] == [9, 5]
-        @test_broken prob[(XY,Y)] == prob[(model.XY,model.Y)] == prob[(:XY,:Y)] == (9, 5)
+        @test_broken prob[(XY,Y)] == prob[(model.XY,model.Y)] == prob[(:XY,:Y)] == (9, 5) # https://github.com/SciML/SciMLBase.jl/issues/709
         @test getu(prob, X)(prob) == getu(prob, model.X)(prob) == getu(prob, :X)(prob) == 4
         @test getu(prob, XY)(prob) == getu(prob, model.XY)(prob) == getu(prob, :XY)(prob) == 9 
         @test getu(prob, [XY,Y])(prob) == getu(prob, [model.XY,model.Y])(prob) == getu(prob, [:XY,:Y])(prob) == [9, 5]  
@@ -117,8 +115,7 @@ end
 
 # Test remake function.
 let 
-    @test_broken false # Currently cannot be run for Ensemble problems: https://github.com/SciML/SciMLBase.jl/issues/661 (as indexing cannot be used to check values).
-    for prob in deepcopy(problems)
+    for prob in [deepcopy(problems); deepcopy(eproblems)]
         # Remake for all u0s.
         rp = remake(prob; u0 = [X => 1, Y => 2])
         @test rp[[X, Y]] == [1, 2]
@@ -156,9 +153,7 @@ end
 # Test integrator indexing.
 let 
     @test_broken false # NOTE: Multiple problems for `nint` (https://github.com/SciML/SciMLBase.jl/issues/662).
-    @test_broken false # NOTE: Multiple problems for `jint` (https://github.com/SciML/SciMLBase.jl/issues/654).
-    @test_broken false # NOTE: Cannot even create a `ssint` (https://github.com/SciML/SciMLBase.jl/issues/660).
-    for int in deepcopy([oint, sint])
+    for int in deepcopy([oint, sint, jint])
         # Get u values.
         @test int[X] == int[model.X] == int[:X] == 4
         @test int[XY] == int[model.XY] == int[:XY] == 9
@@ -208,6 +203,7 @@ let
 end
 
 # Test solve's save_idxs argument.
+# Currently, `save_idxs` is broken with symbolic stuff (https://github.com/SciML/ModelingToolkit.jl/issues/1761).
 let 
     for (prob, solver) in zip(deepcopy([oprob, sprob, jprob]), [Tsit5(), ImplicitEM(), SSAStepper()])
         # Save single variable
@@ -241,10 +237,10 @@ let
         @test getu(sol, (XY,Y))(sol)[1] == getu(sol, (model.XY,model.Y))(sol)[1] == getu(sol, (:XY,:Y))(sol)[1] == (9, 5)       
 
         # Get u values via idxs and functional call.
-        @test osol(0.0; idxs=X) == osol(0.0; idxs=X) == osol(0.0; idxs=X) == 4
-        @test osol(0.0; idxs=XY) == osol(0.0; idxs=XY) == osol(0.0; idxs=XY) == 9
-        @test_broken osol(0.0; idxs=[model.Y,model.XY]) == osol(0.0; idxs=[model.Y,model.XY]) == osol(0.0; idxs=[model.XY,model.X]) == [9, 5]
-        @test_broken osol(0.0; idxs=(:Y,:XY)) == osol(0.0; idxs=(:Y,:XY)) == osol(0.0; idxs=(:XY,:Y)) == (9, 5)
+        @test osol(0.0; idxs=X) == osol(0.0; idxs=model.X) == osol(0.0; idxs=:X) == 4
+        @test osol(0.0; idxs=XY) == osol(0.0; idxs=model.XY) == osol(0.0; idxs=:XY) == 9
+        @test osol(0.0; idxs = [XY,Y]) == osol(0.0; idxs = [model.XY,model.Y]) == osol(0.0; idxs = [:XY,:Y]) == [9, 5]
+        @test_broken osol(0.0; idxs = (XY,Y)) == osol(0.0; idxs = (model.XY,model.Y)) == osol(0.0; idxs = (:XY,:Y)) == (9, 5) # https://github.com/SciML/SciMLBase.jl/issues/711
 
         # Get p values.
         @test sol.ps[kp] == sol.ps[model.kp] == sol.ps[:kp] == 1.0    
@@ -262,11 +258,11 @@ let
             @test sol[X] == sol[model.X] == sol[:X]
             @test sol[XY] == sol[model.XY][1] == sol[:XY]
             @test sol[[XY,Y]] == sol[[model.XY,model.Y]] == sol[[:XY,:Y]]
-            @test_broken sol[(XY,Y)] == sol[(model.XY,model.Y)] == sol[(:XY,:Y)]
+            @test_broken sol[(XY,Y)] == sol[(model.XY,model.Y)] == sol[(:XY,:Y)] # https://github.com/SciML/SciMLBase.jl/issues/710
             @test getu(sol, X)(sol) == getu(sol, model.X)(sol)[1] == getu(sol, :X)(sol)
             @test getu(sol, XY)(sol) == getu(sol, model.XY)(sol)[1] == getu(sol, :XY)(sol)
             @test getu(sol, [XY,Y])(sol) == getu(sol, [model.XY,model.Y])(sol) == getu(sol, [:XY,:Y])(sol)
-            @test_broken getu(sol, (XY,Y))(sol) == getu(sol, (model.XY,model.Y))(sol) == getu(sol, (:XY,:Y))(sol)[1]   
+            @test_broken getu(sol, (XY,Y))(sol) == getu(sol, (model.XY,model.Y))(sol) == getu(sol, (:XY,:Y))(sol)[1] # https://github.com/SciML/SciMLBase.jl/issues/710
 
             # Get p values.
             @test sol.ps[kp] == sol.ps[model.kp] == sol.ps[:kp]
@@ -281,8 +277,7 @@ end
 
 # Tests plotting.
 let 
-    @test_broken false # Currently broken for `ssol` (https://github.com/SciML/SciMLBase.jl/issues/580) 
-    for sol in deepcopy([osol, jsol])
+    for sol in deepcopy([osol, ssol, jsol])
         # Single variable.
         @test length(plot(sol; idxs = X).series_list) == 1
         @test length(plot(sol; idxs = XY).series_list) == 1
@@ -386,4 +381,5 @@ let
     @test jint.cb.condition.ma_jumps.scaled_rates[1] == 16.0
     reset_aggregated_jumps!(jint)
     @test jint.cb.condition.ma_jumps.scaled_rates[1] == 6.0
-end
+end
+