From a7c55ebbfbebbccc52295c7f683241ee4186a96c Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Tue, 26 Mar 2024 21:22:13 +0100 Subject: [PATCH 1/5] Start replacing tests with DifferentiationInterface --- .github/workflows/ci.yml | 80 +++++++++---------- .../BenchmarkTrigger.yml | 0 .../CompatHelper.yml | 0 .../Downstream.yml | 0 .../TagBot.yml | 0 .../clean_preview.yml | 0 .../pr_comment.yml | 0 Project.toml | 3 +- test/runtests.jl | 10 ++- test/softmax.jl | 39 ++++++--- 10 files changed, 80 insertions(+), 52 deletions(-) rename .github/{workflows => workflows_deactivated}/BenchmarkTrigger.yml (100%) rename .github/{workflows => workflows_deactivated}/CompatHelper.yml (100%) rename .github/{workflows => workflows_deactivated}/Downstream.yml (100%) rename .github/{workflows => workflows_deactivated}/TagBot.yml (100%) rename .github/{workflows => workflows_deactivated}/clean_preview.yml (100%) rename .github/{workflows => workflows_deactivated}/pr_comment.yml (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7620fb6b..a1f450f02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: version: # - '1.9' # uncomment when julia 1.10 is out - '1' # automatically expands to the latest stable 1.x release of Julia - - 'nightly' + # - 'nightly' os: - ubuntu-latest # - macOS-latest @@ -40,19 +40,19 @@ jobs: julia-threads: - '1' - include: - - os: windows-latest - version: '1' - arch: x64 - julia-threads: '1' - - os: macOS-latest - version: '1' - arch: x64 - julia-threads: '1' - - os: ubuntu-latest - version: '1' - arch: x64 - julia-threads: '2' + # include: + # - os: windows-latest + # version: '1' + # arch: x64 + # julia-threads: '1' + # - os: macOS-latest + # version: '1' + # arch: x64 + # julia-threads: '1' + # - os: ubuntu-latest + # version: '1' + # arch: x64 + # julia-threads: '2' steps: - uses: actions/checkout@v3 @@ -79,29 +79,29 @@ jobs: with: file: lcov.info - docs: - name: Documentation - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: julia-actions/setup-julia@v1 - with: - version: '1.9' - - uses: julia-actions/cache@v1 - - run: | - julia --project=docs -e ' - using Pkg - Pkg.develop(PackageSpec(path=pwd())) - Pkg.instantiate()' - - run: | - julia --color=yes --project=docs/ -e ' - using NNlib - # using Pkg; Pkg.activate("docs") - using Documenter - using Documenter: doctest - DocMeta.setdocmeta!(NNlib, :DocTestSetup, :(using NNlib); recursive=true) - doctest(NNlib)' - - run: julia --project=docs docs/make.jl - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} + # docs: + # name: Documentation + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - uses: julia-actions/setup-julia@v1 + # with: + # version: '1.9' + # - uses: julia-actions/cache@v1 + # - run: | + # julia --project=docs -e ' + # using Pkg + # Pkg.develop(PackageSpec(path=pwd())) + # Pkg.instantiate()' + # - run: | + # julia --color=yes --project=docs/ -e ' + # using NNlib + # # using Pkg; Pkg.activate("docs") + # using Documenter + # using Documenter: doctest + # DocMeta.setdocmeta!(NNlib, :DocTestSetup, :(using NNlib); recursive=true) + # doctest(NNlib)' + # - run: julia --project=docs docs/make.jl + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} diff --git a/.github/workflows/BenchmarkTrigger.yml b/.github/workflows_deactivated/BenchmarkTrigger.yml similarity index 100% rename from .github/workflows/BenchmarkTrigger.yml rename to .github/workflows_deactivated/BenchmarkTrigger.yml diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows_deactivated/CompatHelper.yml similarity index 100% rename from .github/workflows/CompatHelper.yml rename to .github/workflows_deactivated/CompatHelper.yml diff --git a/.github/workflows/Downstream.yml b/.github/workflows_deactivated/Downstream.yml similarity index 100% rename from .github/workflows/Downstream.yml rename to .github/workflows_deactivated/Downstream.yml diff --git a/.github/workflows/TagBot.yml b/.github/workflows_deactivated/TagBot.yml similarity index 100% rename from .github/workflows/TagBot.yml rename to .github/workflows_deactivated/TagBot.yml diff --git a/.github/workflows/clean_preview.yml b/.github/workflows_deactivated/clean_preview.yml similarity index 100% rename from .github/workflows/clean_preview.yml rename to .github/workflows_deactivated/clean_preview.yml diff --git a/.github/workflows/pr_comment.yml b/.github/workflows_deactivated/pr_comment.yml similarity index 100% rename from .github/workflows/pr_comment.yml rename to .github/workflows_deactivated/pr_comment.yml diff --git a/Project.toml b/Project.toml index 114bf3e2a..a0c945502 100644 --- a/Project.toml +++ b/Project.toml @@ -44,6 +44,7 @@ cuDNN = "1" julia = "1.9" [extras] +ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" ChainRulesTestUtils = "cdddcdb0-9152-4a09-a978-84456f9df70a" @@ -64,4 +65,4 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" [targets] -test = ["AMDGPU", "CUDA", "ChainRulesTestUtils", "Documenter", "FiniteDifferences", "ForwardDiff", "Logging", "ReverseDiff", "StableRNGs", "Test", "UnicodePlots", "Zygote", "cuDNN", "Enzyme", "EnzymeCore", "EnzymeTestUtils", "Interpolations", "ImageTransformations"] +test = ["ADTypes", "AMDGPU", "CUDA", "ChainRulesTestUtils", "Documenter", "FiniteDifferences", "ForwardDiff", "Logging", "ReverseDiff", "StableRNGs", "Test", "UnicodePlots", "Zygote", "cuDNN", "Enzyme", "EnzymeCore", "EnzymeTestUtils", "Interpolations", "ImageTransformations"] diff --git a/test/runtests.jl b/test/runtests.jl index 9434263f8..1fcab7d7e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -42,10 +42,12 @@ end cpu(x) = adapt(CPU(), x) +#= include("gather.jl") include("scatter.jl") include("upsample.jl") include("rotation.jl") +=# function nnlib_testsuite(Backend; skip_tests = Set{String}()) @conditional_testset "Upsample" skip_tests begin @@ -65,7 +67,8 @@ end @testset verbose=true "NNlib.jl" begin if get(ENV, "NNLIB_TEST_CPU", "true") == "true" - @testset "CPU" begin + @testset verbose=true "CPU" begin + #= @testset "Doctests" begin doctest(NNlib, manual=false) end @@ -113,11 +116,13 @@ end @testset "Padding" begin include("padding.jl") end + =# - @testset "Softmax" begin + @testset verbose=true "Softmax" begin include("softmax.jl") end + #= @testset "Utilities" begin include("utils.jl") end @@ -129,6 +134,7 @@ end @testset "Functions" begin include("functions.jl") end + =# end else @info "Skipping CPU tests, set NNLIB_TEST_CPU=true to run them." diff --git a/test/softmax.jl b/test/softmax.jl index 1d5105af6..a7d89f7b2 100644 --- a/test/softmax.jl +++ b/test/softmax.jl @@ -1,3 +1,12 @@ +using Pkg +Pkg.add(url="https://github.com/gdalle/DifferentiationInterface.jl") +Pkg.add(url="https://github.com/gdalle/DifferentiationInterface.jl", subdir="lib/DifferentiationInterfaceTest") + +import ADTypes +import DifferentiationInterface as DI +using DifferentiationInterfaceTest: Scenario +import DifferentiationInterfaceTest as DIT + using Statistics: mean using NNlib: ∇softmax_data, ∇logsoftmax_data @@ -110,21 +119,33 @@ end @test logsumexp(x; dims = 1) ≈ flogsoft(x, dims = 1) end -@testset "AutoDiff" begin +@testset verbose=true "AutoDiff" begin + scenarios = Scenario[] + for f in (softmax, logsoftmax), d in (:, 1, 2) - gradtest(f, (3,4); fkwargs = (dims = d,), check_rrule = true) + push!(scenarios, Scenario(x -> f(x; dims=d), x=randn(3,4))) end - gradtest(x -> softmax(x) .* (1:3), 3) - gradtest(x -> softmax(x) .* (1:3), (3,5), atol = 1e-4) - gradtest(x -> softmax(x, dims = 2) .* (1:3), (3,5), atol = 1e-4) + push!(scenarios, Scenario(x -> softmax(x) .* (1:3), x=randn(3))) + push!(scenarios, Scenario(x -> softmax(x) .* (1:3), x=randn(3,5))) + push!(scenarios, Scenario(x -> softmax(x, dims = 2) .* (1:3), x=randn(3,5))) - gradtest(x -> logsoftmax(x) .* (1:3), 3) - gradtest(x -> logsoftmax(x) .* (1:3), (3,5)) - gradtest(x -> logsoftmax(x, dims = 2) .* (1:3), (3,5)) + push!(scenarios, Scenario(x -> logsoftmax(x) .* (1:3), x=randn(3))) + push!(scenarios, Scenario(x -> logsoftmax(x) .* (1:3), x=randn(3,5))) + push!(scenarios, Scenario(x -> logsoftmax(x, dims = 2) .* (1:3), x=randn(3,5))) for d in (:, 1, 2) - gradtest(logsumexp, (3,4), fkwargs = (dims = d,)) + push!(scenarios, Scenario(x -> logsumexp(x; dims=d), x=randn(3,4))) end + + DIT.test_differentiation( + ADTypes.AutoZygote(), + [DI.gradient], + scenarios; + correctness=ADTypes.AutoFiniteDifferences( + FiniteDifferences.central_fdm(5, 1) + ), + detailed=true, + ) end @testset "Second derivatives" begin From 8c963b9b179cd7465fa236f2faf295020433ed2d Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Tue, 26 Mar 2024 21:29:16 +0100 Subject: [PATCH 2/5] Set rtol --- test/softmax.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/softmax.jl b/test/softmax.jl index a7d89f7b2..50bed5dc6 100644 --- a/test/softmax.jl +++ b/test/softmax.jl @@ -145,6 +145,7 @@ end FiniteDifferences.central_fdm(5, 1) ), detailed=true, + rtol=1e-4, ) end From 2125986e80a4c22ed09b17908c6e7aea530a681e Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Tue, 26 Mar 2024 22:15:22 +0100 Subject: [PATCH 3/5] Add tests for non scalar output --- test/runtests.jl | 9 +++++++++ test/softmax.jl | 11 +---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 1fcab7d7e..afc3f1ccc 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,3 +1,12 @@ +using Pkg +Pkg.add(url="https://github.com/gdalle/DifferentiationInterface.jl") +Pkg.add(url="https://github.com/gdalle/DifferentiationInterface.jl", subdir="lib/DifferentiationInterfaceTest") + +import ADTypes +import DifferentiationInterface as DI +using DifferentiationInterfaceTest: Scenario +import DifferentiationInterfaceTest as DIT + using NNlib, Test, Statistics, Random using ChainRulesCore, ChainRulesTestUtils using Base.Broadcast: broadcasted diff --git a/test/softmax.jl b/test/softmax.jl index 50bed5dc6..23efe9b47 100644 --- a/test/softmax.jl +++ b/test/softmax.jl @@ -1,12 +1,3 @@ -using Pkg -Pkg.add(url="https://github.com/gdalle/DifferentiationInterface.jl") -Pkg.add(url="https://github.com/gdalle/DifferentiationInterface.jl", subdir="lib/DifferentiationInterfaceTest") - -import ADTypes -import DifferentiationInterface as DI -using DifferentiationInterfaceTest: Scenario -import DifferentiationInterfaceTest as DIT - using Statistics: mean using NNlib: ∇softmax_data, ∇logsoftmax_data @@ -139,7 +130,7 @@ end DIT.test_differentiation( ADTypes.AutoZygote(), - [DI.gradient], + [DI.gradient, DI.pullback, DI.jacobian], scenarios; correctness=ADTypes.AutoFiniteDifferences( FiniteDifferences.central_fdm(5, 1) From ada4f258f4d60182c9211adcfeaa5b0690e7a66a Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Tue, 26 Mar 2024 22:22:52 +0100 Subject: [PATCH 4/5] No pullback cause FiniteDifferences doesn't have it --- test/softmax.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/softmax.jl b/test/softmax.jl index 23efe9b47..ff43a85c0 100644 --- a/test/softmax.jl +++ b/test/softmax.jl @@ -130,7 +130,7 @@ end DIT.test_differentiation( ADTypes.AutoZygote(), - [DI.gradient, DI.pullback, DI.jacobian], + [DI.gradient, DI.jacobian], scenarios; correctness=ADTypes.AutoFiniteDifferences( FiniteDifferences.central_fdm(5, 1) From e85eeed7e884e3d105a7d34a233a8ae6b49f19be Mon Sep 17 00:00:00 2001 From: Guillaume Dalle <22795598+gdalle@users.noreply.github.com> Date: Wed, 27 Mar 2024 16:42:40 +0100 Subject: [PATCH 5/5] Now FiniteDifferences backend can handle pullbacks --- test/softmax.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/softmax.jl b/test/softmax.jl index ff43a85c0..bcfa6d6c4 100644 --- a/test/softmax.jl +++ b/test/softmax.jl @@ -130,7 +130,7 @@ end DIT.test_differentiation( ADTypes.AutoZygote(), - [DI.gradient, DI.jacobian], + [DI.gradient, DI.pullback], scenarios; correctness=ADTypes.AutoFiniteDifferences( FiniteDifferences.central_fdm(5, 1)