Merge master, inline retval

.github/dependabot.yml

@@ -0,0 +1,7 @@
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/workflows/Invalidations.yml

@@ -0,0 +1,40 @@
+name: Invalidations
+
+on:
+  pull_request:
+
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: always.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  evaluate:
+    # Only run on PRs to the default branch.
+    # In the PR trigger above branches can be specified only explicitly whereas this check should work for master, main, or any other default branch
+    if: github.base_ref == github.event.repository.default_branch
+    runs-on: ubuntu-latest
+    steps:
+    - uses: julia-actions/setup-julia@v1
+      with:
+        version: '1'
+    - uses: actions/checkout@v3
+    - uses: julia-actions/julia-buildpkg@v1
+    - uses: julia-actions/julia-invalidations@v1
+      id: invs_pr
+
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.repository.default_branch }}
+    - uses: julia-actions/julia-buildpkg@v1
+    - uses: julia-actions/julia-invalidations@v1
+      id: invs_default
+
+    - name: Report invalidation counts
+      run: |
+        echo "Invalidations on default branch: ${{ steps.invs_default.outputs.total }} (${{ steps.invs_default.outputs.deps }} via deps)" >> $GITHUB_STEP_SUMMARY
+        echo "This branch: ${{ steps.invs_pr.outputs.total }} (${{ steps.invs_pr.outputs.deps }} via deps)" >> $GITHUB_STEP_SUMMARY
+    - name: Check if the PR does increase number of invalidations
+      if: steps.invs_pr.outputs.total > steps.invs_default.outputs.total
+      run: exit 1

.github/workflows/ci.yml

@@ -7,6 +7,7 @@ on:
     branches:
       - master
     tags: '*'
+  workflow_dispatch:
 jobs:
   test:
     name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
@@ -16,20 +17,20 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.0'
+          - '1.6'
           - '1'
           - 'nightly'
         os:
           - ubuntu-latest
         arch:
           - x64
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - uses: actions/cache@v1
+      - uses: actions/cache@v3
         env:
           cache-name: cache-artifacts
         with:
@@ -42,14 +43,14 @@ jobs:
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@v3
         with:
           file: lcov.info
   docs:
     name: Documentation
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: julia-actions/setup-julia@v1
         with:
           version: '1'

Project.toml

@@ -1,6 +1,6 @@
 name = "ForwardDiff"
 uuid = "f6369f11-7733-5829-9624-2563aa707210"
-version = "0.10.24"
+version = "0.11-DEV"
 
 [deps]
 CommonSubexpressions = "bbf7d656-a473-5ed7-a52c-81e309532950"
@@ -12,28 +12,37 @@ NaNMath = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
 Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+SIMD = "fdea26ae-647d-5447-a871-4b548cad5224"
 SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 
+[weakdeps]
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
+
+[extensions]
+ForwardDiffStaticArraysExt = "StaticArrays"
+
 [compat]
-Calculus = "0.2, 0.3, 0.4, 0.5"
+Calculus = "0.5"
 CommonSubexpressions = "0.3"
-DiffResults = "0.0.1, 0.0.2, 0.0.3, 0.0.4, 1.0.1"
-DiffRules = "1.4.0"
-DiffTests = "0.0.1, 0.1"
+DiffResults = "1.1"
+DiffRules = "1.4"
+DiffTests = "0.1"
 LogExpFunctions = "0.3"
-NaNMath = "0.2.2, 0.3"
+NaNMath = "1"
 Preferences = "1"
-SpecialFunctions = "0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10, 1.0, 2"
-StaticArrays = "0.8.3, 0.9, 0.10, 0.11, 0.12, 1.0"
-julia = "1"
+SIMD = "3"
+SpecialFunctions = "1, 2"
+StaticArrays = "1.5"
+julia = "1.6"
 
 [extras]
 Calculus = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9"
 DiffTests = "de460e47-3fe3-5279-bb4a-814414816d5d"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Calculus", "DiffTests", "SparseArrays", "Test", "InteractiveUtils"]
+test = ["Calculus", "DiffTests", "SparseArrays", "StaticArrays", "Test", "InteractiveUtils"]

README.md

@@ -8,43 +8,62 @@
 
 ForwardDiff implements methods to take **derivatives**, **gradients**, **Jacobians**, **Hessians**, and higher-order derivatives of native Julia functions (or any callable object, really) using **forward mode automatic differentiation (AD)**.
 
-While performance can vary depending on the functions you evaluate, the algorithms implemented by ForwardDiff **generally outperform non-AD algorithms in both speed and accuracy.**
+While performance can vary depending on the functions you evaluate, the algorithms implemented by ForwardDiff generally outperform non-AD algorithms (such as finite-differencing) in both speed and accuracy.
 
 Here's a simple example showing the package in action:
 
 ```julia
 julia> using ForwardDiff
 
-julia> f(x::Vector) = sum(sin, x) + prod(tan, x) * sum(sqrt, x);
+julia> f(x::Vector) = sin(x[1]) + prod(x[2:end]);  # returns a scalar
 
-julia> x = rand(5) # small size for example's sake
-5-element Array{Float64,1}:
- 0.986403
- 0.140913
- 0.294963
- 0.837125
- 0.650451
+julia> x = vcat(pi/4, 2:4)
+4-element Vector{Float64}:
+ 0.7853981633974483
+ 2.0
+ 3.0
+ 4.0
 
-julia> g = x -> ForwardDiff.gradient(f, x); # g = ∇f
-
-julia> g(x)
-5-element Array{Float64,1}:
- 1.01358
- 2.50014
- 1.72574
- 1.10139
- 1.2445
+julia> ForwardDiff.gradient(f, x)
+4-element Vector{Float64}:
+  0.7071067811865476
+ 12.0
+  8.0
+  6.0
 
 julia> ForwardDiff.hessian(f, x)
-5x5 Array{Float64,2}:
- 0.585111  3.48083  1.7706    0.994057  1.03257
- 3.48083   1.06079  5.79299   3.25245   3.37871
- 1.7706    5.79299  0.423981  1.65416   1.71818
- 0.994057  3.25245  1.65416   0.251396  0.964566
- 1.03257   3.37871  1.71818   0.964566  0.140689
- ```
-
- Trying to switch to the latest version of ForwardDiff? See our [upgrade guide](http://www.juliadiff.org/ForwardDiff.jl/stable/user/upgrade/) for details regarding user-facing changes between releases.
+4×4 Matrix{Float64}:
+ -0.707107  0.0  0.0  0.0
+  0.0       0.0  4.0  3.0
+  0.0       4.0  0.0  2.0
+  0.0       3.0  2.0  0.0
+```
+
+Functions like `f` which map a vector to a scalar are the best case for reverse-mode automatic differentiation,
+but ForwardDiff may still be a good choice if `x` is not too large, as it is much simpler.
+The best case for forward-mode differentiation is a function which maps a scalar to a vector, like this `g`:
+
+```julia
+julia> g(y::Real) = [sin(y), cos(y), tan(y)];  # returns a vector
+
+julia> ForwardDiff.derivative(g, pi/4)
+3-element Vector{Float64}:
+  0.7071067811865476
+ -0.7071067811865475
+  1.9999999999999998
+
+julia> ForwardDiff.jacobian(x) do x  # anonymous function, returns a length-2 vector
+         [sin(x[1]), prod(x[2:end])]
+       end
+2×4 Matrix{Float64}:
+ 0.707107   0.0  0.0  0.0
+ 0.0       12.0  8.0  6.0
+```
+
+See [ForwardDiff's documentation](https://juliadiff.org/ForwardDiff.jl/stable) for full details on how to use this package.
+ForwardDiff relies on [DiffRules](https://github.com/JuliaDiff/DiffRules.jl) for the derivatives of many simple function such as `sin`.
+
+See the [JuliaDiff web page](https://juliadiff.org) for other automatic differentiation packages.
 
 ## Publications
 

docs/src/dev/how_it_works.md

@@ -39,7 +39,7 @@ central feature that allows ForwardDiff to take derivatives.
 
 In order to implement the above property, elementary numerical functions on a `Dual`
 number are overloaded to evaluate both the original function, *and* evaluate the derivative
-of the function, propogating the derivative via multiplication. For example, `Base.sin`
+of the function, propagating the derivative via multiplication. For example, `Base.sin`
 can be overloaded on `Dual` like so:
 
 ```julia

docs/src/user/advanced.md

@@ -72,7 +72,7 @@ julia> @time gradient!(out, rosenbrock, x, cfg10);
   0.282529 seconds (4 allocations: 160 bytes)
 ```
 
-If you do not explicity provide a chunk size, ForwardDiff will try to guess one for you
+If you do not explicitly provide a chunk size, ForwardDiff will try to guess one for you
 based on your input vector:
 
 ```julia
@@ -84,6 +84,12 @@ julia> @time ForwardDiff.gradient!(out, rosenbrock, x, cfg);
   0.281853 seconds (4 allocations: 160 bytes)
 ```
 
+The underlying heuristic will compute a suitable chunk size smaller or equal
+to the `ForwardDiff.DEFAULT_CHUNK_THRESHOLD` constant. As of ForwardDiff
+v0.10.32 and Julia 1.6, this constant can be configured at load time via
+[Preferences.jl](https://github.com/JuliaPackaging/Preferences.jl) by setting the
+`default_chunk_threshold` value.
+
 If your input dimension is constant across calls, you should explicitly select a chunk size
 rather than relying on ForwardDiff's heuristic. There are two reasons for this. The first is
 that ForwardDiff's heuristic depends only on the input dimension, whereas in reality the
@@ -148,8 +154,24 @@ julia> using ForwardDiff, Preferences
 julia> set_preferences!(ForwardDiff, "nansafe_mode" => true)
 ```
 
+Note that Julia has to be restarted and ForwardDiff has to be reloaded after changing
+this preference.
+
+Alternatively, you can set the preference before loading ForwardDiff, for example via:
+
+```julia
+julia> using Preferences, UUIDs
+
+julia> set_preferences!(UUID("f6369f11-7733-5829-9624-2563aa707210"), "nansafe_mode" => true)
+
+julia> using ForwardDiff
+
+julia> log(ForwardDiff.Dual{:tag}(0.0, 0.0))
+Dual{:tag}(-Inf,0.0)
+```
+
 In the future, we plan on allowing users and downstream library authors to dynamically
-enable [NaN`-safe mode via the `AbstractConfig`
+enable [`NaN`-safe mode via the `AbstractConfig`
 API](https://github.com/JuliaDiff/ForwardDiff.jl/issues/181).
 
 ## Hessian of a vector-valued function
@@ -226,8 +248,8 @@ want to disable this checking.
 
 1. (preferred) Provide an extra `Val{false}()` argument to the differentiation function, e.g.
    ```julia
-   cfg = ForwarDiff.GradientConfig(g, x)
-   ForwarDiff.gradient(f, x, cfg, Val{false}())
+   cfg = ForwardDiff.GradientConfig(g, x)
+   ForwardDiff.gradient(f, x, cfg, Val{false}())
    ```
    If using as part of a library, the tag can be checked manually via
    ```julia

docs/src/user/limitations.md

@@ -12,3 +12,6 @@ function being differentiated):
 - **The target function must be written generically enough to accept numbers of type `T<:Real` as input (or arrays of these numbers).** The function doesn't require a specific type signature, as long as the type signature is generic enough to avoid breaking this rule. This also means that any storage assigned used within the function must be generic as well (see [this comment](https://github.com/JuliaDiff/ForwardDiff.jl/issues/136#issuecomment-237941790) for an example).
 
 - **The types of array inputs must be subtypes of** `AbstractArray` **.** Non-`AbstractArray` array-like types are not officially supported.
+
+ForwardDiff is not natively compatible with rules defined by the [ChainRules.jl](https://github.com/JuliaDiff/ChainRules.jl) ecosystem.
+You can use [ForwardDiffChainRules.jl](https://github.com/ThummeTo/ForwardDiffChainRules.jl) to bridge this gap.