From 5cb9fd2286f220d95115cc230773fe24c7d45d00 Mon Sep 17 00:00:00 2001 From: Victor Guerra Date: Mon, 23 Mar 2020 14:01:49 +0100 Subject: [PATCH 1/5] [AutoDiff][TF-1200] Adding derivatives for stdlib `pow` function. Adding JVP and VJP function derivatives for `pow` function defined in stdlib. Resolves TF-1200. --- .../TgmathDerivatives.swift.gyb | 24 +++++++++++++++++++ .../stdlib/tgmath_derivatives.swift.gyb | 9 +++++++ 2 files changed, 33 insertions(+) diff --git a/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb b/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb index 92fde0c6fbfa3..65c59646c309f 100644 --- a/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb +++ b/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb @@ -130,6 +130,7 @@ func _${derivative_kind}Trunc ( } %end # for derivative_kind in ['jvp', 'vjp']: +// Unary functions %for derivative_kind in ['jvp', 'vjp']: % linear_map_kind = 'differential' if derivative_kind == 'jvp' else 'pullback' % for T in ['Float', 'Double', 'Float80']: @@ -271,3 +272,26 @@ func _${derivative_kind}Erfc(_ x: ${T}) -> (value: ${T}, ${linear_map_kind}: (${ % end # if T == 'Float80': % end # for T in ['Float', 'Double', 'Float80']: %end # for derivative_kind in ['jvp', 'vjp']: + +// Binary functions +%for T in ['Float', 'Double', 'Float80']: +% if T == 'Float80': +#if !(os(Windows) || os(Android)) && (arch(i386) || arch(x86_64)) +% end +@inlinable +@derivative(of: pow) +func _vjpPow(_ x: ${T}, _ y: ${T}) -> (value: ${T}, pullback: (${T}) -> (${T}, ${T})) { + let value = pow(x, y) + return (value, { v in (v * y * pow(x, y - 1), v * value * log(x)) }) +} + +@inlinable +@derivative(of: pow) +func _jvpPow(_ x: ${T}, _ y: ${T}) -> (value: ${T}, differential: (${T}, ${T}) -> ${T}) { + let value = pow(x, y) + return (value, { (dx, dy) in dx * y * pow(x, y - 1) + dy * value * log(x) }) +} +% if T == 'Float80': +#endif +% end # if T == 'Float80': +%end # for T in ['Float', 'Double', 'Float80']: diff --git a/test/AutoDiff/stdlib/tgmath_derivatives.swift.gyb b/test/AutoDiff/stdlib/tgmath_derivatives.swift.gyb index 08d019f7b2a5e..883243b79d3d8 100644 --- a/test/AutoDiff/stdlib/tgmath_derivatives.swift.gyb +++ b/test/AutoDiff/stdlib/tgmath_derivatives.swift.gyb @@ -114,6 +114,15 @@ DerivativeTests.test("${op}_${T}") { %end } } + + // pow + let dpow = ${op}(at: 3 as ${T}, 2 as ${T}, in: pow) +%if op == 'gradient': + expectEqualWithTolerance(6, dpow.0) + expectEqualWithTolerance(9.887510598012987222, dpow.1) +%else: # if op == 'derivative' + expectEqualWithTolerance(15.887510598012987222, dpow) +%end } %if T == 'Float80': From ba460162a8c10fb72e71f3d29eabdf7148123e14 Mon Sep 17 00:00:00 2001 From: Victor Guerra Date: Wed, 25 Mar 2020 09:02:02 +0100 Subject: [PATCH 2/5] Binary functions only defined for `Float` and `Float80` --- stdlib/public/Differentiation/TgmathDerivatives.swift.gyb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb b/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb index 65c59646c309f..bc886f646e3fc 100644 --- a/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb +++ b/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb @@ -274,7 +274,7 @@ func _${derivative_kind}Erfc(_ x: ${T}) -> (value: ${T}, ${linear_map_kind}: (${ %end # for derivative_kind in ['jvp', 'vjp']: // Binary functions -%for T in ['Float', 'Double', 'Float80']: +%for T in ['Float', 'Float80']: % if T == 'Float80': #if !(os(Windows) || os(Android)) && (arch(i386) || arch(x86_64)) % end From 476319ecb15b61e19a71423d5dabf8fee7844a2d Mon Sep 17 00:00:00 2001 From: Victor Guerra Date: Sun, 19 Apr 2020 17:20:01 +0200 Subject: [PATCH 3/5] Fixing VJP and JVP for pow.Extending test cases. The versions of VJP and JVP match the results that TensorFlow woudl give. As well, this extends the test cases to include combinations of positive and negative values and corner cases. --- .../TgmathDerivatives.swift.gyb | 8 ++- .../stdlib/tgmath_derivatives.swift.gyb | 71 ++++++++++++++++--- 2 files changed, 67 insertions(+), 12 deletions(-) diff --git a/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb b/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb index bc886f646e3fc..c8ebe4e478e1e 100644 --- a/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb +++ b/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb @@ -282,14 +282,18 @@ func _${derivative_kind}Erfc(_ x: ${T}) -> (value: ${T}, ${linear_map_kind}: (${ @derivative(of: pow) func _vjpPow(_ x: ${T}, _ y: ${T}) -> (value: ${T}, pullback: (${T}) -> (${T}, ${T})) { let value = pow(x, y) - return (value, { v in (v * y * pow(x, y - 1), v * value * log(x)) }) + return (value, { v in ( + v * y * pow(x, y - 1), v * value * log(x.isLessThanOrEqualTo(0) ? ${T}(1) : x) + ) }) } @inlinable @derivative(of: pow) func _jvpPow(_ x: ${T}, _ y: ${T}) -> (value: ${T}, differential: (${T}, ${T}) -> ${T}) { let value = pow(x, y) - return (value, { (dx, dy) in dx * y * pow(x, y - 1) + dy * value * log(x) }) + return (value, { (dx, dy) in + dx * y * pow(x, y - 1) + dy * value * log(x.isLessThanOrEqualTo(0) ? ${T}(1) : x) + }) } % if T == 'Float80': #endif diff --git a/test/AutoDiff/stdlib/tgmath_derivatives.swift.gyb b/test/AutoDiff/stdlib/tgmath_derivatives.swift.gyb index 883243b79d3d8..b8187df140fa6 100644 --- a/test/AutoDiff/stdlib/tgmath_derivatives.swift.gyb +++ b/test/AutoDiff/stdlib/tgmath_derivatives.swift.gyb @@ -26,7 +26,7 @@ func expectEqualWithTolerance(_ expected: TestLiteralType, _ actual: T, ulps allowed: T = 3, file: String = #file, line: UInt = #line) where T: BinaryFloatingPoint { - if actual == T(expected) || actual.isNaN && expected.isNaN { + if actual == T(expected) || actual.isNaN && expected.isNaN || actual.isInfinite && expected.isInfinite { return } // Compute error in ulp, compare to tolerance. @@ -38,17 +38,40 @@ func expectEqualWithTolerance(_ expected: TestLiteralType, _ actual: T, file: file, line: line) } +func computeDividedDifference ( + _ f: (T, T) -> T, + _ x: T, + _ y: T, + eps: T = 0.01 +) -> (dfdx: T, dfdy: T) { + let dfdx = (f(x + eps, y) - f(x, y)) / eps + let dfdy = (f(x, y + eps) - f(x, y)) / eps + return (dfdx, dfdy) +} + func checkGradient( _ f: @differentiable (T, T) -> T, _ x: T, - _ y: T) + _ y: T, + ulps: T = 192) where T == T.TangentVector { let eps = T(0.01) let grad = gradient(at: x, y, in: f) - let dfdx = (f(x + eps, y) - f(x, y)) / eps - let dfdy = (f(x, y + eps) - f(x, y)) / eps - expectEqualWithTolerance(TestLiteralType(dfdx), grad.0, ulps: 192) - expectEqualWithTolerance(TestLiteralType(dfdy), grad.1, ulps: 192) + let (dfdx, dfdy) = computeDividedDifference(f, x, y, eps: eps) + expectEqualWithTolerance(TestLiteralType(dfdx), grad.0, ulps: ulps) + expectEqualWithTolerance(TestLiteralType(dfdy), grad.1, ulps: ulps) +} + +func checkDerivative( + _ f: @differentiable (T, T) -> T, + _ x: T, + _ y: T, + ulps: T = 192) +where T == T.TangentVector { + let eps = T(0.01) + let deriv = derivative(at: x, y, in: f) + let (dfdx, dfdy) = computeDividedDifference(f, x, y, eps: eps) + expectEqualWithTolerance(TestLiteralType(dfdx + dfdy), deriv, ulps: ulps) } %for op in ['derivative', 'gradient']: @@ -116,13 +139,41 @@ DerivativeTests.test("${op}_${T}") { } // pow - let dpow = ${op}(at: 3 as ${T}, 2 as ${T}, in: pow) + let eps:${T} = 0.01 + let ulps:${T} = eps/eps.ulp + for a in -3...3 { + let x = ${T}(a) + for b in -3...3 { + let y = ${T}(b) + var expectedValues: (dx: ${T}, dy: ${T})? + if x.isZero { + if y.isLess(than: 0) { + expectedValues = (dx: ${T}.infinity, dy: ${T}.nan) + } else if y.isZero { + expectedValues = (dx: ${T}.nan, dy: ${T}.zero) + } else if !y.isEqual(to: 1) { + expectedValues = (dx: ${T}.zero, dy: ${T}.zero) + } + } else if x.isLess(than: 2) { + expectedValues = (dx: y * pow(x, y - 1), ${T}.zero) + } + if let (expectedDx, expectedDy) = expectedValues { + let dpow = ${op}(at: x, y, in: pow) %if op == 'gradient': - expectEqualWithTolerance(6, dpow.0) - expectEqualWithTolerance(9.887510598012987222, dpow.1) + expectEqualWithTolerance(TestLiteralType(expectedDx), dpow.0) + expectEqualWithTolerance(TestLiteralType(expectedDy), dpow.1) %else: # if op == 'derivative' - expectEqualWithTolerance(15.887510598012987222, dpow) + expectEqualWithTolerance(TestLiteralType(expectedDx + expectedDy), dpow) %end + } else { +%if op == 'gradient': + checkGradient({ pow($0, $1) }, x, y, ulps: ulps) +%else: # if op == 'derivative' + checkDerivative({ pow($0, $1) }, x, y, ulps: ulps) +%end + } + } + } } %if T == 'Float80': From dbc3396cb7a6d1ebd98b51b9b387a2960ef9f450 Mon Sep 17 00:00:00 2001 From: Victor Guerra Date: Sun, 19 Apr 2020 18:31:56 +0200 Subject: [PATCH 4/5] remove `Double` from comment. --- stdlib/public/Differentiation/TgmathDerivatives.swift.gyb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb b/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb index c8ebe4e478e1e..1a770907f5a55 100644 --- a/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb +++ b/stdlib/public/Differentiation/TgmathDerivatives.swift.gyb @@ -298,4 +298,4 @@ func _jvpPow(_ x: ${T}, _ y: ${T}) -> (value: ${T}, differential: (${T}, ${T}) - % if T == 'Float80': #endif % end # if T == 'Float80': -%end # for T in ['Float', 'Double', 'Float80']: +%end # for T in ['Float', 'Float80']: From 24939a5da3c67f229a5e5c181da59aacf7bb8131 Mon Sep 17 00:00:00 2001 From: Victor Guerra Date: Wed, 22 Apr 2020 23:13:10 +0200 Subject: [PATCH 5/5] Splitting `pow`'s derivative test cases. We now have 3 sections for the tests: negative base, 0 base and positive base. --- .../stdlib/tgmath_derivatives.swift.gyb | 69 +++++++++++++------ 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/test/AutoDiff/stdlib/tgmath_derivatives.swift.gyb b/test/AutoDiff/stdlib/tgmath_derivatives.swift.gyb index b8187df140fa6..331955bf4fe39 100644 --- a/test/AutoDiff/stdlib/tgmath_derivatives.swift.gyb +++ b/test/AutoDiff/stdlib/tgmath_derivatives.swift.gyb @@ -141,37 +141,62 @@ DerivativeTests.test("${op}_${T}") { // pow let eps:${T} = 0.01 let ulps:${T} = eps/eps.ulp - for a in -3...3 { + + // Checks for negative base. + for a in -3..<0 { let x = ${T}(a) for b in -3...3 { let y = ${T}(b) - var expectedValues: (dx: ${T}, dy: ${T})? - if x.isZero { - if y.isLess(than: 0) { - expectedValues = (dx: ${T}.infinity, dy: ${T}.nan) - } else if y.isZero { - expectedValues = (dx: ${T}.nan, dy: ${T}.zero) - } else if !y.isEqual(to: 1) { - expectedValues = (dx: ${T}.zero, dy: ${T}.zero) - } - } else if x.isLess(than: 2) { - expectedValues = (dx: y * pow(x, y - 1), ${T}.zero) - } - if let (expectedDx, expectedDy) = expectedValues { - let dpow = ${op}(at: x, y, in: pow) + let expectedDx = y * pow(x, y - 1) + let expectedDy = ${T}.zero + let dpow = ${op}(at: x, y, in: pow) %if op == 'gradient': - expectEqualWithTolerance(TestLiteralType(expectedDx), dpow.0) - expectEqualWithTolerance(TestLiteralType(expectedDy), dpow.1) + expectEqualWithTolerance(TestLiteralType(expectedDx), dpow.0) + expectEqualWithTolerance(TestLiteralType(expectedDy), dpow.1) %else: # if op == 'derivative' - expectEqualWithTolerance(TestLiteralType(expectedDx + expectedDy), dpow) + expectEqualWithTolerance(TestLiteralType(expectedDx + expectedDy), dpow) %end - } else { + } + } + + // Checks for 0 base. + for b in -3...3 { + let y = ${T}(b) + var expectedValues: (dx: ${T}, dy: ${T})? + if y.isLess(than: 0) { + expectedValues = (dx: ${T}.infinity, dy: ${T}.nan) + } else if y.isZero { + expectedValues = (dx: ${T}.nan, dy: ${T}.zero) + } else if !y.isEqual(to: 1) { + expectedValues = (dx: ${T}.zero, dy: ${T}.zero) + } + if let (expectedDx, expectedDy) = expectedValues { + let dpow = ${op}(at: 0.0, y, in: pow) +%if op == 'gradient': + expectEqualWithTolerance(TestLiteralType(expectedDx), dpow.0) + expectEqualWithTolerance(TestLiteralType(expectedDy), dpow.1) +%else: # if op == 'derivative' + expectEqualWithTolerance(TestLiteralType(expectedDx + expectedDy), dpow) +%end + } else { +%if op == 'gradient': + checkGradient({ pow($0, $1) }, 0.0, y, ulps: ulps) +%else: # if op == 'derivative' + checkDerivative({ pow($0, $1) }, 0.0, y, ulps: ulps) +%end + } + } + + // Checks for positive base. + for a in 1...3 { + let x = ${T}(a) + for b in -3...3 { + let y = ${T}(b) %if op == 'gradient': - checkGradient({ pow($0, $1) }, x, y, ulps: ulps) + checkGradient({ pow($0, $1) }, x, y, ulps: ulps) %else: # if op == 'derivative' - checkDerivative({ pow($0, $1) }, x, y, ulps: ulps) + checkDerivative({ pow($0, $1) }, x, y, ulps: ulps) %end - } } } }