diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a246fbbd..ecda8d9be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## 3.1.2 2023-11-13 + +### Fix + +Fixed an issue where, when using the DynamoDbEncryptionInterceptor, +an encrypted item in the Attributes field of a DeleteItem, PutItem, or UpdateItem +response was passed through unmodified instead of being decrypted. + +## 3.1.1 2023-11-07 + +### Fix + +Issue when a DynamoDB Set attribute is marked as SIGN_ONLY in the AWS Database Encryption SDK (DB-ESDK) for DynamoDB. + +DB-ESDK for DynamoDB supports SIGN_ONLY and ENCRYPT_AND_SIGN attribute actions. In version 3.1.0 and below, when a Set type is assigned a SIGN_ONLY attribute action, there is a chance that signature validation of the record containing a Set will fail on read, even if the Set attributes contain the same values. The probability of a failure depends on the order of the elements in the Set combined with how DynamoDB returns this data, which is undefined. + +This update addresses the issue by ensuring that any Set values are canonicalized in the same order while written to DynamoDB as when read back from DynamoDB. + +See: https://github.com/aws/aws-database-encryption-sdk-dynamodb-java/DecryptWithPermute/README.md for additional details + + ## 3.1.0 2023-09-07 ### Features diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/ConfigToInfo.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/ConfigToInfo.dfy index 4dcd2033f..69e80e25a 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/ConfigToInfo.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/ConfigToInfo.dfy @@ -328,6 +328,7 @@ module SearchConfigToInfo { if |badNames| == 0 then None else + // We happen to order these values, but this ordering MUST NOT be relied upon. var badSeq := SortedSets.ComputeSetToOrderedSequence2(badNames, CharLess); Some(badSeq[0]) } @@ -399,6 +400,7 @@ module SearchConfigToInfo { if |badNames| == 0 then None else + // We happen to order these values, but this ordering MUST NOT be relied upon. var badSeq := SortedSets.ComputeSetToOrderedSequence2(badNames, CharLess); Some(badSeq[0]) } diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DDBSupport.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DDBSupport.dfy index 65a299690..c609d5ac2 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DDBSupport.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DDBSupport.dfy @@ -44,6 +44,7 @@ module DynamoDBSupport { Success(true) else var bad := set k <- item | ReservedPrefix <= k; + // We happen to order these values, but this ordering MUST NOT be relied upon. var badSeq := SortedSets.ComputeSetToOrderedSequence2(bad, CharLess); if |badSeq| == 0 then Failure("") @@ -173,6 +174,7 @@ module DynamoDBSupport { var both := newAttrs.Keys * item.Keys; var bad := set k <- both | newAttrs[k] != item[k]; if 0 < |bad| { + // We happen to order these values, but this ordering MUST NOT be relied upon. var badSeq := SortedSets.ComputeSetToOrderedSequence2(bad, CharLess); return Failure(E("Supplied Beacons do not match calculated beacons : " + Join(badSeq, ", "))); } diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DynamoToStruct.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DynamoToStruct.dfy index 41fb54d4b..6e592d3ee 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DynamoToStruct.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/DynamoToStruct.dfy @@ -23,15 +23,8 @@ module DynamoToStruct { type StructuredDataTerminalType = x : StructuredData | x.content.Terminal? witness * type TerminalDataMap = map -//= specification/dynamodb-encryption-client/ddb-item-conversion.md#overview -//= type=TODO -//# The conversion from DDB Item to Structured Data must be lossless, -//# meaning that converting a DDB Item to -//# a Structured Data and back to a DDB Item again -//# MUST result in the exact same DDB Item. - // This file exists for these two functions : ItemToStructured and StructuredToItem - // which provide lossless conversion between an AttributeMap and a StructuredDataMap + // which provide conversion between an AttributeMap and a StructuredDataMap // Convert AttributeMap to StructuredDataMap //= specification/dynamodb-encryption-client/ddb-item-conversion.md#convert-ddb-item-to-structured-data @@ -107,6 +100,7 @@ module DynamoToStruct { else var badNames := set k <- s | !IsValid_AttributeName(k) :: k; OneBadKey(s, badNames, IsValid_AttributeName); + // We happen to order these values, but this ordering MUST NOT be relied upon. var orderedAttrNames := SetToOrderedSequence(badNames, CharLess); var attrNameList := Join(orderedAttrNames, ","); MakeError("Not valid attribute names : " + attrNameList) @@ -317,7 +311,11 @@ module DynamoToStruct { //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#number //= type=implication - //# Number MUST be serialized as UTF-8 encoded bytes. + //# This value MUST be normalized in the same way as DynamoDB normalizes numbers. + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#number + //= type=implication + //# This normalized value MUST then be serialized as UTF-8 encoded bytes. ensures a.N? && ret.Success? && !prefix ==> && Norm.NormalizeNumber(a.N).Success? && var nn := Norm.NormalizeNumber(a.N).value; @@ -488,30 +486,52 @@ module DynamoToStruct { function method StringSetAttrToBytes(ss: StringSetAttributeValue): (ret: Result, string>) ensures ret.Success? ==> Seq.HasNoDuplicates(ss) { - :- Need(|Seq.ToSet(ss)| == |ss|, "String Set had duplicate values"); + var asSet := Seq.ToSet(ss); + :- Need(|asSet| == |ss|, "String Set had duplicate values"); Seq.LemmaNoDuplicatesCardinalityOfSet(ss); - var count :- U32ToBigEndian(|ss|); - var body :- CollectString(ss); + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //# Entries in a String Set MUST be ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). + var sortedList := SortedSets.ComputeSetToOrderedSequence2(asSet, CharLess); + var count :- U32ToBigEndian(|sortedList|); + var body :- CollectString(sortedList); Success(count + body) } function method NumberSetAttrToBytes(ns: NumberSetAttributeValue): (ret: Result, string>) ensures ret.Success? ==> Seq.HasNoDuplicates(ns) { - :- Need(|Seq.ToSet(ns)| == |ns|, "Number Set had duplicate values"); + var asSet := Seq.ToSet(ns); + :- Need(|asSet| == |ns|, "Number Set had duplicate values"); Seq.LemmaNoDuplicatesCardinalityOfSet(ns); - var count :- U32ToBigEndian(|ns|); - var body :- CollectString(ns); + + var normList :- Seq.MapWithResult(n => Norm.NormalizeNumber(n), ns); + var asSet := Seq.ToSet(normList); + :- Need(|asSet| == |normList|, "Number Set had duplicate values after normalization."); + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //# Entries in a Number Set MUST be ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //# This ordering MUST be applied after normalization of the number value. + var sortedList := SortedSets.ComputeSetToOrderedSequence2(asSet, CharLess); + var count :- U32ToBigEndian(|sortedList|); + var body :- CollectString(sortedList); Success(count + body) } function method BinarySetAttrToBytes(bs: BinarySetAttributeValue): (ret: Result, string>) ensures ret.Success? ==> Seq.HasNoDuplicates(bs) { - :- Need(|Seq.ToSet(bs)| == |bs|, "Binary Set had duplicate values"); + var asSet := Seq.ToSet(bs); + :- Need(|asSet| == |bs|, "Binary Set had duplicate values"); Seq.LemmaNoDuplicatesCardinalityOfSet(bs); - var count :- U32ToBigEndian(|bs|); - var body :- CollectBinary(bs); + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //# Entries in a Binary Set MUST be ordered lexicographically by their underlying bytes in ascending order. + var sortedList := SortedSets.ComputeSetToOrderedSequence2(asSet, ByteLess); + var count :- U32ToBigEndian(|sortedList|); + var body :- CollectBinary(sortedList); Success(count + body) } @@ -749,6 +769,9 @@ module DynamoToStruct { ensures (ret.Success? && |mapToSerialize| == 0) ==> (ret.value == serialized) ensures (ret.Success? && |mapToSerialize| == 0) ==> (|ret.value| == |serialized|) { + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#key-value-pair-entries + //# Entries in a serialized Map MUST be ordered by key value, + //# ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). var keys := SortedSets.ComputeSetToOrderedSequence2(mapToSerialize.Keys, CharLess); CollectOrderedMapSubset(keys, mapToSerialize, serialized) } @@ -1136,6 +1159,7 @@ module DynamoToStruct { OneBadResult(m); var badValues := FlattenErrors(m); assert(|badValues| > 0); + // We happen to order these values, but this ordering MUST NOT be relied upon. var badValueSeq := SetToOrderedSequence(badValues, CharLess); Failure(Join(badValueSeq, "\n")) } diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/SearchInfo.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/SearchInfo.dfy index 700318273..bee6418a7 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryption/src/SearchInfo.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryption/src/SearchInfo.dfy @@ -344,12 +344,6 @@ module SearchableEncryptionInfo { versions[currWrite].IsVirtualField(field) } - function method GenerateClosure(fields : seq) : seq - requires ValidState() - { - versions[currWrite].GenerateClosure(fields) - } - method GeneratePlainBeacons(item : DDB.AttributeMap) returns (output : Result) requires ValidState() { @@ -559,6 +553,7 @@ module SearchableEncryptionInfo { requires version == 1 requires keySource.ValidState() { + // We happen to order these values, but this ordering MUST NOT be relied upon. var beaconNames := SortedSets.ComputeSetToOrderedSequence2(beacons.Keys, CharLess); var stdKeys := Seq.Filter((k : string) => k in beacons && beacons[k].Standard?, beaconNames); FilterPreservesHasNoDuplicates((k : string) => k in beacons && beacons[k].Standard?, beaconNames); @@ -575,6 +570,7 @@ module SearchableEncryptionInfo { keySource : KeySource, virtualFields : VirtualFieldMap, beacons : BeaconMap, + // The ordering of `beaconNames` MUST NOT be relied upon. beaconNames : seq, stdNames : seq, encryptedFields : set @@ -614,13 +610,6 @@ module SearchableEncryptionInfo { [field] } - function method GenerateClosure(fields : seq) : seq - { - var fieldLists := Seq.Map((s : string) => GetFields(s), fields); - var fieldSet := set f <- fieldLists, g <- f :: g; - SortedSets.ComputeSetToOrderedSequence2(fieldSet, CharLess) - } - method getKeyMap(keyId : MaybeKeyId) returns (output : Result) requires ValidState() ensures ValidState() diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryption/test/DynamoToStruct.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryption/test/DynamoToStruct.dfy index c7290bb8e..ffecf051e 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryption/test/DynamoToStruct.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryption/test/DynamoToStruct.dfy @@ -282,14 +282,264 @@ module DynamoToStructTest { expect newMapValue.value == mapValue; } - //= specification/dynamodb-encryption-client/ddb-item-conversion.md#overview + method {:test} TestNormalizeNAttr() { + var numberValue := AttributeValue.N("000123.000"); + var encodedNumberData := StructuredDataTerminal(value := [49,50,51], typeId := [0,2]); + var encodedNumberValue := StructuredData(content := Terminal(encodedNumberData), attributes := None); + var numberStruct := AttrToStructured(numberValue); + expect numberStruct.Success?; + expect numberStruct.value == encodedNumberValue; + + var newNumberValue := StructuredToAttr(encodedNumberValue); + expect newNumberValue.Success?; + expect newNumberValue.value == AttributeValue.N("123"); + } + + method {:test} TestNormalizeNInSet() { + var numberSetValue := AttributeValue.NS(["001.00"]); + var encodedNumberSetData := StructuredDataTerminal(value := [0,0,0,1, 0,0,0,1, 49], typeId := [1,2]); + var encodedNumberSetValue := StructuredData(content := Terminal(encodedNumberSetData), attributes := None); + var numberSetStruct := AttrToStructured(numberSetValue); + expect numberSetStruct.Success?; + expect numberSetStruct.value == encodedNumberSetValue; + + var newNumberSetValue := StructuredToAttr(encodedNumberSetValue); + expect newNumberSetValue.Success?; + expect newNumberSetValue.value == AttributeValue.NS(["1"]); + } + + method {:test} TestNormalizeNInList() { + var nValue := AttributeValue.N("001.00"); + var normalizedNValue := AttributeValue.N("1"); + + var listValue := AttributeValue.L([nValue]); + var encodedListData := StructuredDataTerminal(value := [ + 0,0,0,1, // 1 member in list + 0,2, 0,0,0,1, 49 // 1st member is N("1") + ], + typeId := [3,0]); + var encodedListValue := StructuredData(content := Terminal(encodedListData), attributes := None); + var listStruct := AttrToStructured(listValue); + expect listStruct.Success?; + expect listStruct.value == encodedListValue; + + var newListValue := StructuredToAttr(listStruct.value); + expect newListValue.Success?; + expect newListValue.value == AttributeValue.L([normalizedNValue]); + } + + method {:test} TestNormalizeNInMap() { + var nValue := AttributeValue.N("001.00"); + var normalizedNValue := AttributeValue.N("1"); + + var mapValue := AttributeValue.M(map["keyA" := nValue]); + var k := 'k' as uint8; + var e := 'e' as uint8; + var y := 'y' as uint8; + var A := 'A' as uint8; + + var encodedMapData := StructuredDataTerminal( + value := [ + 0,0,0,1, // there is 1 entry in the map + 0,1, 0,0,0,4, k,e,y,A, // 1st entry's key + 0,2, 0,0,0,1, // 1st entry's value is a N and is 1 byte long + 49 // "1" + ], + typeId := [2,0]); + + var encodedMapValue := StructuredData(content := Terminal(encodedMapData), attributes := None); + var mapStruct := AttrToStructured(mapValue); + expect mapStruct.Success?; + expect mapStruct.value == encodedMapValue; + + var newMapValue := StructuredToAttr(mapStruct.value); + expect newMapValue.Success?; + expect newMapValue.value == AttributeValue.M(map["keyA" := normalizedNValue]); + } + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries //= type=test - //# The conversion from DDB Item to Structured Data must be lossless, - //# meaning that converting a DDB Item to - //# a Structured Data and back to a DDB Item again - //# MUST result in the exact same DDB Item. - method {:test} TestRoundTrip() { + //# Entries in a Number Set MUST be ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). + method {:test} TestSortNSAttr() { + var numberSetValue := AttributeValue.NS(["1","2","10"]); + var encodedNumberSetData := StructuredDataTerminal(value := [0,0,0,3, 0,0,0,1, 49, 0,0,0,2, 49,48, 0,0,0,1, 50], typeId := [1,2]); + var encodedNumberSetValue := StructuredData(content := Terminal(encodedNumberSetData), attributes := None); + var numberSetStruct := AttrToStructured(numberSetValue); + expect numberSetStruct.Success?; + expect numberSetStruct.value == encodedNumberSetValue; + + var newNumberSetValue := StructuredToAttr(encodedNumberSetValue); + expect newNumberSetValue.Success?; + expect newNumberSetValue.value == AttributeValue.NS(["1","10","2"]); + } + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //= type=test + //# This ordering MUST be applied after normalization of the number value. + method {:test} TestSortNSAfterNormalize() { + var numberSetValue := AttributeValue.NS(["1","02","10"]); + var encodedNumberSetData := StructuredDataTerminal(value := [0,0,0,3, 0,0,0,1, 49, 0,0,0,2, 49,48, 0,0,0,1, 50], typeId := [1,2]); + var encodedNumberSetValue := StructuredData(content := Terminal(encodedNumberSetData), attributes := None); + var numberSetStruct := AttrToStructured(numberSetValue); + expect numberSetStruct.Success?; + expect numberSetStruct.value == encodedNumberSetValue; + + var newNumberSetValue := StructuredToAttr(encodedNumberSetValue); + expect newNumberSetValue.Success?; + expect newNumberSetValue.value == AttributeValue.NS(["1","10","2"]); + } + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //= type=test + //# Entries in a String Set MUST be ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). + method {:test} TestSortSSAttr() { + var stringSetValue := AttributeValue.SS(["&","｡","𐀂"]); + // Note that string values are UTF-8 encoded, but sorted by UTF-16 encoding. + var encodedStringSetData := StructuredDataTerminal(value := [ + 0,0,0,3, // 3 entries in set + 0,0,0,1, // 1st entry is 1 byte + 0x26, // "&" in UTF-8 encoding + 0,0,0,4, // 2nd entry is 4 bytes + 0xF0,0x90,0x80,0x82, // "𐀂" in UTF-8 encoding + 0,0,0,3, // 3rd entry is 3 bytes + 0xEF,0xBD,0xA1 // "｡" in UTF-8 encoding + ], + typeId := [1,1] + ); + var encodedStringSetValue := StructuredData(content := Terminal(encodedStringSetData), attributes := None); + var stringSetStruct := AttrToStructured(stringSetValue); + expect stringSetStruct.Success?; + expect stringSetStruct.value == encodedStringSetValue; + + var newStringSetValue := StructuredToAttr(encodedStringSetValue); + expect newStringSetValue.Success?; + expect newStringSetValue.value == AttributeValue.SS(["&","𐀂","｡"]); + } + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#set-entries + //= type=test + //# Entries in a Binary Set MUST be ordered lexicographically by their underlying bytes in ascending order. + method {:test} TestSortBSAttr() { + var binarySetValue := AttributeValue.BS([[1],[2],[1,0]]); + var encodedBinarySetData := StructuredDataTerminal(value := [0,0,0,3, 0,0,0,1, 1, 0,0,0,2, 1,0, 0,0,0,1, 2], typeId := [1,0xff]); + var encodedBinarySetValue := StructuredData(content := Terminal(encodedBinarySetData), attributes := None); + var binarySetStruct := AttrToStructured(binarySetValue); + expect binarySetStruct.Success?; + expect binarySetStruct.value == encodedBinarySetValue; + + var newBinarySetValue := StructuredToAttr(encodedBinarySetValue); + expect newBinarySetValue.Success?; + expect newBinarySetValue.value == AttributeValue.BS([[1],[1,0],[2]]); + } + + method {:test} TestSetsInListAreSorted() { + var nSetValue := AttributeValue.NS(["2","1","10"]); + var sSetValue := AttributeValue.SS(["&","｡","𐀂"]); + var bSetValue := AttributeValue.BS([[1,0],[1],[2]]); + + var sortedNSetValue := AttributeValue.NS(["1","10","2"]); + var sortedSSetValue := AttributeValue.SS(["&","𐀂","｡"]); + var sortedBSetValue := AttributeValue.BS([[1],[1,0],[2]]); + + var listValue := AttributeValue.L([nSetValue, sSetValue, bSetValue]); + var encodedListData := StructuredDataTerminal(value := [ + 0,0,0,3, // 3 members in list + 1,2, 0,0,0,20, // 1st member is a NS and is 20 bytes long + 0,0,0,3, 0,0,0,1, 49, 0,0,0,2, 49,48, 0,0,0,1, 50, // NS + 1,1, 0,0,0,24, // 2nd member is a SS and is 24 bytes long + 0,0,0,3, 0,0,0,1, 0x26, 0,0,0,4, 0xF0,0x90,0x80,0x82, 0,0,0,3, 0xEF,0xBD,0xA1, // SS + 1,0xFF, 0,0,0,20, // 3rd member is a BS and is 20 bytes long + 0,0,0,3, 0,0,0,1, 1, 0,0,0,2, 1,0, 0,0,0,1, 2 // BS + ], + typeId := [3,0]); + var encodedListValue := StructuredData(content := Terminal(encodedListData), attributes := None); + var listStruct := AttrToStructured(listValue); + expect listStruct.Success?; + expect listStruct.value == encodedListValue; + + var newListValue := StructuredToAttr(listStruct.value); + expect newListValue.Success?; + expect newListValue.value == AttributeValue.L([sortedNSetValue, sortedSSetValue, sortedBSetValue]); + } + + method {:test} TestSetsInMapAreSorted() { + var nSetValue := AttributeValue.NS(["2","1","10"]); + var sSetValue := AttributeValue.SS(["&","｡","𐀂"]); + var bSetValue := AttributeValue.BS([[1,0],[1],[2]]); + var sortedNSetValue := AttributeValue.NS(["1","10","2"]); + var sortedSSetValue := AttributeValue.SS(["&","𐀂","｡"]); + var sortedBSetValue := AttributeValue.BS([[1],[1,0],[2]]); + + var mapValue := AttributeValue.M(map["keyA" := sSetValue, "keyB" := nSetValue, "keyC" := bSetValue]); + var k := 'k' as uint8; + var e := 'e' as uint8; + var y := 'y' as uint8; + var A := 'A' as uint8; + var B := 'B' as uint8; + var C := 'C' as uint8; + + var encodedMapData := StructuredDataTerminal( + value := [ + 0,0,0,3, // there are 3 entries in the map + 0,1, 0,0,0,4, k,e,y,A, // 1st entry's key + 1,1, 0,0,0,24, // 1st entry's value is a SS and is 24 bytes long + 0,0,0,3, 0,0,0,1, 0x26, 0,0,0,4, 0xF0,0x90,0x80,0x82, 0,0,0,3, 0xEF,0xBD,0xA1, // SS + 0,1, 0,0,0,4, k,e,y,B, // 2nd entry's key + 1,2, 0,0,0,20, // 2nd entry's value is a NS and is 20 bytes long + 0,0,0,3, 0,0,0,1, 49, 0,0,0,2, 49,48, 0,0,0,1, 50, // NS + 0,1, 0,0,0,4, k,e,y,C, // 3rd entry's key + 1,0xFF, 0,0,0,20, // 3rd entry's value is a BS and is 20 bytes long + 0,0,0,3, 0,0,0,1, 1, 0,0,0,2, 1,0, 0,0,0,1, 2 // BS + ], + typeId := [2,0]); + + var encodedMapValue := StructuredData(content := Terminal(encodedMapData), attributes := None); + var mapStruct := AttrToStructured(mapValue); + expect mapStruct.Success?; + expect mapStruct.value == encodedMapValue; + + var newMapValue := StructuredToAttr(mapStruct.value); + expect newMapValue.Success?; + expect newMapValue.value == AttributeValue.M(map["keyA" := sortedSSetValue, "keyB" := sortedNSetValue, "keyC" := sortedBSetValue]); + } + + //= specification/dynamodb-encryption-client/ddb-attribute-serialization.md#key-value-pair-entries + //= type=test + //# Entries in a serialized Map MUST be ordered by key value, + //# ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). + method {:test} TestSortMapKeys() { + var nullValue := AttributeValue.NULL(true); + + var mapValue := AttributeValue.M(map["&" := nullValue, "｡" := nullValue, "𐀂" := nullValue]); + + // Note that the string values are encoded as UTF-8, but are sorted according to UTF-16 encoding. + var encodedMapData := StructuredDataTerminal( + value := [ + 0,0,0,3, // 3 entries + 0,1, 0,0,0,1, // 1st key is a string 1 byte long + 0x26, // "&" UTF-8 encoded + 0,0, 0,0,0,0, // null value + 0,1, 0,0,0,4, // 2nd key is a string 4 bytes long + 0xF0, 0x90, 0x80, 0x82, // "𐀂" UTF-8 encoded + 0,0, 0,0,0,0, // null value + 0,1, 0,0,0,3, // 3rd key is a string 3 bytes long + 0xEF, 0xBD, 0xA1, // "｡" + 0,0, 0,0,0,0 // null value + ], + typeId := [2,0]); + var encodedMapValue := StructuredData(content := Terminal(encodedMapData), attributes := None); + var mapStruct := AttrToStructured(mapValue); + expect mapStruct.Success?; + expect mapStruct.value == encodedMapValue; + + var newMapValue := StructuredToAttr(mapStruct.value); + expect newMapValue.Success?; + expect newMapValue.value == mapValue; + } + + method {:test} TestRoundTrip() { + // Note - set and number values are carefully pre-normalized. var val1 := AttributeValue.S("astring"); var val2 := AttributeValue.N("12345"); var val3 := AttributeValue.B([1,2,3,4,5]); @@ -297,7 +547,7 @@ module DynamoToStructTest { var val5 := AttributeValue.NULL(true); var val6 := AttributeValue.BS([[1,2,3,4,5],[2,3,4,5,6],[3,4,5,6,7]]); var val7 := AttributeValue.SS(["ab","cdef","ghijk"]); - var val8 := AttributeValue.NS(["1","234.567","0"]); + var val8 := AttributeValue.NS(["0", "1","234.567"]); var val9a := AttributeValue.L([val8, val7, val6]); var val9b := AttributeValue.L([val5, val4, val3]); diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DeleteItemTransform.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DeleteItemTransform.dfy index 2042879e7..ee971d997 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DeleteItemTransform.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DeleteItemTransform.dfy @@ -66,8 +66,79 @@ module DeleteItemTransform { method Output(config: Config, input: DeleteItemOutputTransformInput) returns (output: Result) - ensures output.Success? && output.value.transformedOutput == input.sdkOutput + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-deleteitem + //= type=implication + //# After a [DeleteItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DeleteItem.html) + //# call is made to DynamoDB, + //# the resulting response MUST be modified before + //# being returned to the caller if: + // - there exists an Item Encryptor specified within the + // [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration) + // with a [DynamoDB Table Name](./ddb-item-encryptor.md#dynamodb-table-name) + // equal to the `TableName` on the DeleteItem request. + // - the response contains [Attributes](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DeleteItem.html#DDB-DeleteItem-response-Attributes). + // The response will contain Attributes if the related DeleteItem request's + // [ReturnValues](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DeleteItem.html#DDB-DeleteItem-request-ReturnValues) + // had a value of `ALL_OLD` and an item was deleted. + ensures ( + && output.Success? + && input.originalInput.TableName in config.tableEncryptionConfigs + && input.sdkOutput.Attributes.Some? + ) ==> + && var tableConfig := config.tableEncryptionConfigs[input.originalInput.TableName]; + && var oldHistory := old(tableConfig.itemEncryptor.History.DecryptItem); + && var newHistory := tableConfig.itemEncryptor.History.DecryptItem; + + && |newHistory| == |oldHistory|+1 + && Seq.Last(newHistory).output.Success? + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-deleteitem + //= type=implication + //# In this case, the [Item Encryptor](./ddb-item-encryptor.md) MUST perform + //# [Decrypt Item](./decrypt-item.md) where the input + //# [DynamoDB Item](./decrypt-item.md#dynamodb-item) + //# is the `Attributes` field in the original response + && Seq.Last(newHistory).input.encryptedItem == input.sdkOutput.Attributes.value + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-deleteitem + //= type=implication + //# Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. + && RemoveBeacons(tableConfig, Seq.Last(newHistory).output.value.plaintextItem).Success? + && var item := RemoveBeacons(tableConfig, Seq.Last(newHistory).output.value.plaintextItem).value; + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-deleteitem + //= type=implication + //# The DeleteItem response's `Attributes` field MUST be + //# replaced by the encrypted DynamoDb Item outputted above. + && output.value.transformedOutput.Attributes.Some? + && (item == output.value.transformedOutput.Attributes.value) + + // Passthrough the response if the above specification is not met + ensures ( + && output.Success? + && ( + || input.originalInput.TableName !in config.tableEncryptionConfigs + || input.sdkOutput.Attributes.None? + ) + ) ==> + output.value.transformedOutput == input.sdkOutput + + requires ValidConfig?(config) + ensures ValidConfig?(config) + modifies ModifiesConfig(config) { - return Success(DeleteItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + var tableName := input.originalInput.TableName; + if tableName !in config.tableEncryptionConfigs || input.sdkOutput.Attributes.None? + { + return Success(DeleteItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + } + var tableConfig := config.tableEncryptionConfigs[tableName]; + var decryptRes := tableConfig.itemEncryptor.DecryptItem( + EncTypes.DecryptItemInput(encryptedItem:=input.sdkOutput.Attributes.value) + ); + var decrypted :- MapError(decryptRes); + var item :- RemoveBeacons(tableConfig, decrypted.plaintextItem); + return Success(DeleteItemOutputTransformOutput(transformedOutput := input.sdkOutput.(Attributes := Some(item)))); } } diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DynamoDbMiddlewareSupport.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DynamoDbMiddlewareSupport.dfy index 7dadca820..91432e7f7 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DynamoDbMiddlewareSupport.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/DynamoDbMiddlewareSupport.dfy @@ -32,6 +32,15 @@ module DynamoDbMiddlewareSupport { .MapFailure(e => E(e)) } + // IsSigned returned whether this attribute is signed according to this config + predicate method {:opaque} IsSigned( + config : ValidTableConfig, + attr : string + ) + { + BS.IsSigned(config.itemEncryptor.config.attributeActionsOnEncrypt, attr) + } + // TestConditionExpression fails if a condition expression is not suitable for the // given encryption schema. // Generally this means no encrypted attribute is referenced. diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/PutItemTransform.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/PutItemTransform.dfy index 7cec499cf..5b2e96ff9 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/PutItemTransform.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/PutItemTransform.dfy @@ -83,8 +83,79 @@ module PutItemTransform { method Output(config: Config, input: PutItemOutputTransformInput) returns (output: Result) - ensures output.Success? && output.value.transformedOutput == input.sdkOutput + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-putitem + //= type=implication + //# After a [PutItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html) + //# call is made to DynamoDB, + //# the resulting response MUST be modified before + //# being returned to the caller if: + // - there exists an Item Encryptor specified within the + // [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration) + // with a [DynamoDB Table Name](./ddb-item-encryptor.md#dynamodb-table-name) + // equal to the `TableName` on the PutItem request. + // - the response contains [Attributes](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-response-Attributes). + // The response will contain Attributes if the related PutItem request's + // [ReturnValues](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-ReturnValues) + // had a value of `ALL_OLD` and the PutItem call replaced a pre-existing item. + ensures ( + && output.Success? + && input.originalInput.TableName in config.tableEncryptionConfigs + && input.sdkOutput.Attributes.Some? + ) ==> + && var tableConfig := config.tableEncryptionConfigs[input.originalInput.TableName]; + && var oldHistory := old(tableConfig.itemEncryptor.History.DecryptItem); + && var newHistory := tableConfig.itemEncryptor.History.DecryptItem; + + && |newHistory| == |oldHistory|+1 + && Seq.Last(newHistory).output.Success? + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-putitem + //= type=implication + //# In this case, the [Item Encryptor](./ddb-item-encryptor.md) MUST perform + //# [Decrypt Item](./decrypt-item.md) where the input + //# [DynamoDB Item](./decrypt-item.md#dynamodb-item) + //# is the `Attributes` field in the original response + && Seq.Last(newHistory).input.encryptedItem == input.sdkOutput.Attributes.value + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-putitem + //= type=implication + //# Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. + && RemoveBeacons(tableConfig, Seq.Last(newHistory).output.value.plaintextItem).Success? + && var item := RemoveBeacons(tableConfig, Seq.Last(newHistory).output.value.plaintextItem).value; + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-putitem + //= type=implication + //# The PutItem response's `Attributes` field MUST be + //# replaced by the encrypted DynamoDb Item outputted above. + && output.value.transformedOutput.Attributes.Some? + && (item == output.value.transformedOutput.Attributes.value) + + // Passthrough the response if the above specification is not met + ensures ( + && output.Success? + && ( + || input.originalInput.TableName !in config.tableEncryptionConfigs + || input.sdkOutput.Attributes.None? + ) + ) ==> + output.value.transformedOutput == input.sdkOutput + + requires ValidConfig?(config) + ensures ValidConfig?(config) + modifies ModifiesConfig(config) { - return Success(PutItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + var tableName := input.originalInput.TableName; + if tableName !in config.tableEncryptionConfigs || input.sdkOutput.Attributes.None? + { + return Success(PutItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + } + var tableConfig := config.tableEncryptionConfigs[tableName]; + var decryptRes := tableConfig.itemEncryptor.DecryptItem( + EncTypes.DecryptItemInput(encryptedItem:=input.sdkOutput.Attributes.value) + ); + var decrypted :- MapError(decryptRes); + var item :- RemoveBeacons(tableConfig, decrypted.plaintextItem); + return Success(PutItemOutputTransformOutput(transformedOutput := input.sdkOutput.(Attributes := Some(item)))); } } diff --git a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/UpdateItemTransform.dfy b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/UpdateItemTransform.dfy index 0e66fba1d..6e51ec902 100644 --- a/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/UpdateItemTransform.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbEncryptionTransforms/src/UpdateItemTransform.dfy @@ -65,8 +65,142 @@ module UpdateItemTransform { method Output(config: Config, input: UpdateItemOutputTransformInput) returns (output: Result) - ensures output.Success? && output.value.transformedOutput == input.sdkOutput + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-updateitem + //= type=implication + //# After a [UpdateItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html) + //# call is made to DynamoDB, + //# the resulting response MUST be modified before + //# being returned to the caller if: + // - there exists an Item Encryptor specified within the + // [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration) + // with a [DynamoDB Table Name](./ddb-item-encryptor.md#dynamodb-table-name) + // equal to the `TableName` on the UpdateItem request. + // - the response contains [Attributes](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html#DDB-UpdateItem-response-Attributes). + // - the original UpdateItem request had a + // [ReturnValues](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html#DDB-UpdateItem-request-ReturnValues) + // with a value of `ALL_OLD` or `ALL_NEW`. + ensures ( + && output.Success? + && input.originalInput.TableName in config.tableEncryptionConfigs + && input.sdkOutput.Attributes.Some? + && input.originalInput.ReturnValues.Some? + && ( + || input.originalInput.ReturnValues.value.ALL_OLD? + || input.originalInput.ReturnValues.value.ALL_NEW? + ) + ) ==> + && var tableConfig := config.tableEncryptionConfigs[input.originalInput.TableName]; + && var oldHistory := old(tableConfig.itemEncryptor.History.DecryptItem); + && var newHistory := tableConfig.itemEncryptor.History.DecryptItem; + + && |newHistory| == |oldHistory|+1 + && Seq.Last(newHistory).output.Success? + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-updateitem + //= type=implication + //# In this case, the [Item Encryptor](./ddb-item-encryptor.md) MUST perform + //# [Decrypt Item](./decrypt-item.md) where the input + //# [DynamoDB Item](./decrypt-item.md#dynamodb-item) + //# is the `Attributes` field in the original response + && Seq.Last(newHistory).input.encryptedItem == input.sdkOutput.Attributes.value + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-updateitem + //= type=implication + //# Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. + && RemoveBeacons(tableConfig, Seq.Last(newHistory).output.value.plaintextItem).Success? + && var item := RemoveBeacons(tableConfig, Seq.Last(newHistory).output.value.plaintextItem).value; + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-updateitem + //= type=implication + //# The UpdateItem response's `Attributes` field MUST be + //# replaced by the encrypted DynamoDb Item outputted above. + && output.value.transformedOutput.Attributes.Some? + && (item == output.value.transformedOutput.Attributes.value) + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-updateitem + //= type=implication + //# In all other cases, the UpdateItem response MUST NOT be modified. + ensures ( + && output.Success? + && ( + || input.originalInput.TableName !in config.tableEncryptionConfigs + || input.sdkOutput.Attributes.None? + ) + ) ==> ( + && output.value.transformedOutput == input.sdkOutput + ) + + ensures ( + && output.Success? + && input.originalInput.TableName in config.tableEncryptionConfigs + && input.sdkOutput.Attributes.Some? + && (input.originalInput.ReturnValues.Some? ==> ( + || input.originalInput.ReturnValues.value.UPDATED_NEW? + || input.originalInput.ReturnValues.value.UPDATED_OLD? + ) + ) + ) ==> ( + && var tableConfig := config.tableEncryptionConfigs[input.originalInput.TableName]; + && output.value.transformedOutput == input.sdkOutput + && forall k <- input.sdkOutput.Attributes.value.Keys :: !IsSigned(tableConfig, k) + ) + + //= specification/dynamodb-encryption-client/ddb-sdk-integration.md#decrypt-after-updateitem + //= type=implication + //# Additionally, if a value of `UPDATED_OLD` or `UPDATED_NEW` was used, + //# and any Attributes in the response are authenticated + //# per the [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration), + //# an error MUST be raised. + ensures ( + && input.originalInput.TableName in config.tableEncryptionConfigs + && var tableConfig := config.tableEncryptionConfigs[input.originalInput.TableName]; + && input.sdkOutput.Attributes.Some? + && (input.originalInput.ReturnValues.Some? ==> ( + || input.originalInput.ReturnValues.value.UPDATED_NEW? + || input.originalInput.ReturnValues.value.UPDATED_OLD? + ) + ) + && exists k <- input.sdkOutput.Attributes.value.Keys :: IsSigned(tableConfig, k) + ) ==> + output.Failure? + + requires ValidConfig?(config) + ensures ValidConfig?(config) + modifies ModifiesConfig(config) { - return Success(UpdateItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + var tableName := input.originalInput.TableName; + + if + || tableName !in config.tableEncryptionConfigs + || input.sdkOutput.Attributes.None? + { + return Success(UpdateItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + } + + var tableConfig := config.tableEncryptionConfigs[tableName]; + var attributes := input.sdkOutput.Attributes.value; + + if !( + && input.originalInput.ReturnValues.Some? + && ( + || input.originalInput.ReturnValues.value.ALL_NEW? + || input.originalInput.ReturnValues.value.ALL_OLD?) + ) + { + // This error should not be possible to reach if we assume the DDB API contract is correct. + // We include this runtime check for defensive purposes. + :- Need(forall k <- attributes.Keys :: !IsSigned(tableConfig, k), + E("UpdateItems response contains signed attributes, but does not include the entire item which is required for verification.")); + + return Success(UpdateItemOutputTransformOutput(transformedOutput := input.sdkOutput)); + } + + var decryptRes := tableConfig.itemEncryptor.DecryptItem( + EncTypes.DecryptItemInput(encryptedItem:=attributes) + ); + var decrypted :- MapError(decryptRes); + var item :- RemoveBeacons(tableConfig, decrypted.plaintextItem); + return Success(UpdateItemOutputTransformOutput(transformedOutput := input.sdkOutput.(Attributes := Some(item)))); } } diff --git a/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/AwsCryptographyDbEncryptionSdkDynamoDbItemEncryptorOperations.dfy b/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/AwsCryptographyDbEncryptionSdkDynamoDbItemEncryptorOperations.dfy index 7d56fd9cd..5ea299388 100644 --- a/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/AwsCryptographyDbEncryptionSdkDynamoDbItemEncryptorOperations.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/AwsCryptographyDbEncryptionSdkDynamoDbItemEncryptorOperations.dfy @@ -525,6 +525,7 @@ module AwsCryptographyDbEncryptionSdkDynamoDbItemEncryptorOperations refines Abs function method GetItemNames(item : ComAmazonawsDynamodbTypes.AttributeMap) : string { + // We happen to order these values, but this ordering MUST NOT be relied upon. var keys := SortedSets.ComputeSetToOrderedSequence2(item.Keys, CharLess); if |keys| == 0 then "item is empty" diff --git a/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/Index.dfy b/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/Index.dfy index d226f16ac..9f88ab83e 100644 --- a/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/Index.dfy +++ b/DynamoDbEncryption/dafny/DynamoDbItemEncryptor/src/Index.dfy @@ -93,6 +93,7 @@ module message := "Sort key attribute action MUST be SIGN_ONLY" )); + // We happen to order these values, but this ordering MUST NOT be relied upon. var attributeNames : seq := SortedSets.ComputeSetToOrderedSequence2(config.attributeActionsOnEncrypt.Keys, CharLess); for i := 0 to |attributeNames| invariant forall j | 0 <= j < i :: diff --git a/DynamoDbEncryption/runtimes/java/src/main/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/DynamoDbEncryptionInterceptor.java b/DynamoDbEncryption/runtimes/java/src/main/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/DynamoDbEncryptionInterceptor.java index 78e8848be..bdfa8306a 100644 --- a/DynamoDbEncryption/runtimes/java/src/main/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/DynamoDbEncryptionInterceptor.java +++ b/DynamoDbEncryption/runtimes/java/src/main/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/DynamoDbEncryptionInterceptor.java @@ -252,6 +252,17 @@ public SdkResponse modifyResponse(Context.ModifyResponse context, ExecutionAttri .sdkHttpResponse(originalResponse.sdkHttpResponse()) .build(); break; + } case "DeleteItem": { + DeleteItemResponse transformedResponse = transformer.DeleteItemOutputTransform( + DeleteItemOutputTransformInput.builder() + .sdkOutput((DeleteItemResponse) originalResponse) + .originalInput((DeleteItemRequest) originalRequest) + .build()).transformedOutput(); + outgoingResponse = transformedResponse.toBuilder() + .responseMetadata(((DeleteItemResponse) originalResponse).responseMetadata()) + .sdkHttpResponse(originalResponse.sdkHttpResponse()) + .build(); + break; } case "ExecuteStatement": { ExecuteStatementResponse transformedResponse = transformer.ExecuteStatementOutputTransform( ExecuteStatementOutputTransformInput.builder() diff --git a/DynamoDbEncryption/runtimes/java/src/main/sdkv1/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DoNotEncrypt.java b/DynamoDbEncryption/runtimes/java/src/main/sdkv1/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DoNotEncrypt.java index 45f490968..a6c11bdba 100644 --- a/DynamoDbEncryption/runtimes/java/src/main/sdkv1/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DoNotEncrypt.java +++ b/DynamoDbEncryption/runtimes/java/src/main/sdkv1/com/amazonaws/services/dynamodbv2/datamodeling/encryption/DoNotEncrypt.java @@ -21,6 +21,9 @@ import java.lang.annotation.Target; /** + * Warning: This annotation only works with the DynamoDBMapper for AWS SDK for Java 1.x. + * If you are using the AWS SDK for Java 2.x, use @DynamoDbEncryptionSignOnly instead. + * * Prevents the associated item (class or attribute) from being encrypted. * *

For guidance on performing a safe data model change procedure, please see For guidance on performing a safe data model change procedure, please see putItem = putResponse.attributes(); + assertNotNull(putItem); + assertEquals(partitionValue, putItem.get(TEST_PARTITION_NAME).s()); + assertEquals(sortValue, putItem.get(TEST_SORT_NAME).n()); + assertEquals(attrValue, putItem.get(TEST_ATTR_NAME).s()); // Get Item back from table Map keyToGet = createTestKey(partitionValue, sortValue); @@ -87,6 +99,147 @@ public void TestPutItemGetItem() { assertEquals(attrValue, returnedItem.get(TEST_ATTR_NAME).s()); } + // Test that if we update a DO_NOTHING attribute, + // we correctly decrypt the ALL_* return values + @Test + public void TestUpdateItemReturnAll() { + // Put item into table + String partitionValue = "update_ALL"; + String sortValue = randomNum; + String attrValue = "bar"; + String attrValue2 = "hello world"; + Map item = createTestItem(partitionValue, sortValue, attrValue, attrValue2); + + PutItemRequest putRequest = PutItemRequest.builder() + .tableName(TEST_TABLE_NAME) + .item(item) + .returnValues(ReturnValue.ALL_OLD) + .build(); + + ddbKmsKeyring.putItem(putRequest); + + // Update unsigned attribute, and return ALL_OLD + Map keyToGet = createTestKey(partitionValue, sortValue); + UpdateItemRequest updateRequest = UpdateItemRequest.builder() + .key(keyToGet) + .returnValues(ReturnValue.ALL_OLD) + .updateExpression("SET #D = :d") + .expressionAttributeNames(Collections.singletonMap("#D", "attr2")) + .expressionAttributeValues(Collections.singletonMap(":d", AttributeValue.fromS("updated"))) + .tableName(TEST_TABLE_NAME) + .build(); + + UpdateItemResponse updateResponse = ddbKmsKeyring.updateItem(updateRequest); + assertEquals(200, updateResponse.sdkHttpResponse().statusCode()); + Map returnedItem = updateResponse.attributes(); + assertNotNull(returnedItem); + assertEquals(partitionValue, returnedItem.get(TEST_PARTITION_NAME).s()); + assertEquals(sortValue, returnedItem.get(TEST_SORT_NAME).n()); + assertEquals(attrValue, returnedItem.get(TEST_ATTR_NAME).s()); + assertEquals("hello world", returnedItem.get(TEST_ATTR2_NAME).s()); + + // Update unsigned attribute, and return ALL_NEW + UpdateItemRequest updateRequest2 = updateRequest.toBuilder() + .returnValues(ReturnValue.ALL_NEW) + .expressionAttributeValues(Collections.singletonMap(":d", AttributeValue.fromS("updated2"))) + .build(); + + UpdateItemResponse updateResponse2 = ddbKmsKeyring.updateItem(updateRequest2); + assertEquals(200, updateResponse2.sdkHttpResponse().statusCode()); + Map returnedItem2 = updateResponse2.attributes(); + assertNotNull(returnedItem2); + assertEquals(partitionValue, returnedItem2.get(TEST_PARTITION_NAME).s()); + assertEquals(sortValue, returnedItem2.get(TEST_SORT_NAME).n()); + assertEquals(attrValue, returnedItem2.get(TEST_ATTR_NAME).s()); + assertEquals("updated2", returnedItem2.get(TEST_ATTR2_NAME).s()); + } + + // Test that if we update a DO_NOTHING attribute, + // we correctly pass through the UPDATED_* return values + @Test + public void TestUpdateItemReturnUpdated() { + // Put item into table + String partitionValue = "update_UPDATE"; + String sortValue = randomNum; + String attrValue = "bar"; + String attrValue2 = "hello world"; + Map item = createTestItem(partitionValue, sortValue, attrValue, attrValue2); + + PutItemRequest putRequest = PutItemRequest.builder() + .tableName(TEST_TABLE_NAME) + .item(item) + .returnValues(ReturnValue.ALL_OLD) + .build(); + + ddbKmsKeyring.putItem(putRequest); + + // Update unsigned attribute, and return UPDATED_OLD + Map keyToGet = createTestKey(partitionValue, sortValue); + UpdateItemRequest updateRequest = UpdateItemRequest.builder() + .key(keyToGet) + .returnValues(ReturnValue.UPDATED_OLD) + .updateExpression("SET #D = :d") + .expressionAttributeNames(Collections.singletonMap("#D", "attr2")) + .expressionAttributeValues(Collections.singletonMap(":d", AttributeValue.fromS("updated"))) + .tableName(TEST_TABLE_NAME) + .build(); + + UpdateItemResponse updateResponse = ddbKmsKeyring.updateItem(updateRequest); + assertEquals(200, updateResponse.sdkHttpResponse().statusCode()); + Map returnedItem = updateResponse.attributes(); + assertNotNull(returnedItem); + assertFalse(returnedItem.containsKey(TEST_ATTR_NAME)); + assertEquals("hello world", returnedItem.get(TEST_ATTR2_NAME).s()); + + // Update unsigned attribute, and return ALL_NEW + UpdateItemRequest updateRequest2 = updateRequest.toBuilder() + .returnValues(ReturnValue.UPDATED_NEW) + .expressionAttributeValues(Collections.singletonMap(":d", AttributeValue.fromS("updated2"))) + .build(); + + UpdateItemResponse updateResponse2 = ddbKmsKeyring.updateItem(updateRequest2); + assertEquals(200, updateResponse2.sdkHttpResponse().statusCode()); + Map returnedItem2 = updateResponse2.attributes(); + assertNotNull(returnedItem2); + assertFalse(returnedItem2.containsKey(TEST_ATTR_NAME)); + assertEquals("updated2", returnedItem2.get(TEST_ATTR2_NAME).s()); + } + + @Test + public void TestDeleteItem() { + // Put item into table + String partitionValue = "delete"; + String sortValue = randomNum; + String attrValue = "bar"; + String attrValue2 = "hello world"; + Map item = createTestItem(partitionValue, sortValue, attrValue, attrValue2); + + PutItemRequest putRequest = PutItemRequest.builder() + .tableName(TEST_TABLE_NAME) + .item(item) + .build(); + + PutItemResponse putResponse = ddbKmsKeyring.putItem(putRequest); + assertEquals(200, putResponse.sdkHttpResponse().statusCode()); + + // Delete item from table, set ReturnValues to ALL_OLD to return deleted item + Map keyToGet = createTestKey(partitionValue, sortValue); + + DeleteItemRequest deleteRequest = DeleteItemRequest.builder() + .key(keyToGet) + .tableName(TEST_TABLE_NAME) + .returnValues(ReturnValue.ALL_OLD) + .build(); + + DeleteItemResponse deleteResponse = ddbKmsKeyring.deleteItem(deleteRequest); + assertEquals(200, deleteResponse.sdkHttpResponse().statusCode()); + Map returnedItem = deleteResponse.attributes(); + assertNotNull(returnedItem); + assertEquals(partitionValue, returnedItem.get(TEST_PARTITION_NAME).s()); + assertEquals(sortValue, returnedItem.get(TEST_SORT_NAME).n()); + assertEquals(attrValue, returnedItem.get(TEST_ATTR_NAME).s()); + } + @Test public void TestBatchWriteBatchGet() { // Batch write items to table diff --git a/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientIntegrationTests.java b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientIntegrationTests.java index 207fb41f4..ae84494ba 100644 --- a/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientIntegrationTests.java +++ b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientIntegrationTests.java @@ -15,8 +15,12 @@ import software.amazon.awssdk.enhanced.dynamodb.Key; import software.amazon.awssdk.enhanced.dynamodb.TableSchema; import software.amazon.awssdk.enhanced.dynamodb.model.GetItemEnhancedRequest; +import software.amazon.awssdk.enhanced.dynamodb.model.PutItemEnhancedRequest; +import software.amazon.awssdk.enhanced.dynamodb.model.PutItemEnhancedResponse; +import software.amazon.awssdk.enhanced.dynamodb.model.UpdateItemEnhancedRequest; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.DynamoDbException; +import software.amazon.awssdk.services.dynamodb.model.ReturnValue; import software.amazon.awssdk.services.kms.model.KmsException; import software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.validdatamodels.*; @@ -31,6 +35,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; import static org.testng.Assert.assertEquals; import static software.amazon.cryptography.dbencryptionsdk.dynamodb.TestUtils.*; @@ -41,6 +46,11 @@ public class DynamoDbEncryptionEnhancedClientIntegrationTests { + // Some integration tests MUST mutate the state of the DDB table. + // For such tests, include a random number in the primary key + // to avoid conflicts between distributed test runners sharing a table. + private int randomNum = ThreadLocalRandom.current().nextInt(Integer.MIN_VALUE, Integer.MAX_VALUE ); + private static DynamoDbEnhancedClient initEnhancedClientWithInterceptor( final TableSchema schemaOnEncrypt, final List allowedUnsignedAttributes, @@ -75,6 +85,46 @@ private static DynamoDbEnhancedClient initEnhancedClientWithInterceptor( .build(); } + private static DynamoDbEnhancedClient createEnhancedClientForLegacyClass(DynamoDBEncryptor oldEncryptor, TableSchema schemaOnEncrypt) { + Map legacyActions = new HashMap<>(); + legacyActions.put("partition_key", CryptoAction.SIGN_ONLY); + legacyActions.put("sort_key", CryptoAction.SIGN_ONLY); + legacyActions.put("encryptAndSign", CryptoAction.ENCRYPT_AND_SIGN); + legacyActions.put("signOnly", CryptoAction.SIGN_ONLY); + legacyActions.put("doNothing", CryptoAction.DO_NOTHING); + LegacyOverride legacyOverride = LegacyOverride + .builder() + .encryptor(oldEncryptor) + .policy(LegacyPolicy.FORCE_LEGACY_ENCRYPT_ALLOW_LEGACY_DECRYPT) + .attributeActionsOnEncrypt(legacyActions) + .build(); + + Map tableConfigs = new HashMap<>(); + tableConfigs.put(TEST_TABLE_NAME, + DynamoDbEnhancedTableEncryptionConfig.builder() + .logicalTableName(TEST_TABLE_NAME) + .keyring(createKmsKeyring()) + .allowedUnsignedAttributes(Arrays.asList("doNothing")) + .schemaOnEncrypt(schemaOnEncrypt) + .legacyOverride(legacyOverride) + .build()); + DynamoDbEncryptionInterceptor interceptor = + DynamoDbEnhancedClientEncryption.CreateDynamoDbEncryptionInterceptor( + CreateDynamoDbEncryptionInterceptorInput.builder() + .tableEncryptionConfigs(tableConfigs) + .build() + ); + DynamoDbClient ddb = DynamoDbClient.builder() + .overrideConfiguration( + ClientOverrideConfiguration.builder() + .addExecutionInterceptor(interceptor) + .build()) + .build(); + return DynamoDbEnhancedClient.builder() + .dynamoDbClient(ddb) + .build(); + } + @Test public void TestPutAndGet() { TableSchema schemaOnEncrypt = TableSchema.fromBean(SimpleClass.class); @@ -109,6 +159,35 @@ public void TestPutAndGet() { assertEquals(result.getDoNothing(), "fizzbuzz"); } + @Test + public void TestPutAndGetAllTypes() { + TableSchema schemaOnEncrypt = TableSchema.fromBean(AllTypesClass.class); + List allowedUnsignedAttributes = Collections.singletonList("doNothing"); + DynamoDbEnhancedClient enhancedClient = + initEnhancedClientWithInterceptor(schemaOnEncrypt, allowedUnsignedAttributes, null, null); + + DynamoDbTable table = enhancedClient.table(TEST_TABLE_NAME, schemaOnEncrypt); + + AllTypesClass record = AllTypesClass.createTestItem("EnhancedPutGetAllTypes", 1); + + // Put an item into DDB such that it also returns back the item. + PutItemEnhancedResponse putItemResp = table.putItemWithResponse( + (PutItemEnhancedRequest.Builder requestBuilder) + -> requestBuilder.item(record) + .returnValues(ReturnValue.ALL_OLD)); + assertEquals(putItemResp.attributes(), record); + + // Get the item back from the table + Key key = Key.builder() + .partitionValue("EnhancedPutGetAllTypes").sortValue(1) + .build(); + + // Get the item by using the key. + AllTypesClass result = table.getItem( + (GetItemEnhancedRequest.Builder requestBuilder) -> requestBuilder.key(key)); + assertEquals(result, record); + } + @Test public void TestPutAndGetAnnotatedFlattenedBean() { final String PARTITION = "AnnotatedFlattenedBean"; @@ -236,20 +315,6 @@ public void TestGetLegacyItem() { mapper.save(record); - // Configure EnhancedClient with Legacy behavior - Map legacyActions = new HashMap<>(); - legacyActions.put("partition_key", CryptoAction.SIGN_ONLY); - legacyActions.put("sort_key", CryptoAction.SIGN_ONLY); - legacyActions.put("encryptAndSign", CryptoAction.ENCRYPT_AND_SIGN); - legacyActions.put("signOnly", CryptoAction.SIGN_ONLY); - legacyActions.put("doNothing", CryptoAction.DO_NOTHING); - LegacyOverride legacyOverride = LegacyOverride - .builder() - .encryptor(oldEncryptor) - .policy(LegacyPolicy.FORCE_LEGACY_ENCRYPT_ALLOW_LEGACY_DECRYPT) - .attributeActionsOnEncrypt(legacyActions) - .build(); - TableSchema schemaOnEncrypt = TableSchema.fromBean(LegacyClass.class); DynamoDbEnhancedClient enhancedClient = createEnhancedClientForLegacyClass(oldEncryptor, schemaOnEncrypt); @@ -300,44 +365,58 @@ public void TestWriteLegacyItem() { assertEquals("fizzbuzz", result.getDoNothing()); } - private static DynamoDbEnhancedClient createEnhancedClientForLegacyClass(DynamoDBEncryptor oldEncryptor, TableSchema schemaOnEncrypt) { - Map legacyActions = new HashMap<>(); - legacyActions.put("partition_key", CryptoAction.SIGN_ONLY); - legacyActions.put("sort_key", CryptoAction.SIGN_ONLY); - legacyActions.put("encryptAndSign", CryptoAction.ENCRYPT_AND_SIGN); - legacyActions.put("signOnly", CryptoAction.SIGN_ONLY); - legacyActions.put("doNothing", CryptoAction.DO_NOTHING); - LegacyOverride legacyOverride = LegacyOverride - .builder() - .encryptor(oldEncryptor) - .policy(LegacyPolicy.FORCE_LEGACY_ENCRYPT_ALLOW_LEGACY_DECRYPT) - .attributeActionsOnEncrypt(legacyActions) - .build(); + @Test + public void TestDelete() { + TableSchema schemaOnEncrypt = TableSchema.fromBean(AllTypesClass.class); + List allowedUnsignedAttributes = Collections.singletonList("doNothing"); + DynamoDbEnhancedClient enhancedClient = + initEnhancedClientWithInterceptor(schemaOnEncrypt, allowedUnsignedAttributes, null, null); - Map tableConfigs = new HashMap<>(); - tableConfigs.put(TEST_TABLE_NAME, - DynamoDbEnhancedTableEncryptionConfig.builder() - .logicalTableName(TEST_TABLE_NAME) - .keyring(createKmsKeyring()) - .allowedUnsignedAttributes(Arrays.asList("doNothing")) - .schemaOnEncrypt(schemaOnEncrypt) - .legacyOverride(legacyOverride) - .build()); - DynamoDbEncryptionInterceptor interceptor = - DynamoDbEnhancedClientEncryption.CreateDynamoDbEncryptionInterceptor( - CreateDynamoDbEncryptionInterceptorInput.builder() - .tableEncryptionConfigs(tableConfigs) - .build() - ); - DynamoDbClient ddb = DynamoDbClient.builder() - .overrideConfiguration( - ClientOverrideConfiguration.builder() - .addExecutionInterceptor(interceptor) - .build()) - .build(); - return DynamoDbEnhancedClient.builder() - .dynamoDbClient(ddb) + DynamoDbTable table = enhancedClient.table(TEST_TABLE_NAME, schemaOnEncrypt); + + AllTypesClass record = AllTypesClass.createTestItem("EnhancedDelete", randomNum); + + // Put an item into an Amazon DynamoDB table. + table.putItem(record); + + // Get the item back from the table + Key key = Key.builder() + .partitionValue("EnhancedDelete").sortValue(randomNum) .build(); + + // Get the item by using the key. + AllTypesClass result = table.deleteItem(key); + assertEquals(result, record); + } + + @Test + public void TestUpdate() { + TableSchema schemaOnEncrypt = TableSchema.fromBean(AllTypesClass.class); + List allowedUnsignedAttributes = Collections.singletonList("doNothing"); + DynamoDbEnhancedClient enhancedClient = + initEnhancedClientWithInterceptor(schemaOnEncrypt, allowedUnsignedAttributes, null, null); + + DynamoDbTable table = enhancedClient.table(TEST_TABLE_NAME, schemaOnEncrypt); + + AllTypesClass record = AllTypesClass.createTestItem("EnhancedUpdate", 1); + + // Put an item into an Amazon DynamoDB table. + table.putItem(record); + + AllTypesClass doNothingValue = new AllTypesClass(); + doNothingValue.setDoNothing("updatedDoNothing"); + doNothingValue.setPartitionKey("EnhancedUpdate"); + doNothingValue.setSortKey(1); + + // Perform an update only on "doNothing" attribute + AllTypesClass result = table.updateItem( + (UpdateItemEnhancedRequest.Builder requestBuilder) + -> requestBuilder.item(doNothingValue) + .ignoreNulls(true) + ); + // EnhancedClient uses ReturnValues of ALL_NEW, so compare against put item with update + record.setDoNothing("updatedDoNothing"); + assertEquals(result, record); } @Test( diff --git a/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/validdatamodels/AllTypesClass.java b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/validdatamodels/AllTypesClass.java new file mode 100644 index 000000000..ef50fd2fe --- /dev/null +++ b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/validdatamodels/AllTypesClass.java @@ -0,0 +1,386 @@ +package software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.validdatamodels; + +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbAttribute; +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbBean; +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbIgnoreNulls; +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbPartitionKey; +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbSortKey; +import software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.DynamoDbEncryptionDoNothing; +import software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.DynamoDbEncryptionSignOnly; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * This class is used by the Enhanced Client Tests + */ + +@DynamoDbBean +public class AllTypesClass { + + private String partitionKey; + private int sortKey; + + // One attribute for every DDB attribute type and ENCRYPT_AND_SIGN/SIGN_ONLY pair + private String encryptString; + private String signString; + private Double encryptNum; + private Double signNum; + private ByteBuffer encryptBinary; + private ByteBuffer signBinary; + private Boolean encryptBool; + private Boolean signBool; + private String encryptExpectedNull; // This should always be null, define no setters + private String signExpectedNull; // This should always be null, define no setters + private List encryptList; + private List signList; + private Map encryptMap; + private Map signMap; + private Set encryptStringSet; + private Set signStringSet; + private Set encryptNumSet; + private Set signNumSet; + private Set encryptBinarySet; + private Set signBinarySet; + + // And one doNothing for good measure + private String doNothing; + + @DynamoDbPartitionKey + @DynamoDbAttribute(value = "partition_key") + public String getPartitionKey() { + return this.partitionKey; + } + + @DynamoDbSortKey + @DynamoDbAttribute(value = "sort_key") + public int getSortKey() { + return this.sortKey; + } + + @DynamoDbIgnoreNulls + public String getEncryptString() { + return this.encryptString; + } + + @DynamoDbEncryptionSignOnly + @DynamoDbIgnoreNulls + public String getSignString() { + return this.signString; + } + + @DynamoDbIgnoreNulls + public Double getEncryptNum() { + return encryptNum; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public Double getSignNum() { + return signNum; + } + + @DynamoDbIgnoreNulls + public ByteBuffer getEncryptBinary() { + return encryptBinary; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public ByteBuffer getSignBinary() { + return signBinary; + } + + @DynamoDbIgnoreNulls + public Boolean getEncryptBool() { + return encryptBool; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public Boolean getSignBool() { + return signBool; + } + + // This should always return null + public String getEncryptExpectedNull() { + return encryptExpectedNull; + } + + // This should always return null + @DynamoDbEncryptionSignOnly + public String getSignExpectedNull() { + return signExpectedNull; + } + + @DynamoDbIgnoreNulls + public List getEncryptList() { + return encryptList; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public List getSignList() { + return signList; + } + + @DynamoDbIgnoreNulls + public Map getEncryptMap() { + return encryptMap; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public Map getSignMap() { + return signMap; + } + + @DynamoDbIgnoreNulls + public Set getEncryptStringSet() { + return encryptStringSet; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public Set getSignStringSet() { + return signStringSet; + } + + @DynamoDbIgnoreNulls + public Set getEncryptNumSet() { + return encryptNumSet; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public Set getSignNumSet() { + return signNumSet; + } + + @DynamoDbIgnoreNulls + public Set getEncryptBinarySet() { + return encryptBinarySet; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionSignOnly + public Set getSignBinarySet() { + return signBinarySet; + } + + @DynamoDbIgnoreNulls + @DynamoDbEncryptionDoNothing + public String getDoNothing() { + return this.doNothing; + } + + public void setPartitionKey(String partitionKey) { + this.partitionKey = partitionKey; + } + + public void setSortKey(int sortKey) { + this.sortKey = sortKey; + } + + public void setEncryptString(String encryptString) { + this.encryptString = encryptString; + } + + public void setSignString(String signString) { + this.signString = signString; + } + + public void setEncryptNum(Double encryptNum) { + this.encryptNum = encryptNum; + } + + public void setSignNum(Double signNum) { + this.signNum = signNum; + } + + public void setEncryptBinary(ByteBuffer encryptBinary) { + this.encryptBinary = encryptBinary; + } + + public void setSignBinary(ByteBuffer signBinary) { + this.signBinary = signBinary; + } + + public void setEncryptBool(Boolean encryptBool) { + this.encryptBool = encryptBool; + } + + public void setSignBool(Boolean signBool) { + this.signBool = signBool; + } + + public void setEncryptList(List encryptList) { + this.encryptList = encryptList; + } + + public void setSignList(List signList) { + this.signList = signList; + } + + public void setEncryptMap(Map encryptMap) { + this.encryptMap = encryptMap; + } + + public void setSignMap(Map signMap) { + this.signMap = signMap; + } + + public void setEncryptStringSet(Set encryptStringSet) { + this.encryptStringSet = encryptStringSet; + } + + public void setSignStringSet(Set signStringSet) { + this.signStringSet = signStringSet; + } + + public void setEncryptNumSet(Set encryptNumSet) { + this.encryptNumSet = encryptNumSet; + } + + public void setSignNumSet(Set signNumSet) { + this.signNumSet = signNumSet; + } + + public void setEncryptBinarySet(Set encryptBinarySet) { + this.encryptBinarySet = encryptBinarySet; + } + + public void setSignBinarySet(Set signBinarySet) { + this.signBinarySet = signBinarySet; + } + + public void setDoNothing(String doNothing) { + this.doNothing = doNothing; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + + if (obj.getClass() != this.getClass()) { + return false; + } + + final AllTypesClass other = (AllTypesClass) obj; + + if (!(Objects.equals(other.getPartitionKey(), this.partitionKey))) { + return false; + } + if (other.getSortKey() != this.sortKey) { + return false; + } + if (!(Objects.equals(other.getEncryptString(), this.encryptString))) { + return false; + } + if (!(Objects.equals(other.getSignString(), this.signString))) { + return false; + } + if (!(Objects.equals(other.getEncryptNum(), this.encryptNum))) { + return false; + } + if (!(Objects.equals(other.getSignNum(), this.signNum))) { + return false; + } + if (!(Objects.equals(other.getEncryptBinary(), this.encryptBinary))) { + return false; + } + if (!(Objects.equals(other.getSignBinary(), this.signBinary))) { + return false; + } + if (!(Objects.equals(other.getEncryptBool(), this.encryptBool))) { + return false; + } + if (!(Objects.equals(other.getSignBool(), this.signBool))) { + return false; + } + if (other.getEncryptExpectedNull() != null) { + return false; + } + if (other.getSignExpectedNull() != null) { + return false; + } + if (!(Objects.equals(other.getEncryptList(), this.encryptList))) { + return false; + } + if (!(Objects.equals(other.getSignList(), this.signList))) { + return false; + } + if (!(Objects.equals(other.getEncryptMap(), this.encryptMap))) { + return false; + } + if (!(Objects.equals(other.getSignMap(), this.signMap))) { + return false; + } + if (!(Objects.equals(other.getEncryptStringSet(), this.encryptStringSet))) { + return false; + } + if (!(Objects.equals(other.getSignStringSet(), this.signStringSet))) { + return false; + } + if (!(Objects.equals(other.getEncryptNumSet(), this.encryptNumSet))) { + return false; + } + if (!(Objects.equals(other.getSignNumSet(), this.signNumSet))) { + return false; + } + if (!(Objects.equals(other.getEncryptBinarySet(), this.encryptBinarySet))) { + return false; + } + if (!(Objects.equals(other.getSignBinarySet(), this.signBinarySet))) { + return false; + } + if (!(Objects.equals(other.getDoNothing(), this.doNothing))) { + return false; + } + + return true; + } + + public static AllTypesClass createTestItem(String partitionValue, int sortValue) { + AllTypesClass testItem = new AllTypesClass(); + testItem.setPartitionKey(partitionValue); + testItem.setSortKey(sortValue); + testItem.setEncryptString("encryptString"); + testItem.setSignString("signString"); + testItem.setEncryptNum(111.111); + testItem.setSignNum(999.999); + testItem.setEncryptBinary(StandardCharsets.UTF_8.encode("encryptBinary")); + testItem.setSignBinary(StandardCharsets.UTF_8.encode("sortBinary")); + testItem.setEncryptBool(true); + testItem.setSignBool(false); + testItem.setEncryptList(Arrays.asList("encrypt1", "encrypt2", "encrypt3")); + testItem.setSignList(Arrays.asList("sort1", "sort2", "sort3")); + testItem.setEncryptMap(Collections.singletonMap("encryptMap", 1)); + testItem.setSignMap(Collections.singletonMap("sortMap", 2)); + testItem.setEncryptStringSet(new HashSet<>(Arrays.asList("encrypt1", "encrypt2", "encrypt3"))); + testItem.setSignStringSet(new HashSet<>(Arrays.asList("sort1", "sort2", "sort3"))); + testItem.setEncryptNumSet(new HashSet<>(Arrays.asList(1, 2, 3))); + testItem.setSignNumSet(new HashSet<>(Arrays.asList(4, 5, 6))); + testItem.setEncryptBinarySet(new HashSet<>(Arrays.asList( + StandardCharsets.UTF_8.encode("encrypt1"), + StandardCharsets.UTF_8.encode("encrypt2"), + StandardCharsets.UTF_8.encode("encrypt3") + ))); + testItem.setSignBinarySet(new HashSet<>(Arrays.asList( + StandardCharsets.UTF_8.encode("sort1"), + StandardCharsets.UTF_8.encode("sort2"), + StandardCharsets.UTF_8.encode("sort3") + ))); + testItem.setDoNothing("doNothing"); + return testItem; + } +} diff --git a/Examples/runtimes/java/DynamoDbEncryption/build.gradle.kts b/Examples/runtimes/java/DynamoDbEncryption/build.gradle.kts index bcf62436d..3a9abf704 100644 --- a/Examples/runtimes/java/DynamoDbEncryption/build.gradle.kts +++ b/Examples/runtimes/java/DynamoDbEncryption/build.gradle.kts @@ -69,7 +69,6 @@ repositories { dependencies { implementation("software.amazon.cryptography:aws-database-encryption-sdk-dynamodb:${ddbecVersion}") implementation("software.amazon.cryptography:aws-cryptographic-material-providers:${mplVersion}") - implementation(platform("software.amazon.awssdk:bom:2.19.1")) implementation("software.amazon.awssdk:arns") implementation("software.amazon.awssdk:auth") diff --git a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/BeaconStylesSearchableEncryptionExample.java b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/BeaconStylesSearchableEncryptionExample.java index d3ec132e8..4700374cd 100644 --- a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/BeaconStylesSearchableEncryptionExample.java +++ b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/BeaconStylesSearchableEncryptionExample.java @@ -195,8 +195,8 @@ public static void PutItemQueryItemWithBeaconStyles(String ddbTableName, String item1.put("dessert", AttributeValue.builder().s("cake").build()); item1.put("fruit", AttributeValue.builder().s("banana").build()); ArrayList basket = new ArrayList(); - basket.add("banana"); basket.add("apple"); + basket.add("banana"); basket.add("pear"); item1.put("basket", AttributeValue.builder().ss(basket).build()); @@ -207,9 +207,9 @@ public static void PutItemQueryItemWithBeaconStyles(String ddbTableName, String item2.put("fruit", AttributeValue.builder().s("orange").build()); item2.put("dessert", AttributeValue.builder().s("orange").build()); basket = new ArrayList(); - basket.add("strawberry"); - basket.add("blueberry"); basket.add("blackberry"); + basket.add("blueberry"); + basket.add("strawberry"); item2.put("basket", AttributeValue.builder().ss(basket).build()); // 10. Create the DynamoDb Encryption Interceptor @@ -283,8 +283,8 @@ public static void PutItemQueryItemWithBeaconStyles(String ddbTableName, String // Select records where the fruit attribute exists in a particular set ArrayList basket3 = new ArrayList(); basket3.add("boysenberry"); - basket3.add("orange"); basket3.add("grape"); + basket3.add("orange"); expressionAttributeValues.put(":value", AttributeValue.builder().ss(basket3).build()); scanRequest = ScanRequest.builder() diff --git a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/CompoundBeaconSearchableEncryptionExample.java b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/CompoundBeaconSearchableEncryptionExample.java index baabb5e03..e148d9d7e 100644 --- a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/CompoundBeaconSearchableEncryptionExample.java +++ b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/CompoundBeaconSearchableEncryptionExample.java @@ -328,6 +328,9 @@ public static void PutAndQueryItemWithCompoundBeacon(DynamoDbClient ddb, String .expressionAttributeValues(expressionAttributeValues) .build(); + // GSIs do not update instantly + // so if the results come back empty + // we retry after a short sleep for (int i=0; i<10; ++i) { QueryResponse queryResponse = ddb.query(queryRequest); List> attributeValues = queryResponse.items(); diff --git a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/VirtualBeaconSearchableEncryptionExample.java b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/VirtualBeaconSearchableEncryptionExample.java index d6ae06c77..d4379a3d6 100644 --- a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/VirtualBeaconSearchableEncryptionExample.java +++ b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/VirtualBeaconSearchableEncryptionExample.java @@ -436,6 +436,9 @@ public static void PutItemQueryItemWithVirtualBeacon(String ddbTableName, String .expressionAttributeValues(expressionAttributeValues) .build(); + // GSIs do not update instantly + // so if the results come back empty + // we retry after a short sleep for (int i=0; i<10; ++i) { final QueryResponse queryResponse = ddb.query(queryRequest); List> attributeValues = queryResponse.items(); diff --git a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/complexexample/QueryRequests.java b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/complexexample/QueryRequests.java index c91354121..4075a3994 100644 --- a/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/complexexample/QueryRequests.java +++ b/Examples/runtimes/java/DynamoDbEncryption/src/main/java/software/amazon/cryptography/examples/searchableencryption/complexexample/QueryRequests.java @@ -582,22 +582,34 @@ public static void runQuery14(String ddbTableName, DynamoDbClient ddb) { .expressionAttributeValues(query14AttributeValues) .build(); - QueryResponse query14Response = ddb.query(query14Request); - // Validate query was returned successfully - assert 200 == query14Response.sdkHttpResponse().statusCode(); - - // Assert 1 item was returned: `employee1` - assert query14Response.items().size() == 1; - // Known value test: Assert some properties on one of the items - boolean foundKnownValueItemQuery14 = false; - for (Map item : query14Response.items()) { - if (item.get("partition_key").s().equals("employee1")) { - foundKnownValueItemQuery14 = true; - assert item.get("EmployeeID").s().equals("emp_001"); - assert item.get("Location").m().get("Desk").s().equals("3"); - } + // GSIs do not update instantly + // so if the results come back empty + // we retry after a short sleep + for (int i=0; i<10; ++i) { + QueryResponse query14Response = ddb.query(query14Request); + // Validate query was returned successfully + assert 200 == query14Response.sdkHttpResponse().statusCode(); + + // if no results, sleep and try again + if (query14Response.items().size() == 0) { + try {Thread.sleep(20);} catch (Exception e) {} + continue; + } + + // Assert 1 item was returned: `employee1` + assert query14Response.items().size() == 1; + // Known value test: Assert some properties on one of the items + boolean foundKnownValueItemQuery14 = false; + for (Map item : query14Response.items()) { + if (item.get("partition_key").s().equals("employee1")) { + foundKnownValueItemQuery14 = true; + assert item.get("EmployeeID").s().equals("emp_001"); + assert item.get("Location").m().get("Desk").s().equals("3"); + } + } + assert foundKnownValueItemQuery14; + break; } - assert foundKnownValueItemQuery14; } public static void runQuery15(String ddbTableName, DynamoDbClient ddb) { diff --git a/Examples/runtimes/java/Migration/DDBECToAWSDBE/build.gradle.kts b/Examples/runtimes/java/Migration/DDBECToAWSDBE/build.gradle.kts index fa2bcc413..58f3e3fd0 100644 --- a/Examples/runtimes/java/Migration/DDBECToAWSDBE/build.gradle.kts +++ b/Examples/runtimes/java/Migration/DDBECToAWSDBE/build.gradle.kts @@ -68,7 +68,6 @@ repositories { dependencies { implementation("software.amazon.cryptography:aws-database-encryption-sdk-dynamodb:${ddbecVersion}") implementation("software.amazon.cryptography:aws-cryptographic-material-providers:${mplVersion}") - implementation(platform("software.amazon.awssdk:bom:2.19.1")) implementation("software.amazon.awssdk:dynamodb") implementation("software.amazon.awssdk:dynamodb-enhanced") diff --git a/Examples/runtimes/java/Migration/PlaintextToAWSDBE/build.gradle.kts b/Examples/runtimes/java/Migration/PlaintextToAWSDBE/build.gradle.kts index d548fe6f5..b859f2657 100644 --- a/Examples/runtimes/java/Migration/PlaintextToAWSDBE/build.gradle.kts +++ b/Examples/runtimes/java/Migration/PlaintextToAWSDBE/build.gradle.kts @@ -68,7 +68,6 @@ repositories { dependencies { implementation("software.amazon.cryptography:aws-database-encryption-sdk-dynamodb:${ddbecVersion}") implementation("software.amazon.cryptography:aws-cryptographic-material-providers:${mplVersion}") - implementation(platform("software.amazon.awssdk:bom:2.19.1")) implementation("software.amazon.awssdk:dynamodb") implementation("software.amazon.awssdk:dynamodb-enhanced") diff --git a/README.md b/README.md index cfbd48588..6a7e49dd4 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,8 @@ To use the DB-ESDK for DynamoDB in Java, you must have: * **Via Gradle Kotlin** In a Gradle Java Project, add the following to the _dependencies_ section: ```kotlin - implementation("software.amazon.cryptography:aws-database-encryption-sdk-dynamodb:3.1.0") - implementation("software.amazon.cryptography:aws-cryptographic-material-providers:1.0.0") + implementation("software.amazon.cryptography:aws-database-encryption-sdk-dynamodb:3.1.2") + implementation("software.amazon.cryptography:aws-cryptographic-material-providers:1.0.1") implementation(platform("software.amazon.awssdk:bom:2.19.1")) implementation("software.amazon.awssdk:dynamodb") // The following are optional: @@ -92,12 +92,12 @@ To use the DB-ESDK for DynamoDB in Java, you must have: software.amazon.cryptography aws-database-encryption-sdk-dynamodb - 3.1.0 + 3.1.2 software.amazon.cryptography aws-cryptographic-material-providers - 1.0.0 + 1.0.1 diff --git a/TestVectors/dafny/DDBEncryption/src/Index.dfy b/TestVectors/dafny/DDBEncryption/src/Index.dfy index a982f81c1..8a67f6039 100644 --- a/TestVectors/dafny/DDBEncryption/src/Index.dfy +++ b/TestVectors/dafny/DDBEncryption/src/Index.dfy @@ -3,11 +3,13 @@ include "LibraryIndex.dfy" include "TestVectors.dfy" +include "WriteSetPermutations.dfy" module WrappedDDBEncryptionMain { import opened Wrappers import opened DdbEncryptionTestVectors + import WriteSetPermutations import CreateInterceptedDDBClient import FileIO import JSON.API @@ -25,11 +27,13 @@ module WrappedDDBEncryptionMain { } method ASDF() { + WriteSetPermutations.WriteSetPermutations(); var config := MakeEmptyTestVector(); config :- expect AddJson(config, "records.json"); config :- expect AddJson(config, "configs.json"); config :- expect AddJson(config, "data.json"); config :- expect AddJson(config, "iotest.json"); + config :- expect AddJson(config, "PermTest.json"); config.RunAllTests(); } } diff --git a/TestVectors/dafny/DDBEncryption/src/Permute.dfy b/TestVectors/dafny/DDBEncryption/src/Permute.dfy new file mode 100644 index 000000000..1c030a2f3 --- /dev/null +++ b/TestVectors/dafny/DDBEncryption/src/Permute.dfy @@ -0,0 +1,72 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +include "../../../../submodules/MaterialProviders/libraries/src/Wrappers.dfy" + +module {:options "-functionSyntax:4"} Permutations { + import opened Wrappers + + method GeneratePermutations(source : seq) returns (result : seq>) + { + if |source| == 0 { + return []; + } + if |source| == 1 { + return [source]; + } + var A := new T[|source|](i requires 0 <= i < |source| => source[i]); + result := Permute(A.Length, A); + } + + method Swap(A : array, x : nat, y : nat) + requires 0 <= x < A.Length + requires 0 <= y < A.Length + modifies A + { + var tmp := A[x]; + A[x] := A[y]; + A[y] := tmp; + } + + // https://en.wikipedia.org/wiki/Heap%27s_algorithm + // Each step generates the k! permutations that end with the same n-k final elements + method Permute(k : nat, A : array) returns (result : seq>) + requires 0 < k <= A.Length + modifies A + { + if k == 1 { + return [A[..]]; + } else { + var result : seq> := []; + for i := 0 to k { + var next := Permute(k - 1, A); + result := result + next; + if (k % 1) == 0 { + Swap(A, i, k-1); + } else { + Swap(A, 0, k-1); + } + } + return result; + } + } + + method {:test} BasicTests() { + var zero := GeneratePermutations([]); + var one := GeneratePermutations([1]); + var two := GeneratePermutations([1,2]); + var three := GeneratePermutations([1,2,3]); + var four := GeneratePermutations([1,2,3,4]); + expect zero == []; + expect one == [[1]]; + expect two == [[1,2],[2,1]]; + expect three == [[1, 2, 3], [2, 1, 3], [3, 1, 2], [1, 3, 2], [1, 2, 3], [2, 1, 3]]; + expect four == [ + [1, 2, 3, 4], [2, 1, 3, 4], [3, 1, 2, 4], [1, 3, 2, 4], [1, 2, 3, 4], [2, 1, 3, 4], + [4, 1, 3, 2], [1, 4, 3, 2], [3, 4, 1, 2], [4, 3, 1, 2], [4, 1, 3, 2], [1, 4, 3, 2], + [1, 2, 3, 4], [2, 1, 3, 4], [3, 1, 2, 4], [1, 3, 2, 4], [1, 2, 3, 4], [2, 1, 3, 4], + [2, 1, 4, 3], [1, 2, 4, 3], [4, 2, 1, 3], [2, 4, 1, 3], [2, 1, 4, 3], [1, 2, 4, 3] + ]; + } +} + diff --git a/TestVectors/dafny/DDBEncryption/src/TestVectors.dfy b/TestVectors/dafny/DDBEncryption/src/TestVectors.dfy index 1bf899e93..abd94d234 100644 --- a/TestVectors/dafny/DDBEncryption/src/TestVectors.dfy +++ b/TestVectors/dafny/DDBEncryption/src/TestVectors.dfy @@ -39,6 +39,8 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { import KeyVectorsTypes = AwsCryptographyMaterialProvidersTestVectorKeysTypes import KeyVectors import CreateInterceptedDDBClient + import SortedSets + import Seq predicate IsValidInt32(x: int) { -0x8000_0000 <= x < 0x8000_0000} type ConfigName = string @@ -73,6 +75,11 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { failures : seq ) + datatype RoundTripTest = RoundTripTest ( + configs : map, + records : seq + ) + datatype WriteTest = WriteTest ( config : TableConfig, records : seq, @@ -113,6 +120,7 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { configsForIoTest : PairList, configsForModTest : PairList, writeTests : seq, + roundTripTests : seq, decryptTests : seq ) { @@ -129,6 +137,12 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { print |ioTests|, " ioTests.\n"; print |configsForIoTest|, " configsForIoTest.\n"; print |configsForModTest|, " configsForModTest.\n"; + if |roundTripTests| != 0 { + print |roundTripTests[0].configs|, " configs and ", |roundTripTests[0].records|, " records for round trip.\n"; + } + if |roundTripTests| > 1 { + print |roundTripTests[1].configs|, " configs and ", |roundTripTests[1].records|, " records for round trip.\n"; + } Validate(); BasicIoTest(); RunIoTests(); @@ -136,11 +150,13 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { ConfigModTest(); ComplexTests(); WriteTests(); + RoundTripTests(); DecryptTests(); var client :- expect CreateInterceptedDDBClient.CreateVanillaDDBClient(); DeleteTable(client); } + method Validate() { var bad := false; for i := 0 to |globalRecords| { @@ -496,6 +512,57 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { } } + method RoundTripTests() + { + print "RoundTripTests\n"; + for i := 0 to |roundTripTests| { + + var configs := roundTripTests[i].configs; + var records := roundTripTests[i].records; + var keys := SortedSets.ComputeSetToOrderedSequence2(configs.Keys, CharLess); + + for j := 0 to |keys| { + var client :- expect newGazelle(configs[keys[j]]); + for k := 0 to |records| { + OneRoundTripTest(client, records[k]); + } + } + } + } + + method OneRoundTripTest(client : DDB.IDynamoDBClient, record : Record) { + var putInput := DDB.PutItemInput( + TableName := TableName, + Item := record.item, + Expected := None, + ReturnValues := None, + ReturnConsumedCapacity := None, + ReturnItemCollectionMetrics := None, + ConditionalOperator := None, + ConditionExpression := None, + ExpressionAttributeNames := None, + ExpressionAttributeValues := None + ); + var _ :- expect client.PutItem(putInput); + + var getInput := DDB.GetItemInput( + TableName := TableName, + Key := map[HashName := record.item[HashName]], + AttributesToGet := None, + ConsistentRead := None, + ReturnConsumedCapacity := None, + ProjectionExpression := None, + ExpressionAttributeNames := None + ); + var out :- expect client.GetItem(getInput); + expect out.Item.Some?; + if NormalizeItem(out.Item.value) != NormalizeItem(record.item) { + print "\n", NormalizeItem(out.Item.value), "\n", NormalizeItem(record.item), "\n"; + } + expect NormalizeItem(out.Item.value) == NormalizeItem(record.item); + } + + method DecryptTests() { print "DecryptTests\n"; @@ -685,7 +752,7 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { { var exp := NormalizeItem(expected); for i := 0 to |actual| { - if actual[i] == exp { + if NormalizeItem(actual[i]) == exp { return true; } } @@ -781,6 +848,23 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { DDB.AttributeValue.N(nn.value) else value + case SS(s) => + var asSet := Seq.ToSet(s); + DDB.AttributeValue.SS(SortedSets.ComputeSetToOrderedSequence2(asSet, CharLess)) + case NS(s) => + var normList := Seq.MapWithResult(n => Norm.NormalizeNumber(n), s); + if normList.Success? then + var asSet := Seq.ToSet(normList.value); + DDB.AttributeValue.NS(SortedSets.ComputeSetToOrderedSequence2(asSet, CharLess)) + else + value + case BS(s) => + var asSet := Seq.ToSet(s); + DDB.AttributeValue.BS(SortedSets.ComputeSetToOrderedSequence2(asSet, ByteLess)) + case L(list) => + DDB.AttributeValue.L(Seq.Map(n => Normalize(n), list)) + case M(m) => + DDB.AttributeValue.M(map k <- m :: k := Normalize(m[k])) case _ => value } } @@ -962,7 +1046,7 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { function MakeEmptyTestVector() : TestVectorConfig { - TestVectorConfig(MakeCreateTableInput(), [], map[], [], map[], map[], [], [], [], [], [], [], []) + TestVectorConfig(MakeCreateTableInput(), [], map[], [], map[], map[], [], [], [], [], [], [], [], []) } method ParseTestVector(data : JSON, prev : TestVectorConfig) returns (output : Result) @@ -980,6 +1064,7 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { var gsi : seq := []; var tableEncryptionConfigs : map := map[]; var writeTests : seq := []; + var roundTripTests : seq := []; var decryptTests : seq := []; for i := 0 to |data.obj| { @@ -996,6 +1081,7 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { case "GSI" => gsi :- GetGSIs(data.obj[i].1); case "tableEncryptionConfigs" => tableEncryptionConfigs :- GetTableConfigs(data.obj[i].1); case "WriteTests" => writeTests :- GetWriteTests(data.obj[i].1); + case "RoundTripTest" => roundTripTests :- GetRoundTripTests(data.obj[i].1); case "DecryptTests" => decryptTests :- GetDecryptTests(data.obj[i].1); case _ => return Failure("Unexpected top level tag " + data.obj[i].0); } @@ -1016,11 +1102,29 @@ module {:options "-functionSyntax:4"} DdbEncryptionTestVectors { configsForIoTest := prev.configsForIoTest + ioPairs, configsForModTest := prev.configsForModTest + queryPairs, writeTests := prev.writeTests + writeTests, + roundTripTests := prev.roundTripTests + roundTripTests, decryptTests := prev.decryptTests + decryptTests ) ); } + method GetRoundTripTests(data : JSON) returns (output : Result, string>) + { + :- Need(data.Object?, "RoundTripTest Test must be an object."); + var configs : map := map[]; + var records : seq := []; + + for i := 0 to |data.obj| { + var obj := data.obj[i]; + match obj.0 { + case "Configs" => var src :- GetTableConfigs(obj.1); configs := src; + case "Records" => var src :- GetRecords(obj.1); records := src; + case _ => return Failure("Unexpected part of a write test : '" + obj.0 + "'"); + } + } + return Success([RoundTripTest(configs, records)]); + } + method GetWriteTests(data : JSON) returns (output : Result , string>) { :- Need(data.Array?, "Write Test list must be an array."); diff --git a/TestVectors/dafny/DDBEncryption/src/WriteSetPermutations.dfy b/TestVectors/dafny/DDBEncryption/src/WriteSetPermutations.dfy new file mode 100644 index 000000000..e3e3b313d --- /dev/null +++ b/TestVectors/dafny/DDBEncryption/src/WriteSetPermutations.dfy @@ -0,0 +1,142 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +include "../Model/AwsCryptographyDynamoDbEncryptionTypesWrapped.dfy" +include "CreateInterceptedDDBClient.dfy" +include "JsonItem.dfy" +include "Permute.dfy" + +module {:options "-functionSyntax:4"} WriteSetPermutations { + import opened JSON.Values + import BoundedInts + import JSON.API + import FileIO + import opened StandardLibrary.String + import opened Permutations + import Base64 + + type Bytes = seq + type BytesList = seq + type StringList = seq + + function GetConfigs() : JSON + { + Object([("AllSign", + Object([("attributeActionsOnEncrypt", + Object([ + ("RecNum", String("SIGN_ONLY")), + ("StringSet", String("SIGN_ONLY")), + ("NumberSet", String("SIGN_ONLY")), + ("BinarySet", String("SIGN_ONLY")) + ]) + )])), + ("AllEncrypt", + Object([("attributeActionsOnEncrypt", + Object([ + ("RecNum", String("SIGN_ONLY")), + ("StringSet", String("ENCRYPT_AND_SIGN")), + ("NumberSet", String("ENCRYPT_AND_SIGN")), + ("BinarySet", String("ENCRYPT_AND_SIGN")) + ]) + )]) + )]) + } + + function {:tailrecursion} GetStringArray(str : StringList, acc : seq := []) : JSON + { + if |str| == 0 then + Array(acc) + else + GetStringArray(str[1..], acc + [String(str[0])]) + } + + function {:tailrecursion} EncodeStrings(bytes : BytesList, acc : seq := []) : seq + { + if |bytes| == 0 then + acc + else + EncodeStrings(bytes[1..], acc + [Base64.Encode(bytes[0])]) + } + + function GetBinaryArray(bytes : BytesList) : JSON + { + var strs := EncodeStrings(bytes); + GetStringArray(strs) + } + + + function {:opaque} GetRecord(recNum : int, str : StringList, num : StringList, bytes : BytesList) : JSON + { + var numStr := Base10Int2String(recNum); + Object([ + ("RecNum", Object([("N", String(numStr))])), + ("StringSet", Object([("SS", GetStringArray(str))])), + ("NumberSet", Object([("NS", GetStringArray(num))])), + ("BinarySet", Object([("BS", GetBinaryArray(bytes))])) + ]) + } + + + function {:opaque} {:tailrecursion} GetRecords2( + recNum : int, + str : seq, + num : seq, + bytes : seq, + acc : seq := []) : seq + decreases str + decreases num + decreases bytes + { + if |str| == 0 || |num| == 0 || |bytes| == 0 then + acc + else + var newRec := GetRecord(recNum, str[0], num[0], bytes[0]); + GetRecords2(recNum+1, str[1..], num[1..], bytes[1..], acc + [newRec]) + } + + method GetRecords() returns (result : seq) + { + var recs : seq := []; + var recs1 := [GetRecord(200, ["aaa"], ["111"], [[1,2,3]])]; + + var p2s := GeneratePermutations(["aaa", "bbb"]); + var p2n := GeneratePermutations(["111", "222"]); + var p2b := GeneratePermutations([[1,2,3], [2,3,4]]); + var recs2 := GetRecords2(201, p2s, p2n, p2b); + + var p3s := GeneratePermutations(["aaa", "bbb", "ccc"]); + var p3n := GeneratePermutations(["111", "222", "333"]); + var p3b := GeneratePermutations([[1,2,3], [2,3,4], [3,4,5]]); + var recs3 := GetRecords2(203, p3s, p3n, p3b); + + var p4s := GeneratePermutations(["aaa", "bbb", "ccc", "ddd"]); + var p4n := GeneratePermutations(["111", "222", "333", "444"]); + var p4b := GeneratePermutations([[1,2,3], [2,3,4], [3,4,5], [4,5,6]]); + var recs4 := GetRecords2(209, p4s, p4n, p4b); + + return recs1 + recs2 + recs3 + recs4; + } + + function BytesBv(bits: seq): seq + { + seq(|bits|, i requires 0 <= i < |bits| => bits[i] as bv8) + } + + method WriteSetPermutations() + { + var configs := GetConfigs(); + var records := GetRecords(); + var whole := Object([("RoundTripTest", Object([ + ("Records", Array(records)), + ("Configs", configs) + ]))]); + + var jsonBytes :- expect API.Serialize(whole); + var jsonBv := BytesBv(jsonBytes); + + var _ :- expect FileIO.WriteBytesToFile( + "PermTest.json", + jsonBv + ); + } +} diff --git a/TestVectors/runtimes/java/build.gradle.kts b/TestVectors/runtimes/java/build.gradle.kts index e26dfab41..154fb36f6 100644 --- a/TestVectors/runtimes/java/build.gradle.kts +++ b/TestVectors/runtimes/java/build.gradle.kts @@ -1,3 +1,6 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + import java.io.File import java.io.FileInputStream import java.util.Properties diff --git a/TestVectors/runtimes/java/data.json b/TestVectors/runtimes/java/data.json index 4bbc334ad..24fe5fc60 100644 --- a/TestVectors/runtimes/java/data.json +++ b/TestVectors/runtimes/java/data.json @@ -1,4 +1,251 @@ { + "RoundTripTest": { + "Configs": { + "AllSign": { + "attributeActionsOnEncrypt": { + "RecNum": "SIGN_ONLY", + "String": "SIGN_ONLY", + "Number": "SIGN_ONLY", + "Bytes": "SIGN_ONLY", + "StringSet": "SIGN_ONLY", + "NumberSet": "SIGN_ONLY", + "BinarySet": "SIGN_ONLY", + "Map": "SIGN_ONLY", + "List": "SIGN_ONLY", + "Null": "SIGN_ONLY", + "Bool": "SIGN_ONLY" + } + }, + "AllNothing": { + "attributeActionsOnEncrypt": { + "RecNum": "SIGN_ONLY", + "String": "DO_NOTHING", + "Number": "DO_NOTHING", + "Bytes": "DO_NOTHING", + "StringSet": "DO_NOTHING", + "NumberSet": "DO_NOTHING", + "BinarySet": "DO_NOTHING", + "Map": "DO_NOTHING", + "List": "DO_NOTHING", + "Null": "DO_NOTHING", + "Bool": "DO_NOTHING" + }, + "allowedUnsignedAttributes": [ + "String", + "Number", + "Bytes", + "StringSet", + "NumberSet", + "BinarySet", + "Map", + "List", + "Null", + "Bool" + ] + }, + "AllEncrypt": { + "attributeActionsOnEncrypt": { + "RecNum": "SIGN_ONLY", + "String": "ENCRYPT_AND_SIGN", + "Number": "ENCRYPT_AND_SIGN", + "Bytes": "ENCRYPT_AND_SIGN", + "StringSet": "ENCRYPT_AND_SIGN", + "NumberSet": "ENCRYPT_AND_SIGN", + "BinarySet": "ENCRYPT_AND_SIGN", + "Map": "ENCRYPT_AND_SIGN", + "List": "ENCRYPT_AND_SIGN", + "Null": "ENCRYPT_AND_SIGN", + "Bool": "ENCRYPT_AND_SIGN" + } + } + }, + "Records": [ + { + "RecNum": { + "N": "100" + }, + "StringSet": { + "SS": [ + "aaa", + "bbb" + ] + }, + "NumberSet": { + "NS": [ + "1.0", + "2.0" + ] + }, + "BinarySet": { + "BS": [ + "b25l", + "dHdv" + ] + } + }, + { + "RecNum": { + "N": "101" + }, + "StringSet": { + "SS": [ + "bbb", + "aaa" + ] + }, + "NumberSet": { + "NS": [ + "2.0", + "1.0" + ] + }, + "BinarySet": { + "BS": [ + "dHdv", + "b25l" + ] + }, + "Bool": { + "BOOL": true + } + }, + { + "RecNum": { + "N": "102" + }, + "StringSet": { + "SS": [ + "ddd", + "bbb", + "aaa", + "ccc", + "eee" + ] + }, + "NumberSet": { + "NS": [ + "2.0", + "1.0", + "-000.000", + "+123.456", + "123.456e5", + ".99999999999999999999999999999999999999E+126", + "1234567890123456789012345678901234567800000000000000000000000000000" + ] + }, + "BinarySet": { + "BS": [ + "dHdv", + "b25l" + ] + }, + "Bool": { + "BOOL": true + }, + "Map": { + "M": { + "eee": { + "SS": [ + "ddd", + "bbb", + "aaa", + "ccc", + "eee" + ] + }, + "aaa": { + "NS": [ + "2.0", + "1.0", + "-000.000", + "+123.456", + "123.456e5", + ".99999999999999999999999999999999999999E+126", + "1234567890123456789012345678901234567800000000000000000000000000000" + ] + }, + "ccc": { + "BS": [ + "dHdv", + "b25l" + ] + } + } + }, + "List": { + "L": [ + { + "SS": [ + "ddd", + "bbb", + "aaa", + "ccc", + "eee" + ] + }, + { + "NS": [ + "2.0", + "1.0", + "-000.000", + "+123.456", + "123.456e5", + ".99999999999999999999999999999999999999E+126", + "1234567890123456789012345678901234567800000000000000000000000000000" + ] + }, + { + "BS": [ + "dHdv", + "b25l" + ] + } + ] + } + }, + { + "RecNum": { + "N": "103" + }, + "StringSet": { + "SS": [ + "aaa" + ] + }, + "NumberSet": { + "NS": [ + "1.0" + ] + }, + "BinarySet": { + "BS": [ + "b25l" + ] + }, + "String": { + "S": "" + }, + "Number": { + "N": "0" + }, + "Bytes": { + "B": "" + }, + "Map": { + "M": {} + }, + "List": { + "L": [] + }, + "Null": { + "NULL": "" + }, + "Bool": { + "BOOL": false + } + } + ] + }, "IoPairs": [ [ "1", @@ -31,7 +278,9 @@ ":six": "Seis", ":seven": "Siete", ":eight": "Ocho", - ":NumberTest" : {"N" : "0800.000e0"}, + ":NumberTest": { + "N": "0800.000e0" + }, ":nine": "Nueve", ":cmp1a": "F_Cuatro.S_Junk", ":cmp1b": "F_444.S_Junk", @@ -195,13 +444,15 @@ { "Query": "cmp1c < Comp1", "Fail": [ - 0,1 + 0, + 1 ] }, { "Query": "cmp1c = Comp1", "Fail": [ - 0,1 + 0, + 1 ] }, { diff --git a/codebuild/release/release-prod.yml b/codebuild/release/release-prod.yml index 095f7e6ed..b8f2a3263 100644 --- a/codebuild/release/release-prod.yml +++ b/codebuild/release/release-prod.yml @@ -4,8 +4,6 @@ version: 0.2 env: - variables: - BRANCH: "main" parameter-store: ACCOUNT: /CodeBuild/AccountId secrets-manager: @@ -31,7 +29,6 @@ phases: - cd aws-database-encryption-sdk-dynamodb-java/ pre_build: commands: - - git checkout $BRANCH - aws secretsmanager get-secret-value --region us-west-2 --secret-id Maven-GPG-Keys-Release --query SecretBinary --output text | base64 -d > ~/mvn_gpg.tgz - tar -xvf ~/mvn_gpg.tgz -C ~ # Create default location where GPG looks for creds and keys diff --git a/codebuild/release/release.yml b/codebuild/release/release.yml index 9111a41cc..149b84661 100644 --- a/codebuild/release/release.yml +++ b/codebuild/release/release.yml @@ -28,7 +28,7 @@ batch: - identifier: validate_staging_corretto11 depend-on: - - release_staging + - validate_staging_corretto8 buildspec: codebuild/staging/validate-staging.yml env: variables: @@ -38,7 +38,7 @@ batch: - identifier: validate_staging_corretto17 depend-on: - - release_staging + - validate_staging_corretto11 buildspec: codebuild/staging/validate-staging.yml env: variables: @@ -73,7 +73,7 @@ batch: - identifier: validate_release_corretto11 depend-on: - - upload_to_sonatype + - validate_release_corretto8 buildspec: codebuild/release/validate-release.yml env: variables: @@ -83,7 +83,7 @@ batch: - identifier: validate_release_corretto17 depend-on: - - upload_to_sonatype + - validate_release_corretto11 buildspec: codebuild/release/validate-release.yml env: variables: diff --git a/project.properties b/project.properties index aa393fed9..80d93735b 100644 --- a/project.properties +++ b/project.properties @@ -1,4 +1,4 @@ -projectJavaVersion=3.1.0 -mplDependencyJavaVersion=1.0.0 +projectJavaVersion=3.1.2 +mplDependencyJavaVersion=1.0.1 dafnyRuntimeJavaVersion=4.2.0 smithyDafnyJavaConversionVersion=0.1 diff --git a/specification/dynamodb-encryption-client/ddb-attribute-serialization.md b/specification/dynamodb-encryption-client/ddb-attribute-serialization.md index 87c8b4307..a0be3ec80 100644 --- a/specification/dynamodb-encryption-client/ddb-attribute-serialization.md +++ b/specification/dynamodb-encryption-client/ddb-attribute-serialization.md @@ -60,9 +60,11 @@ String MUST be serialized as UTF-8 encoded bytes. #### Number -Number MUST be serialized as UTF-8 encoded bytes. Note that DynamoDB Number Attribute Values are strings. +This value MUST be normalized in the same way as DynamoDB normalizes numbers. +This normalized value MUST then be serialized as UTF-8 encoded bytes. + #### Binary Binary MUST be serialized with the identity function; @@ -107,8 +109,18 @@ Each of these entries MUST be serialized as: All [Set Entry Values](#set-entry-value) are the same type. Binary Sets MUST NOT contain duplicate entries. +Entries in a Binary Set MUST be ordered lexicographically by their underlying bytes in ascending order. + Number Sets MUST NOT contain duplicate entries. +Entries in a Number Set MUST be ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). +This ordering MUST be applied after normalization of the number value. +Note that because normalized number characters are all in the ASCII range (U+0000 to U+007F), +this ordering is equivalent to the [code point ordering](./string-ordering.md#code-point-order). + String Sets MUST NOT contain duplicate entries. +Entries in a String Set MUST be ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). +Note that though the entries are sorted by UTF016 binary order, +the values are serialized in the set with UTF-8 encoding. ###### Set Entry Length @@ -157,6 +169,11 @@ Each key-value pair MUST be serialized as: This sequence MUST NOT contain duplicate [Map Keys](#map-key). +Entries in a serialized Map MUST be ordered by key value, +ordered in ascending [UTF-16 binary order](./string-ordering.md#utf-16-binary-order). +Note that even though the values are sorted according to UTF-16 binary order, +string values are actually encoded within the map as UTF-8. + ###### Key Type Key Type MUST be the [Type ID](#type-id) for Strings. diff --git a/specification/dynamodb-encryption-client/ddb-encryption-branch-key-id-supplier.md b/specification/dynamodb-encryption-client/ddb-encryption-branch-key-id-supplier.md index 43462aa33..f9a48124e 100644 --- a/specification/dynamodb-encryption-client/ddb-encryption-branch-key-id-supplier.md +++ b/specification/dynamodb-encryption-client/ddb-encryption-branch-key-id-supplier.md @@ -1,3 +1,6 @@ +[//]: # "Copyright Amazon.com Inc. or its affiliates. All Rights Reserved." +[//]: # "SPDX-License-Identifier: CC-BY-SA-4.0" + # DynamoDb Encryption Branch Key Supplier ## Overview diff --git a/specification/dynamodb-encryption-client/ddb-item-conversion.md b/specification/dynamodb-encryption-client/ddb-item-conversion.md index 76168263b..644624d58 100644 --- a/specification/dynamodb-encryption-client/ddb-item-conversion.md +++ b/specification/dynamodb-encryption-client/ddb-item-conversion.md @@ -26,10 +26,12 @@ This document describes how a DynamoDB Item is converted to the Structured Encryption Library's [Structured Data](../structured-encryption/structures.md#structured-data), and vice versa. -The conversion from DDB Item to Structured Data must be lossless, -meaning that converting a DDB Item to -a Structured Data and back to a DDB Item again -MUST result in the exact same DDB Item. +Round Trip conversion between DDB Item and Structured Data is technically lossless, but it is not identity. +The conversion normalizes some values, the same way that +DynamoDB PuItem followed by GetItem normalizes some values. +The sets still have the same members, and the numbers still have the same values, +but the members of the set might appear in a different order, +and the numeric value might be formatted differently. ## Convert DDB Item to Structured Data @@ -73,4 +75,4 @@ has the following requirements: - Conversion from a Structured Data Map MUST fail if it has duplicate keys - Conversion from a Structured Data Number Set MUST fail if it has duplicate values - Conversion from a Structured Data String Set MUST fail if it has duplicate values -- Conversion from a Structured Data Binary Set MUST fail if it has duplicate values \ No newline at end of file +- Conversion from a Structured Data Binary Set MUST fail if it has duplicate values diff --git a/specification/dynamodb-encryption-client/ddb-sdk-integration.md b/specification/dynamodb-encryption-client/ddb-sdk-integration.md index 0cb2716fb..3f48a692a 100644 --- a/specification/dynamodb-encryption-client/ddb-sdk-integration.md +++ b/specification/dynamodb-encryption-client/ddb-sdk-integration.md @@ -124,6 +124,9 @@ MUST have the following modified behavior: - [Encrypt before BatchWriteItem](#encrypt-before-batchwriteitem) - [Encrypt before TransactWriteItems](#encrypt-before-transactwriteitems) - [Decrypt after GetItem](#decrypt-after-getitem) +- [Decrypt after PutItem](#decrypt-after-putitem) +- [Decrypt after UpdateItem](#decrypt-after-updateitem) +- [Decrypt after DeleteItem](#decrypt-after-deleteitem) - [Decrypt after BatchGetItem](#decrypt-after-batchgetitem) - [Decrypt after Scan](#decrypt-after-scan) - [Decrypt after Query](#decrypt-after-query) @@ -334,6 +337,92 @@ Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. The GetItem response's `Item` field MUST be replaced by the encrypted DynamoDb Item outputted above. +### Decrypt after PutItem + +After a [PutItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html) +call is made to DynamoDB, +the resulting response MUST be modified before +being returned to the caller if: +- there exists an Item Encryptor specified within the + [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration) + with a [DynamoDB Table Name](./ddb-item-encryptor.md#dynamodb-table-name) + equal to the `TableName` on the PutItem request. +- the response contains [Attributes](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-response-Attributes). + The response will contain Attributes if the related PutItem request's + [ReturnValues](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_PutItem.html#DDB-PutItem-request-ReturnValues) + had a value of `ALL_OLD` and the PutItem call replaced a pre-existing item. + +In this case, the [Item Encryptor](./ddb-item-encryptor.md) MUST perform +[Decrypt Item](./decrypt-item.md) where the input +[DynamoDB Item](./decrypt-item.md#dynamodb-item) +is the `Attributes` field in the original response + +Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. + +The PutItem response's `Attributes` field MUST be +replaced by the encrypted DynamoDb Item outputted above. + +### Decrypt after DeleteItem + +After a [DeleteItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DeleteItem.html) +call is made to DynamoDB, +the resulting response MUST be modified before +being returned to the caller if: +- there exists an Item Encryptor specified within the + [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration) + with a [DynamoDB Table Name](./ddb-item-encryptor.md#dynamodb-table-name) + equal to the `TableName` on the DeleteItem request. +- the response contains [Attributes](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DeleteItem.html#DDB-DeleteItem-response-Attributes). + The response will contain Attributes if the related DeleteItem request's + [ReturnValues](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DeleteItem.html#DDB-DeleteItem-request-ReturnValues) + had a value of `ALL_OLD` and an item was deleted. + +In this case, the [Item Encryptor](./ddb-item-encryptor.md) MUST perform +[Decrypt Item](./decrypt-item.md) where the input +[DynamoDB Item](./decrypt-item.md#dynamodb-item) +is the `Attributes` field in the original response + +Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. + +The DeleteItem response's `Attributes` field MUST be +replaced by the encrypted DynamoDb Item outputted above. + +### Decrypt after UpdateItem + +After a [UpdateItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html) +call is made to DynamoDB, +the resulting response MUST be modified before +being returned to the caller if: +- there exists an Item Encryptor specified within the + [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration) + with a [DynamoDB Table Name](./ddb-item-encryptor.md#dynamodb-table-name) + equal to the `TableName` on the UpdateItem request. +- the response contains [Attributes](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html#DDB-UpdateItem-response-Attributes). +- the original UpdateItem request had a + [ReturnValues](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html#DDB-UpdateItem-request-ReturnValues) + with a value of `ALL_OLD` or `ALL_NEW`. + +In this case, the [Item Encryptor](./ddb-item-encryptor.md) MUST perform +[Decrypt Item](./decrypt-item.md) where the input +[DynamoDB Item](./decrypt-item.md#dynamodb-item) +is the `Attributes` field in the original response + +Beacons MUST be [removed](ddb-support.md#removebeacons) from the result. + +The UpdateItem response's `Attributes` field MUST be +replaced by the encrypted DynamoDb Item outputted above. + +In all other cases, the UpdateItem response MUST NOT be modified. + +Additionally, if a value of `UPDATED_OLD` or `UPDATED_NEW` was used, +and any Attributes in the response are authenticated +per the [DynamoDB Encryption Client Config](#dynamodb-encryption-client-configuration), +an error MUST be raised. +Given that we [validate UpdateItem requests](#validate-before-updateitem), +and thus updates will not modify any signed field, +an error here would indicate a bug in +our library or a bug within DynamoDB. + ### Decrypt after BatchGetItem After a [BatchGetItem](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchGetItem.html) diff --git a/specification/dynamodb-encryption-client/ddb-support.md b/specification/dynamodb-encryption-client/ddb-support.md index 7697759e7..b276942f4 100644 --- a/specification/dynamodb-encryption-client/ddb-support.md +++ b/specification/dynamodb-encryption-client/ddb-support.md @@ -1,3 +1,6 @@ +[//]: # "Copyright Amazon.com Inc. or its affiliates. All Rights Reserved." +[//]: # "SPDX-License-Identifier: CC-BY-SA-4.0" + # DynamoDB Support Layer The DynamoDB Support Layer provides everything necessary to the middleware interceptors, diff --git a/specification/dynamodb-encryption-client/string-ordering.md b/specification/dynamodb-encryption-client/string-ordering.md new file mode 100644 index 000000000..7b2cb8bc8 --- /dev/null +++ b/specification/dynamodb-encryption-client/string-ordering.md @@ -0,0 +1,103 @@ +[//]: # "Copyright Amazon.com Inc. or its affiliates. All Rights Reserved." +[//]: # "SPDX-License-Identifier: CC-BY-SA-4.0" + +# String Ordering + +## Version + +1.0.0 + +### Changelog + +- 1.0.0 + + - Initial record + +## Definitions + +### Conventions used in this document + +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" +in this document are to be interpreted as described in [RFC 2119](https://tools.ietf.org/html/rfc2119). + +### Unicode + +For the latest version see: +https://www.unicode.org/versions/latest/ + +For Version 15.0 see: +https://www.unicode.org/versions/Unicode15.0.0/ + +### Unicode scalar value + +Any Unicode code point except [surrogate](#surrogates) code points. + +See [section 3.9 Unicode Encoding Forms](https://www.unicode.org/versions/Unicode15.0.0/ch03.pdf) of the Unicode specification. + +### UTF-16 code unit + +A 16-bit value representing a Unicode code point in a UTF-16 encoding. +Includes [surrogate](#surrogates) code points. + +See [section 3.9 Unicode Encoding Forms](https://www.unicode.org/versions/Unicode15.0.0/ch03.pdf) of the Unicode specification. + +### Surrogates + +Unicode code points in the range U+D800 to U+DFFF. +These code points are only used in UTF-16 encodings +to represent Unicode values above U+FFFF. + +See [section 3.8 Surrogates](https://www.unicode.org/versions/Unicode15.0.0/ch03.pdf) of the Unicode specification. + +## Overview + +There are several instances throughout this specification where an order must be +imposed on an unordered data structure during serialization for the purposes of canonicalization. +This means that we must clearly specify the canonical ordering wherever +such serialization is required. +This is especially importing when ordering strings, +as different encodings may lend themselves to slightly different orderings. + +Wherever strings need to be ordered, +this specification will require either a [code point order](#code-point-order) +or a [UTF-16 binary order](#utf-16-binary-order). + +## UTF-16 Binary Order + +When ordering strings, +these strings MUST be compared according to their UTF-16 encoding, +lexicographically per [UTF-16 code unit](#utf-16-code-unit). +UTF-16 code units for [high or low surrogates](#surrogates) MUST be compared individually, +and the [Unicode scalar value](#unicode-scalar-value) represented by a surrogate pair +MUST NOT be compared. + +Note that this is not equivalent to the [code point order](#code-point-order). +Specifically, the range of characters with Unicode code point U+E000 to U+0xFFFF +(code points representable by 16 bits, but after the surrogate range) +MUST be considered "greater than" any character with a Unicode code point of U+10000 to U+10FFFF. + +As an example, consider the following two characters: + +| char | Unicode code point | UTF-16 encoding | +| ---- | ------------------ | --------------- | +| `｡` | U+FF61 | 0xFF61 | +| `𐀂` | U+10002 | 0xD800 0xDC02 | + +This ordering will order `｡` _after_ `𐀂`, despite `𐀂` having a higher Unicode code point. + +## Code Point Order + +This is the ordering referred to in the Unicode specification as a [code point order](https://www.unicode.org/versions/Unicode15.0.0/ch05.pdf). + +When ordering strings, +these strings are compared lexicographically per [Unicode scalar value](#unicode-scalar-value) represented by the string. +This means that if a string is UTF-16 encoded, +higher order Unicode characters, encoded as a surrogate pair, +must be handled as the Unicode scalar value represented by that surrogate pair, +instead of each surrogate code point being handled individually. + +Note that this is equivalent to lexicographically comparing a UTF-8 encoded string per byte. +This is also equivalent to lexicographically comparing a UTF-32 encoded string per 32-bit code unit. + +Currently, this specification does not directly use code point order for sorting string values, +but may use this ordering for new behaviors in the future. diff --git a/specification/structured-encryption/footer.md b/specification/structured-encryption/footer.md index 560259c42..b3dfc0444 100644 --- a/specification/structured-encryption/footer.md +++ b/specification/structured-encryption/footer.md @@ -1,3 +1,5 @@ +[//]: # "Copyright Amazon.com Inc. or its affiliates. All Rights Reserved." +[//]: # "SPDX-License-Identifier: CC-BY-SA-4.0" # Structured Encryption Footer diff --git a/specification/structured-encryption/header.md b/specification/structured-encryption/header.md index 5d8ec6212..ec0593ff5 100644 --- a/specification/structured-encryption/header.md +++ b/specification/structured-encryption/header.md @@ -1,3 +1,5 @@ +[//]: # "Copyright Amazon.com Inc. or its affiliates. All Rights Reserved." +[//]: # "SPDX-License-Identifier: CC-BY-SA-4.0" # Structured Encryption Header