From 186d507ff8f26f42e08694afb4a86cceec91b17a Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Sun, 16 Jul 2023 11:14:31 -0700 Subject: [PATCH 1/3] fix(EnhancedClient): Correctly Identify Index attributes Table Schema's store Index information in `indices`, NOT `keyAttributes`. `keyAttributes` includes Index information, but also includes other information. --- .../DynamoDbEnhancedClientEncryption.java | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/DynamoDbEncryption/runtimes/java/src/main/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEnhancedClientEncryption.java b/DynamoDbEncryption/runtimes/java/src/main/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEnhancedClientEncryption.java index 5553a6b7d..c9ecd4f1b 100644 --- a/DynamoDbEncryption/runtimes/java/src/main/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEnhancedClientEncryption.java +++ b/DynamoDbEncryption/runtimes/java/src/main/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEnhancedClientEncryption.java @@ -1,14 +1,18 @@ package software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient; +import java.util.*; +import java.util.stream.Collectors; + +import software.amazon.awssdk.enhanced.dynamodb.IndexMetadata; +import software.amazon.awssdk.enhanced.dynamodb.KeyAttributeMetadata; +import software.amazon.awssdk.enhanced.dynamodb.TableMetadata; + import software.amazon.cryptography.dbencryptionsdk.dynamodb.model.DynamoDbTablesEncryptionConfig; import software.amazon.cryptography.dbencryptionsdk.dynamodb.model.DynamoDbEncryptionException; import software.amazon.cryptography.dbencryptionsdk.dynamodb.model.DynamoDbTableEncryptionConfig; import software.amazon.cryptography.dbencryptionsdk.structuredencryption.model.CryptoAction; import software.amazon.cryptography.dbencryptionsdk.dynamodb.DynamoDbEncryptionInterceptor; -import java.util.*; -import java.util.stream.Collectors; - import static software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.DoNothingTag.CUSTOM_DDB_ENCRYPTION_DO_NOTHING_PREFIX; import static software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.SignOnlyTag.CUSTOM_DDB_ENCRYPTION_SIGN_ONLY_PREFIX; @@ -27,12 +31,33 @@ public static DynamoDbEncryptionInterceptor CreateDynamoDbEncryptionInterceptor( .build(); } + private static Set attributeNamesUsedInIndices( + final TableMetadata tableMetadata + ) { + Set partitionAttributeNames = tableMetadata.indices().stream() + .map(IndexMetadata::partitionKey) + .filter(Optional::isPresent) + .map(Optional::get) + .map(KeyAttributeMetadata::name) + .collect(Collectors.toSet()); + Set sortAttributeNames = tableMetadata.indices().stream() + .map(IndexMetadata::sortKey) + .filter(Optional::isPresent) + .map(Optional::get) + .map(KeyAttributeMetadata::name) + .collect(Collectors.toSet()); + Set allIndexAttributes = new HashSet<>(); + allIndexAttributes.addAll(partitionAttributeNames); + allIndexAttributes.addAll(sortAttributeNames); + return allIndexAttributes; + } + private static DynamoDbTableEncryptionConfig getTableConfig(DynamoDbEnhancedTableEncryptionConfig configWithSchema) { Map actions = new HashMap<>(); Set signOnlyAttributes = configWithSchema.schemaOnEncrypt().tableMetadata().customMetadataObject(CUSTOM_DDB_ENCRYPTION_SIGN_ONLY_PREFIX, Set.class).orElseGet(HashSet::new); Set doNothingAttributes = configWithSchema.schemaOnEncrypt().tableMetadata().customMetadataObject(CUSTOM_DDB_ENCRYPTION_DO_NOTHING_PREFIX, Set.class).orElseGet(HashSet::new); - Set keyAttributes = configWithSchema.schemaOnEncrypt().tableMetadata().keyAttributes().stream().map(val -> val.name()).collect(Collectors.toSet()); + Set keyAttributes = attributeNamesUsedInIndices(configWithSchema.schemaOnEncrypt().tableMetadata()); if (!Collections.disjoint(keyAttributes, doNothingAttributes)) { throw DynamoDbEncryptionException.builder() From be804515b16aff8e65bf4120f200833b62993d8d Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Sun, 16 Jul 2023 12:28:26 -0700 Subject: [PATCH 2/3] test(enhancedClient): Verify EC's null behavior does not interfere --- .../runtimes/java/build.gradle.kts | 3 + ...tionEnhancedClientNullIntegrationTest.java | 167 ++++++++++++++++++ .../datamodels/SimpleViaLombok.java | 37 ++++ 3 files changed, 207 insertions(+) create mode 100644 DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientNullIntegrationTest.java create mode 100644 DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/datamodels/SimpleViaLombok.java diff --git a/DynamoDbEncryption/runtimes/java/build.gradle.kts b/DynamoDbEncryption/runtimes/java/build.gradle.kts index 327e78bb5..10761ee41 100644 --- a/DynamoDbEncryption/runtimes/java/build.gradle.kts +++ b/DynamoDbEncryption/runtimes/java/build.gradle.kts @@ -105,6 +105,9 @@ dependencies { testImplementation("junit:junit:4.13.2") // https://mvnrepository.com/artifact/edu.umd.cs.mtc/multithreadedtc testImplementation("edu.umd.cs.mtc:multithreadedtc:1.01") + // https://mvnrepository.com/artifact/org.projectlombok/lombok + testImplementation("org.projectlombok:lombok:1.18.28") + testAnnotationProcessor("org.projectlombok:lombok:1.18.28") } publishing { diff --git a/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientNullIntegrationTest.java b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientNullIntegrationTest.java new file mode 100644 index 000000000..fa9433bd7 --- /dev/null +++ b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientNullIntegrationTest.java @@ -0,0 +1,167 @@ +package software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient; + +import org.testng.annotations.Test; + +import java.time.Clock; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.enhanced.dynamodb.DynamoDbEnhancedClient; +import software.amazon.awssdk.enhanced.dynamodb.DynamoDbTable; +import software.amazon.awssdk.enhanced.dynamodb.Expression; +import software.amazon.awssdk.enhanced.dynamodb.Key; +import software.amazon.awssdk.enhanced.dynamodb.TableSchema; +import software.amazon.awssdk.enhanced.dynamodb.extensions.AutoGeneratedTimestampRecordExtension; +import software.amazon.awssdk.enhanced.dynamodb.internal.converter.attribute.InstantAsStringAttributeConverter; +import software.amazon.awssdk.enhanced.dynamodb.model.GetItemEnhancedRequest; +import software.amazon.awssdk.enhanced.dynamodb.model.PutItemEnhancedRequest; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + +import software.amazon.cryptography.dbencryptionsdk.dynamodb.DynamoDbEncryptionInterceptor; +import software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.datamodels.SimpleViaLombok; +import software.amazon.cryptography.materialproviders.IKeyring; + +import static java.lang.Math.abs; +import static software.amazon.cryptography.dbencryptionsdk.dynamodb.TestUtils.TEST_TABLE_NAME; +import static software.amazon.cryptography.dbencryptionsdk.dynamodb.TestUtils.createKmsKeyring; + +/** + * From the DynamoDB Enhanced Client's Developer Guide: + * > When you use the putItem API, + * > the enhanced client does not include null-valued attributes of + * > a mapped data object in the request to DynamoDB. + * This test asserts the Enhanced Client's Null behavior does not cause an issue. + * It SHOULD NOT. + * The DB-ESDK for DynamoDB prevents Signed Attributes + * from being used in an update expression. + * As such, all writes with a Signed Attribute are only permitted + * with a full write. + * Even though the Enhanced Client does not include the null values + * in the request to DynamoDB, + * since the request is made with a full WRITE, + * the pre-existing NOW null values will be dropped. + */ +public class DynamoDbEncryptionEnhancedClientNullIntegrationTest { + final static String PARTITION = "TestEnhancedClientPutNull"; + final static int SORT = 20230716; + final static String COULD_NOT_READ_MSG = String.format( + "Testing Null Write with Enhanced Client Required a particular Read that could not complete. " + + "Recommend Deleting the record via the console and retrying. " + + "Record's Key is Partition: %s, Sort: %s", PARTITION, SORT); + public static final int TOLERANCE = 1000; + + @Test + public void TestEnhancedClientPutNull() { + final String UNSIGNED_PREFIX = ":"; + final IKeyring kmsKeyring = createKmsKeyring(); + final TableSchema schemaOnEncrypt = + TableSchema.fromImmutableClass(SimpleViaLombok.class); + final InstantAsStringAttributeConverter instantConverter = + InstantAsStringAttributeConverter.create(); + final Key key = Key.builder() + .partitionValue(PARTITION).sortValue(SORT) + .build(); + final Clock clock = Clock.systemUTC(); + + DynamoDbClient ddb = initDDBWithEncryption(UNSIGNED_PREFIX, kmsKeyring, schemaOnEncrypt); + DynamoDbEnhancedClient enhancedClient = DynamoDbEnhancedClient.builder() + .extensions(AutoGeneratedTimestampRecordExtension.create()) + .dynamoDbClient(ddb) + .build(); + DynamoDbTable table = enhancedClient.table(TEST_TABLE_NAME, schemaOnEncrypt); + + final SimpleViaLombok.SimpleViaLombokBuilder itemBuilder = SimpleViaLombok.builder(); + itemBuilder.partitionKey(PARTITION).sortKey(SORT); + itemBuilder.attribute1("In the Jungle"); + itemBuilder.attribute2("The mighty Jungle"); + itemBuilder.attribute3("The Lion Sleeps Tonight"); + SimpleViaLombok item = itemBuilder.build(); + Instant firstWriteTime = clock.instant(); + table.putItem(item); + + // Get the item back from the table + SimpleViaLombok firstWriteDecrypted = getItemWithTimeStamp( + firstWriteTime, table, key, TOLERANCE); + + // We are going to Modify the Values to be Null, + // Write that to DDB with Encryption + final SimpleViaLombok.SimpleViaLombokBuilder nullBuilder = SimpleViaLombok.builder(); + nullBuilder.partitionKey(PARTITION).sortKey(SORT); + nullBuilder.attribute1(null).attribute2(null).attribute3(null); + SimpleViaLombok nullItem = nullBuilder.build(); + final Instant secondWriteTime = clock.instant(); + // We need to be sure that we are replacing This Test Runs' Write, + // so we will use a condition expression + table.putItem( + (PutItemEnhancedRequest.Builder requestBuilder) -> + requestBuilder.conditionExpression( + Expression.builder().expression("#a = :b") + .putExpressionName("#a", ":lastModifiedTimeStamp") + .putExpressionValue(":b", + instantConverter.transformFrom(firstWriteDecrypted.getLastModifiedTimeStamp())) + .build()) + .item(nullItem) + ); + + SimpleViaLombok secondWriteDecrypted = getItemWithTimeStamp(secondWriteTime, table, key, TOLERANCE); + assert Objects.isNull(secondWriteDecrypted.getAttribute1()); + assert Objects.isNull(secondWriteDecrypted.getAttribute2()); + assert Objects.isNull(secondWriteDecrypted.getAttribute3()); + } + + private static SimpleViaLombok getItemWithTimeStamp( + final Instant writeTime, + final DynamoDbTable table, + final Key key, + final int tolerance + ) { + int retries = 3; + final GetItemEnhancedRequest request = + GetItemEnhancedRequest.builder() + .consistentRead(true).key(key).build(); + try { + while (retries != 0) { + SimpleViaLombok read = table.getItem(request); + Instant readTime = read.getLastModifiedTimeStamp(); + long diff = writeTime.toEpochMilli() - readTime.toEpochMilli(); + if (abs(diff) < tolerance) return read; + // We did not get the record we want, sleep for 1 Seconds + Thread.sleep(TOLERANCE); + retries = retries - 1; + } + } catch (InterruptedException ex) { + throw new RuntimeException(COULD_NOT_READ_MSG, ex); + } + throw new RuntimeException(COULD_NOT_READ_MSG); + } + + private static DynamoDbClient initDDBWithEncryption( + final String unsignedPrefix, + final IKeyring kmsKeyring, + TableSchema schemaOnEncrypt + ) { + Map tableConfigs = new HashMap<>(); + tableConfigs.put(TEST_TABLE_NAME, + DynamoDbEnhancedTableEncryptionConfig.builder() + .logicalTableName(TEST_TABLE_NAME) + .keyring(kmsKeyring) + .allowedUnsignedAttributePrefix(unsignedPrefix) + .schemaOnEncrypt(schemaOnEncrypt) + .build()); + DynamoDbEncryptionInterceptor interceptor = + DynamoDbEnhancedClientEncryption.CreateDynamoDbEncryptionInterceptor( + CreateDynamoDbEncryptionInterceptorInput.builder() + .tableEncryptionConfigs(tableConfigs) + .build() + ); + return DynamoDbClient.builder() + .overrideConfiguration( + ClientOverrideConfiguration.builder() + .addExecutionInterceptor(interceptor) + .build()) + .build(); + } +} diff --git a/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/datamodels/SimpleViaLombok.java b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/datamodels/SimpleViaLombok.java new file mode 100644 index 000000000..60eff6222 --- /dev/null +++ b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/datamodels/SimpleViaLombok.java @@ -0,0 +1,37 @@ +package software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.datamodels; + +import java.time.Instant; + +import lombok.Builder; +import lombok.Getter; +import lombok.Value; + +import software.amazon.awssdk.enhanced.dynamodb.extensions.annotations.DynamoDbAutoGeneratedTimestampAttribute; +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbAttribute; +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbImmutable; +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbPartitionKey; +import software.amazon.awssdk.enhanced.dynamodb.mapper.annotations.DynamoDbSortKey; + +import software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.DynamoDbEncryptionDoNothing; +import software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.DynamoDbEncryptionSignOnly; + +@Value +@Builder +@DynamoDbImmutable(builder = SimpleViaLombok.SimpleViaLombokBuilder.class) +public class SimpleViaLombok { + @Getter(onMethod_= {@DynamoDbPartitionKey, @DynamoDbAttribute(value = "partition_key")}) + String partitionKey; + @Getter(onMethod_= {@DynamoDbSortKey, @DynamoDbAttribute(value = "sort_key")}) + int sortKey; + String attribute1; + @Getter(onMethod_=@DynamoDbEncryptionSignOnly) + String attribute2; + @Getter(onMethod_= {@DynamoDbEncryptionDoNothing, @DynamoDbAttribute(value = ":attribute3")}) + String attribute3; + @Getter(onMethod_= { + @DynamoDbEncryptionDoNothing, + @DynamoDbAttribute(value = ":lastModifiedTimeStamp"), + @DynamoDbAutoGeneratedTimestampAttribute + }) + Instant lastModifiedTimeStamp; +} From 6cfa8577585dfa1702ae8b9478808c10740e7b61 Mon Sep 17 00:00:00 2001 From: texastony <5892063+texastony@users.noreply.github.com> Date: Tue, 18 Jul 2023 12:21:17 -0700 Subject: [PATCH 3/3] refactor(EnhancedClient-Tests): Move Data Models --- .../DynamoDbEncryptionEnhancedClientNullIntegrationTest.java | 2 +- .../{datamodels => validdatamodels}/SimpleViaLombok.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/{datamodels => validdatamodels}/SimpleViaLombok.java (98%) diff --git a/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientNullIntegrationTest.java b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientNullIntegrationTest.java index fa9433bd7..478911926 100644 --- a/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientNullIntegrationTest.java +++ b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/DynamoDbEncryptionEnhancedClientNullIntegrationTest.java @@ -21,7 +21,7 @@ import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.cryptography.dbencryptionsdk.dynamodb.DynamoDbEncryptionInterceptor; -import software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.datamodels.SimpleViaLombok; +import software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.validdatamodels.SimpleViaLombok; import software.amazon.cryptography.materialproviders.IKeyring; import static java.lang.Math.abs; diff --git a/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/datamodels/SimpleViaLombok.java b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/validdatamodels/SimpleViaLombok.java similarity index 98% rename from DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/datamodels/SimpleViaLombok.java rename to DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/validdatamodels/SimpleViaLombok.java index 60eff6222..41a69f435 100644 --- a/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/datamodels/SimpleViaLombok.java +++ b/DynamoDbEncryption/runtimes/java/src/test/java/software/amazon/cryptography/dbencryptionsdk/dynamodb/enhancedclient/validdatamodels/SimpleViaLombok.java @@ -1,4 +1,4 @@ -package software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.datamodels; +package software.amazon.cryptography.dbencryptionsdk.dynamodb.enhancedclient.validdatamodels; import java.time.Instant;