Merge feature/master/codegen-validation to master

.changes/next-release/feature-AWSSDKforJavav2-7cf1e5c.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "Add code generation validation for missing request URI on an operation."
+}

.changes/next-release/feature-AWSSDKforJavav2-bd762da.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "Add support for defining service model validators and generating valdiation reports during code generation."
+}

.changes/next-release/feature-AWSSDKforJavav2-f004fae.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "Add support for validating that shared models between two services are identical."
+}

.changes/next-release/feature-CodeGeneratorMavenPlugin-64c91e8.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "Code Generator Maven Plugin",
+    "contributor": "",
+    "description": "Update the generator plugin to support model validation during code generation. In addition, this adds the `writeValidationReport` flag to support writing the validation report to disk."
+}

codegen-maven-plugin/pom.xml

@@ -57,6 +57,11 @@
             <groupId>software.amazon.awssdk</groupId>
             <version>${awsjavasdk.version}</version>
         </dependency>
+        <dependency>
+            <artifactId>utils</artifactId>
+            <groupId>software.amazon.awssdk</groupId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>

codegen-maven-plugin/src/main/java/software/amazon/awssdk/codegen/maven/plugin/GenerationMojo.java

@@ -17,11 +17,17 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.apache.maven.plugin.AbstractMojo;
 import org.apache.maven.plugin.MojoExecutionException;
@@ -30,21 +36,26 @@
 import org.apache.maven.project.MavenProject;
 import software.amazon.awssdk.codegen.C2jModels;
 import software.amazon.awssdk.codegen.CodeGenerator;
+import software.amazon.awssdk.codegen.IntermediateModelBuilder;
+import software.amazon.awssdk.codegen.internal.Jackson;
 import software.amazon.awssdk.codegen.internal.Utils;
 import software.amazon.awssdk.codegen.model.config.customization.CustomizationConfig;
+import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
 import software.amazon.awssdk.codegen.model.rules.endpoints.EndpointTestSuiteModel;
 import software.amazon.awssdk.codegen.model.service.EndpointRuleSetModel;
 import software.amazon.awssdk.codegen.model.service.Paginators;
 import software.amazon.awssdk.codegen.model.service.ServiceModel;
 import software.amazon.awssdk.codegen.model.service.Waiters;
 import software.amazon.awssdk.codegen.utils.ModelLoaderUtils;
+import software.amazon.awssdk.codegen.validation.ModelInvalidException;
+import software.amazon.awssdk.codegen.validation.ModelValidationReport;
+import software.amazon.awssdk.utils.StringUtils;
 
 /**
  * The Maven mojo to generate Java client code using software.amazon.awssdk:codegen module.
  */
 @Mojo(name = "generate")
 public class GenerationMojo extends AbstractMojo {
-
     private static final String MODEL_FILE = "service-2.json";
     private static final String CUSTOMIZATION_CONFIG_FILE = "customization.config";
     private static final String WAITERS_FILE = "waiters-2.json";
@@ -62,6 +73,8 @@ public class GenerationMojo extends AbstractMojo {
     @Parameter(property = "writeIntermediateModel", defaultValue = "false")
     private boolean writeIntermediateModel;
 
+    @Parameter(property = "writeValidationReport", defaultValue = "false")
+    private boolean writeValidationReport;
 
     @Parameter(defaultValue = "${project}", readonly = true)
     private MavenProject project;
@@ -76,22 +89,72 @@ public void execute() throws MojoExecutionException {
         this.resourcesDirectory = Paths.get(outputDirectory).resolve("generated-resources").resolve("sdk-resources");
         this.testsDirectory = Paths.get(outputDirectory).resolve("generated-test-sources").resolve("sdk-tests");
 
-        findModelRoots().forEach(p -> {
-            Path modelRootPath = p.modelRoot;
-            getLog().info("Loading from: " + modelRootPath.toString());
-            generateCode(C2jModels.builder()
-                                  .customizationConfig(p.customizationConfig)
-                                  .serviceModel(loadServiceModel(modelRootPath))
-                                  .waitersModel(loadWaiterModel(modelRootPath))
-                                  .paginatorsModel(loadPaginatorModel(modelRootPath))
-                                  .endpointRuleSetModel(loadEndpointRuleSetModel(modelRootPath))
-                                  .endpointTestSuiteModel(loadEndpointTestSuiteModel(modelRootPath))
-                                  .build());
+        List<GenerationParams> generationParams;
+
+        try {
+            generationParams = initGenerationParams();
+        } catch (ModelInvalidException e) {
+            if (writeValidationReport) {
+                ModelValidationReport report = new ModelValidationReport();
+                report.setValidationEntries(e.validationEntries());
+                emitValidationReport(report);
+            }
+            throw e;
+        }
+
+        Map<String, IntermediateModel> serviceNameToModelMap = new HashMap<>();
+
+        generationParams.forEach(
+            params -> {
+                IntermediateModel model = params.intermediateModel;
+                String lowercaseServiceName = StringUtils.lowerCase(model.getMetadata().getServiceName());
+                IntermediateModel previous = serviceNameToModelMap.put(lowercaseServiceName, model);
+                if (previous != null) {
+                    String warning = String.format("Multiple service models found with service name %s. Model validation "
+                                                   + "will likely be incorrect", lowercaseServiceName);
+                    getLog().warn(warning);
+                }
+            });
+
+        // Update each param with the intermediate model it shares models with, if any
+        generationParams.forEach(params -> {
+            CustomizationConfig customizationConfig = params.intermediateModel.getCustomizationConfig();
+
+            if (customizationConfig.getShareModelConfig() != null) {
+                String shareModelWithName = customizationConfig.getShareModelConfig().getShareModelWith();
+                params.withShareModelsTarget(serviceNameToModelMap.get(shareModelWithName));
+            }
         });
+
+        generationParams.forEach(this::generateCode);
+
         project.addCompileSourceRoot(sourcesDirectory.toFile().getAbsolutePath());
         project.addTestCompileSourceRoot(testsDirectory.toFile().getAbsolutePath());
     }
 
+    private List<GenerationParams> initGenerationParams() throws MojoExecutionException {
+        List<ModelRoot> modelRoots = findModelRoots().collect(Collectors.toList());
+
+        return modelRoots.stream().map(r -> {
+            Path modelRootPath = r.modelRoot;
+            getLog().info("Loading from: " + modelRootPath.toString());
+            C2jModels c2jModels = C2jModels.builder()
+                                           .customizationConfig(r.customizationConfig)
+                                           .serviceModel(loadServiceModel(modelRootPath))
+                                           .waitersModel(loadWaiterModel(modelRootPath))
+                                           .paginatorsModel(loadPaginatorModel(modelRootPath))
+                                           .endpointRuleSetModel(loadEndpointRuleSetModel(modelRootPath))
+                                           .endpointTestSuiteModel(loadEndpointTestSuiteModel(modelRootPath))
+                                           .build();
+            String intermediateModelFileNamePrefix = intermediateModelFileNamePrefix(c2jModels);
+            IntermediateModel intermediateModel = new IntermediateModelBuilder(c2jModels).build();
+            return new GenerationParams().withIntermediateModel(intermediateModel)
+                                         .withIntermediateModelFileNamePrefix(intermediateModelFileNamePrefix);
+        }).collect(Collectors.toList());
+    }
+
+
+
     private Stream<ModelRoot> findModelRoots() throws MojoExecutionException {
         try {
             return Files.find(codeGenResources.toPath(), 10, this::isModelFile)
@@ -111,13 +174,15 @@ private boolean isModelFile(Path p, BasicFileAttributes a) {
         return p.toString().endsWith(MODEL_FILE);
     }
 
-    private void generateCode(C2jModels models) {
+    private void generateCode(GenerationParams params) {
         CodeGenerator.builder()
-                     .models(models)
+                     .intermediateModel(params.intermediateModel)
+                     .shareModelsTarget(params.shareModelsTarget)
                      .sourcesDirectory(sourcesDirectory.toFile().getAbsolutePath())
                      .resourcesDirectory(resourcesDirectory.toFile().getAbsolutePath())
                      .testsDirectory(testsDirectory.toFile().getAbsolutePath())
-                     .intermediateModelFileNamePrefix(intermediateModelFileNamePrefix(models))
+                     .intermediateModelFileNamePrefix(params.intermediateModelFileNamePrefix)
+                     .emitValidationReport(writeValidationReport)
                      .build()
                      .execute();
     }
@@ -169,6 +234,17 @@ private <T> Optional<T> loadOptionalModel(Class<T> clzz, Path location) {
         return ModelLoaderUtils.loadOptionalModel(clzz, location.toFile());
     }
 
+    private void emitValidationReport(ModelValidationReport report) {
+        Path modelsDir = sourcesDirectory.resolve("models");
+        try (Writer writer = Files.newBufferedWriter(modelsDir.resolve("validation-report.json"),
+                                                     StandardCharsets.UTF_8);) {
+
+            Jackson.writeWithObjectMapper(report, writer);
+        } catch (IOException e) {
+            getLog().warn("Failed to write validation report to " + modelsDir, e);
+        }
+    }
+
     private static class ModelRoot {
         private final Path modelRoot;
         private final CustomizationConfig customizationConfig;
@@ -178,4 +254,25 @@ private ModelRoot(Path modelRoot, CustomizationConfig customizationConfig) {
             this.customizationConfig = customizationConfig;
         }
     }
+
+    private static class GenerationParams {
+        private IntermediateModel intermediateModel;
+        private IntermediateModel shareModelsTarget;
+        private String intermediateModelFileNamePrefix;
+
+        public GenerationParams withIntermediateModel(IntermediateModel intermediateModel) {
+            this.intermediateModel = intermediateModel;
+            return this;
+        }
+
+        public GenerationParams withShareModelsTarget(IntermediateModel shareModelsTarget) {
+            this.shareModelsTarget = shareModelsTarget;
+            return this;
+        }
+
+        public GenerationParams withIntermediateModelFileNamePrefix(String intermediateModelFileNamePrefix) {
+            this.intermediateModelFileNamePrefix = intermediateModelFileNamePrefix;
+            return this;
+        }
+    }
 }

codegen/pom.xml

@@ -239,5 +239,10 @@
             <artifactId>mockito-core</artifactId>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>nl.jqno.equalsverifier</groupId>
+            <artifactId>equalsverifier</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

codegen/src/main/java/software/amazon/awssdk/codegen/AddShapes.java

@@ -21,6 +21,7 @@
 import static software.amazon.awssdk.codegen.internal.Utils.isMapShape;
 import static software.amazon.awssdk.codegen.internal.Utils.isScalar;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -37,10 +38,15 @@
 import software.amazon.awssdk.codegen.model.intermediate.VariableModel;
 import software.amazon.awssdk.codegen.model.service.Location;
 import software.amazon.awssdk.codegen.model.service.Member;
+import software.amazon.awssdk.codegen.model.service.Operation;
 import software.amazon.awssdk.codegen.model.service.ServiceModel;
 import software.amazon.awssdk.codegen.model.service.Shape;
 import software.amazon.awssdk.codegen.naming.NamingStrategy;
 import software.amazon.awssdk.codegen.utils.ProtocolUtils;
+import software.amazon.awssdk.codegen.validation.ModelInvalidException;
+import software.amazon.awssdk.codegen.validation.ValidationEntry;
+import software.amazon.awssdk.codegen.validation.ValidationErrorId;
+import software.amazon.awssdk.codegen.validation.ValidationErrorSeverity;
 import software.amazon.awssdk.utils.StringUtils;
 import software.amazon.awssdk.utils.Validate;
 
@@ -346,11 +352,20 @@ private boolean isGreedy(Shape parentShape, Map<String, Shape> allC2jShapes, Par
      * @throws RuntimeException If operation can't be found.
      */
     private String findRequestUri(Shape parentShape, Map<String, Shape> allC2jShapes) {
-        return builder.getService().getOperations().values().stream()
-                .filter(o -> o.getInput() != null)
-                .filter(o -> allC2jShapes.get(o.getInput().getShape()).equals(parentShape))
-                .map(o -> o.getHttp().getRequestUri())
-                .findFirst().orElseThrow(() -> new RuntimeException("Could not find request URI for input shape"));
+        Optional<Operation> operation = builder.getService().getOperations().values().stream()
+                                               .filter(o -> o.getInput() != null)
+                                               .filter(o -> allC2jShapes.get(o.getInput().getShape()).equals(parentShape))
+                                               .findFirst();
+
+        return operation.map(o -> o.getHttp().getRequestUri())
+                        .orElseThrow(() -> {
+                            String detailMsg = "Could not find request URI for input shape";
+                            ValidationEntry entry =
+                                new ValidationEntry().withErrorId(ValidationErrorId.REQUEST_URI_NOT_FOUND)
+                                                     .withDetailMessage(detailMsg)
+                                                     .withSeverity(ValidationErrorSeverity.DANGER);
+                            return ModelInvalidException.builder().validationEntries(Collections.singletonList(entry)).build();
+                        });
     }
 
     private String deriveUnmarshallerLocationName(Shape memberShape, String memberName, Member member) {