Validate unknown member reference

Add validation for the case where an endpoint test references a
non-existent operation input shape member.

As part of this change, `CodeGenerator` has ben updated to catch
`ModelInvalidException` and write the validation entries in the
validation-report.json.

Author: dagnir

codegen/src/main/java/software/amazon/awssdk/codegen/CodeGenerator.java

@@ -29,6 +29,7 @@
 import software.amazon.awssdk.codegen.internal.Jackson;
 import software.amazon.awssdk.codegen.internal.Utils;
 import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
+import software.amazon.awssdk.codegen.validation.ModelInvalidException;
 import software.amazon.awssdk.codegen.validation.ModelValidationContext;
 import software.amazon.awssdk.codegen.validation.ModelValidationReport;
 import software.amazon.awssdk.codegen.validation.ModelValidator;
@@ -131,6 +132,13 @@ public void execute() {
 
         } catch (Exception e) {
             log.error(() -> "Failed to generate code. ", e);
+
+            if (e instanceof ModelInvalidException && emitValidationReport) {
+                ModelInvalidException invalidException = (ModelInvalidException) e;
+                report.setValidationEntries(invalidException.validationEntries());
+                writeValidationReport(report);
+            }
+
             throw new RuntimeException(
                     "Failed to generate code. Exception message : " + e.getMessage(), e);
         }

codegen/src/main/java/software/amazon/awssdk/codegen/emitters/tasks/BaseGeneratorTasks.java

@@ -71,6 +71,8 @@ protected void compute() {
                 ForkJoinTask.invokeAll(createTasks());
                 log.info("  Completed " + taskName + ".");
             }
+        } catch (RuntimeException e) {
+            throw e;
         } catch (Exception e) {
             throw new RuntimeException(e);
         }

codegen/src/main/java/software/amazon/awssdk/codegen/poet/rules/EndpointRulesClientTestSpec.java

@@ -61,6 +61,10 @@
 import software.amazon.awssdk.codegen.poet.PoetExtension;
 import software.amazon.awssdk.codegen.poet.PoetUtils;
 import software.amazon.awssdk.codegen.utils.AuthUtils;
+import software.amazon.awssdk.codegen.validation.ModelInvalidException;
+import software.amazon.awssdk.codegen.validation.ValidationEntry;
+import software.amazon.awssdk.codegen.validation.ValidationErrorId;
+import software.amazon.awssdk.codegen.validation.ValidationErrorSeverity;
 import software.amazon.awssdk.core.SdkSystemSetting;
 import software.amazon.awssdk.core.async.AsyncRequestBody;
 import software.amazon.awssdk.core.rules.testing.AsyncTestCase;
@@ -445,6 +449,20 @@ private CodeBlock requestCreation(OperationModel opModel, Map<String, TreeNode>
         if (opParams != null) {
             opParams.forEach((n, v) -> {
                 MemberModel memberModel = opModel.getInputShape().getMemberByC2jName(n);
+
+                if (memberModel == null) {
+                    String detailMsg = String.format("Endpoint test definition references member '%s' on the input shape '%s' "
+                                                     + "but no such member is defined.", n, opModel.getInputShape().getC2jName());
+                    ValidationEntry entry =
+                        new ValidationEntry()
+                            .withSeverity(ValidationErrorSeverity.DANGER)
+                            .withErrorId(ValidationErrorId.UNKNOWN_SHAPE_MEMBER)
+                            .withDetailMessage(detailMsg);
+
+                    throw ModelInvalidException.builder()
+                                               .validationEntries(Collections.singletonList(entry))
+                                               .build();
+                }
                 CodeBlock memberValue = createMemberValue(memberModel, v);
                 b.add(".$N($L)", memberModel.getFluentSetterMethodName(), memberValue);
             });

codegen/src/main/java/software/amazon/awssdk/codegen/validation/ModelInvalidException.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.codegen.validation;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Exception thrown during code generation to signal that the model is invalid.
+ */
+public class ModelInvalidException extends RuntimeException {
+    private final List<ValidationEntry> validationEntries;
+
+    private ModelInvalidException(Builder b) {
+        super("Validation failed with the following errors: " + b.validationEntries);
+        this.validationEntries = Collections.unmodifiableList(new ArrayList<>(b.validationEntries));
+    }
+
+    public List<ValidationEntry> validationEntries() {
+        return validationEntries;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static class Builder {
+        private List<ValidationEntry> validationEntries;
+
+        public Builder validationEntries(List<ValidationEntry> validationEntries) {
+            if (validationEntries == null) {
+                this.validationEntries = Collections.emptyList();
+            } else {
+                this.validationEntries = validationEntries;
+            }
+
+            return this;
+        }
+
+        public ModelInvalidException build() {
+            return new ModelInvalidException(this);
+        }
+    }
+}

codegen/src/main/java/software/amazon/awssdk/codegen/validation/ValidationEntry.java

@@ -15,6 +15,8 @@
 
 package software.amazon.awssdk.codegen.validation;
 
+import software.amazon.awssdk.utils.ToString;
+
 public final class ValidationEntry {
     private ValidationErrorId errorId;
     private ValidationErrorSeverity severity;
@@ -58,4 +60,13 @@ public ValidationEntry withDetailMessage(String detailMessage) {
         setDetailMessage(detailMessage);
         return this;
     }
+
+    @Override
+    public String toString() {
+        return ToString.builder("ValidationEntry")
+            .add("errorId", errorId)
+            .add("severity", severity)
+            .add("detailMessage", detailMessage)
+            .build();
+    }
 }

codegen/src/main/java/software/amazon/awssdk/codegen/validation/ValidationErrorId.java

@@ -20,9 +20,7 @@ public enum ValidationErrorId {
         "The shared models between two services differ in their definition, which causes differences in the source"
         + " files generated by the code generator."
     ),
-    MEMBER_WITH_UNKNOWN_SHAPE(
-        "The shape declares a member targeting an unknown shape."
-    ),
+    UNKNOWN_SHAPE_MEMBER("The model references an unknown shape member."),
     ;
 
     private final String description;

codegen/src/test/java/software/amazon/awssdk/codegen/CodeGeneratorTest.java

@@ -22,7 +22,9 @@
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.FileVisitResult;
@@ -37,9 +39,13 @@
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.codegen.internal.Jackson;
 import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
+import software.amazon.awssdk.codegen.model.rules.endpoints.EndpointTestSuiteModel;
 import software.amazon.awssdk.codegen.poet.ClientTestModels;
+import software.amazon.awssdk.codegen.validation.ModelInvalidException;
 import software.amazon.awssdk.codegen.validation.ModelValidator;
+import software.amazon.awssdk.codegen.validation.ValidationErrorId;
 
 public class CodeGeneratorTest {
     private static final String VALIDATION_REPORT_NAME = "validation-report.json";
@@ -125,6 +131,30 @@ void execute_c2jModelsAndIntermediateModel_generateSameCode() throws IOException
         }
     }
 
+    @Test
+    void execute_endpointsTestReferencesUnknownOperationMember_throwsValidationError() throws IOException {
+        ModelValidator mockValidator = mock(ModelValidator.class);
+        when(mockValidator.validateModels(any())).thenReturn(Collections.emptyList());
+
+        C2jModels referenceModels = ClientTestModels.awsJsonServiceC2jModels();
+
+        C2jModels c2jModelsWithBadTest =
+            C2jModels.builder()
+                     .endpointTestSuiteModel(getBrokenEndpointTestSuiteModel())
+                     .customizationConfig(referenceModels.customizationConfig())
+                     .serviceModel(referenceModels.serviceModel())
+                     .paginatorsModel(referenceModels.paginatorsModel())
+                     .build();
+
+        assertThatThrownBy(() -> generateCodeFromC2jModels(c2jModelsWithBadTest, outputDir, true,
+                                                           Collections.singletonList(mockValidator)))
+            .hasCauseInstanceOf(ModelInvalidException.class)
+            .matches(e -> {
+                ModelInvalidException exception = (ModelInvalidException) e.getCause();
+                return exception.validationEntries().get(0).getErrorId() == ValidationErrorId.UNKNOWN_SHAPE_MEMBER;
+            });
+    }
+
     private void generateCodeFromC2jModels(C2jModels c2jModels, Path outputDir) {
         generateCodeFromC2jModels(c2jModels, outputDir, false, null);
     }
@@ -170,6 +200,18 @@ private static Path validationReportPath(Path root) {
         return root.resolve(Paths.get("generated-sources", "sdk", "models", VALIDATION_REPORT_NAME));
     }
 
+    private EndpointTestSuiteModel getBrokenEndpointTestSuiteModel() throws IOException {
+        InputStream resourceAsStream = getClass().getResourceAsStream("incorrect-endpoint-tests.json");
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        byte[] buffer = new byte[1024];
+        int read;
+        while ((read = resourceAsStream.read(buffer)) != -1) {
+            baos.write(buffer, 0, read);
+        }
+        String json = StandardCharsets.UTF_8.decode(ByteBuffer.wrap(baos.toByteArray())).toString();
+        return Jackson.load(EndpointTestSuiteModel.class, json);
+    }
+
     private static void deleteDirectory(Path dir) throws IOException {
         Files.walkFileTree(dir, new FileVisitor<Path>() {
 

codegen/src/test/resources/software/amazon/awssdk/codegen/incorrect-endpoint-tests.json

@@ -0,0 +1,26 @@
+{
+  "testCases": [
+    {
+      "documentation": "Test references undefined operation member",
+      "expect": {
+        "error": "Some error"
+      },
+      "operationInputs": [
+        {
+          "builtInParams": {
+            "AWS::Region": "us-east-1"
+          },
+          "operationName": "APostOperation",
+          "operationParams": {
+            "Foo": "bar"
+          }
+        }
+      ],
+      "params": {
+        "Region": "us-east-1",
+        "UseFIPS": false,
+        "UseDualStack": false
+      }
+    }
+  ]
+}