Fix multiple doc comment parse issues

This commit fixes several issues where documentation comments were
not emitting events individually, an issue where a valid documentation
comment was being declared invalid, and an invalid documentation
comment was being prepended to a valid documtation comment on the next
member.

smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlInternalTokenizer.java

@@ -149,7 +149,7 @@ void expectAndSkipBr() {
         }
     }
 
-    private void clearDocCommentLinesForBr() {
+    void clearDocCommentLinesForBr() {
         if (!docCommentLines.isEmpty()) {
             validationEventListener.accept(LoaderUtils.emitBadDocComment(getCurrentTokenLocation(),
                                                                          removePendingDocCommentLines()));

smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlModelLoader.java

@@ -448,7 +448,7 @@ private void parseFirstShapeStatement(SourceLocation possibleDocCommentLocation)
             try {
                 tokenizer.skipWsAndDocs();
                 String docLines = tokenizer.removePendingDocCommentLines();
-                List<IdlTraitParser.Result> traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this);
+                List<IdlTraitParser.Result> traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this, false);
                 if (docLines != null) {
                     traits.add(new IdlTraitParser.Result(DocumentationTrait.ID.toString(),
                                                          new StringNode(docLines, possibleDocCommentLocation),
@@ -467,7 +467,7 @@ private void parseSubsequentShapeStatements() {
         while (tokenizer.hasNext()) {
             try {
                 boolean hasDocComment = tokenizer.getCurrentToken() == IdlToken.DOC_COMMENT;
-                List<IdlTraitParser.Result> traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this);
+                List<IdlTraitParser.Result> traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this, false);
                 if (parseShapeDefinition(traits, hasDocComment)) {
                     parseShapeOrApply(traits);
                 }
@@ -672,7 +672,7 @@ private void parseEnumShape(ShapeId id, SourceLocation location, AbstractShapeBu
 
         while (tokenizer.getCurrentToken() != IdlToken.EOF && tokenizer.getCurrentToken() != IdlToken.RBRACE) {
             List<IdlTraitParser.Result> memberTraits = IdlTraitParser
-                    .parseDocsAndTraitsBeforeShape(this);
+                    .parseDocsAndTraitsBeforeShape(this, true);
             SourceLocation memberLocation = tokenizer.getCurrentTokenLocation();
             tokenizer.expect(IdlToken.IDENTIFIER);
             String memberName = internString(tokenizer.getCurrentTokenLexeme());
@@ -723,6 +723,9 @@ private void parseMembers(LoadOperation.DefineShape op) {
         Set<String> definedMembers = new HashSet<>();
 
         tokenizer.skipWsAndDocs();
+        // Clear out any doc comments between the shape type or
+        // identifier and the opening LBRACE token.
+        tokenizer.clearDocCommentLinesForBr();
         tokenizer.expect(IdlToken.LBRACE);
         tokenizer.next();
         tokenizer.skipWs();
@@ -740,7 +743,7 @@ private void parseMember(LoadOperation.DefineShape operation, Set<String> define
         ShapeId parent = operation.toShapeId();
 
         // Parse optional member traits.
-        List<IdlTraitParser.Result> memberTraits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this);
+        List<IdlTraitParser.Result> memberTraits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this, true);
         SourceLocation memberLocation = tokenizer.getCurrentTokenLocation();
 
         // Handle the case of a dangling documentation comment followed by "}".
@@ -988,7 +991,7 @@ private void parseInlineableOperationMember(
     }
 
     private ShapeId parseInlineStructure(String name, IdlTraitParser.Result defaultTrait) {
-        List<IdlTraitParser.Result> traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this);
+        List<IdlTraitParser.Result> traits = IdlTraitParser.parseDocsAndTraitsBeforeShape(this, true);
         if (defaultTrait != null) {
             traits.add(defaultTrait);
         }

smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlTraitParser.java

@@ -69,7 +69,7 @@ private IdlTraitParser() { }
      * @param loader IDL parser.
      * @return Return the parsed traits.
      */
-    static List<Result> parseDocsAndTraitsBeforeShape(IdlModelLoader loader) {
+    static List<Result> parseDocsAndTraitsBeforeShape(IdlModelLoader loader, boolean isParsingMember) {
         IdlInternalTokenizer tokenizer = loader.getTokenizer();
         tokenizer.skipWs();
 
@@ -91,6 +91,12 @@ static List<Result> parseDocsAndTraitsBeforeShape(IdlModelLoader loader) {
             traits.add(docComment);
         }
         tokenizer.skipWsAndDocs();
+        // If there's a doc comment between traits and their target member,
+        // clear them so they're not prepended to a doc comment on the next
+        // member shape.
+        if (isParsingMember) {
+            tokenizer.clearDocCommentLinesForBr();
+        }
 
         return traits;
     }

smithy-model/src/test/resources/software/amazon/smithy/model/errorfiles/loader/detects-invalid-documentation-comments.errors

@@ -1,7 +1,9 @@
 [WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 1 (before control) | Model.BadDocumentationComment
 [WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 2 (before namespace) | Model.BadDocumentationComment
 [WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 3 (before use) | Model.BadDocumentationComment
-[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 4 (inside trait)\nInvalid 5 (not before traits)\nValid docs for Foo | Model.BadDocumentationComment
-[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 6 (not before traits)\nInvalid 7 (inside shape line)\nInvalid 8 (dangling) | Model.BadDocumentationComment
-[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 9 (not before traits)\nInvalid 10 (dangling) | Model.BadDocumentationComment
+[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 4 (inside trait)\nInvalid 5 (not before traits) | Model.BadDocumentationComment
+[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 6 (not before traits)\nInvalid 7 (inside shape line) | Model.BadDocumentationComment
+[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 8 (dangling) | Model.BadDocumentationComment
+[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 9 (not before traits) | Model.BadDocumentationComment
+[WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 10 (dangling) | Model.BadDocumentationComment
 [WARNING] -: Found documentation comments ('///') attached to nothing. Documentation comments must appear on their own lines, directly before shapes and members, and before any traits. The invalid comments were: Invalid 11 (dangling) | Model.BadDocumentationComment

smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.json

@@ -30,6 +30,12 @@
                         "smithy.api#documentation": "Docs on another member!"
                     }
                 },
+                "boot": {
+                    "target": "smithy.api#String",
+                    "traits": {
+                        "smithy.api#documentation": "I'll only have this documentation."
+                    }
+                },
                 "bam": {
                     "target": "smithy.api#String"
                 }

smithy-model/src/test/resources/software/amazon/smithy/model/loader/valid/doc-comments/doc-comments.smithy

@@ -29,6 +29,9 @@ structure MyStructure {
     /// IMPORTANT: These docs are ignored since they come after traits!
     baz: String,
 
+    /// I'll only have this documentation.
+    boot: String
+
     // no docs.
     bam: String,
 }