Add tests for parsing tag handles

ballerina/tests/parse_string.bal

@@ -40,7 +40,8 @@ function basicTypeDataForParseString() returns [string, typedesc<anydata>, anyda
     [">-\nThis is a folded\nblock scalar string", string, "This is a folded block scalar string"],
     ["123", int, 123],
     ["12.23", float, 12.23],
-    ["12.23", decimal, 12.23d]
+    ["12.23", decimal, 12.23d],
+    ["'escaped single ''quote'", string, "escaped single 'quote"]
 ];
 
 @test:Config {

ballerina/tests/parser_tests.bal

@@ -0,0 +1,40 @@
+// Copyright (c) 2024, WSO2 LLC. (https://www.wso2.com).
+//
+// WSO2 LLC. licenses this file to you under the Apache License,
+// Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import ballerina/io;
+import ballerina/test;
+
+const PARSER_TESTS_PATH = FILE_PATH + "parser/";
+
+@test:Config {
+    dataProvider: tagHandleData
+}
+isolated function testTagHandles(string inputPath, TestCase expectedValue) returns error? {
+    string filePath = PARSER_TESTS_PATH + inputPath;
+    string content = check io:fileReadString(filePath);
+    TestCase actual = check parseString(content);
+    test:assertEquals(actual, expectedValue);
+}
+
+function tagHandleData() returns [string, TestCase][] => [
+    ["tag_handle_1.yaml", {case: "uri_scanner"}],
+    ["tag_handle_2.yaml", {case: "yaml_version"}],
+    ["tag_handle_3.yaml", {case: "verbitam"}]
+];
+
+type TestCase record {|
+    string case;
+|};

ballerina/tests/resources/parser/tag_handle_1.yaml

@@ -0,0 +1,3 @@
+%TAG ! tag:example.com,2002:
+---
+case: uri_scanner

ballerina/tests/resources/parser/tag_handle_2.yaml

@@ -0,0 +1,3 @@
+%YAML 1.2
+---
+case: "yaml_version"

ballerina/tests/resources/parser/tag_handle_3.yaml

@@ -0,0 +1 @@
+case:  !<tag:yaml.org,2002:str> verbitam

native/src/main/java/io/ballerina/lib/data/yaml/lexer/Scanner.java

@@ -107,9 +107,13 @@ public static void iterate(LexerState sm, Scan scan, Token.TokenType token, bool
             if (scan.scan(sm)) {
                 if (include && sm.peek() != '\n') {
                     sm.forward();
+                    sm.tokenize(token);
+                    return;
                 }
                 if (include && sm.peek() != '\r' && sm.peek(1) != '\n') {
                     sm.forward(2);
+                    sm.tokenize(token);
+                    return;
                 }
                 sm.tokenize(token);
                 return;
@@ -134,7 +138,7 @@ public static class SingleQuoteCharScanner implements Scan {
          * Process double quoted scalar values.
          */
         @Override
-        public boolean scan(LexerState sm) throws Error.YamlParserException {
+        public boolean scan(LexerState sm) {
             // Process nb-json characters
             if (matchPattern(sm, List.of(JSON_PATTERN), List.of(new Utils.CharPattern('\'')))) {
                 sm.appendToLexeme(Character.toString(sm.peek()));
@@ -148,10 +152,9 @@ public boolean scan(LexerState sm) throws Error.YamlParserException {
                     sm.forward();
                     return false;
                 }
-                return true;
             }
 
-            throw new Error.YamlParserException("invalid single quote char", sm.getLine(), sm.getColumn());
+            return true;
         }
     }
 
@@ -179,12 +182,7 @@ public boolean scan(LexerState sm) throws Error.YamlParserException {
                 return false;
             }
 
-            // Terminate when delimiter is found
-            if (sm.peek() == '\"') {
-                return true;
-            }
-
-            throw new Error.YamlParserException("invalid double quoted character", sm.getLine(), sm.getColumn());
+            return true;
         }
     }
 
@@ -234,7 +232,8 @@ public boolean scan(LexerState sm) throws Error.YamlParserException {
                 return false;
             }
             int currentChar = sm.peek();
-            if (WHITE_SPACE_PATTERN.pattern(currentChar) || currentChar == '.') {
+            if (WHITE_SPACE_PATTERN.pattern(currentChar) || currentChar == '.'
+                    || LINE_BREAK_PATTERN.pattern(currentChar)) {
                 return true;
             }
             throw new Error.YamlParserException("invalid digit character", sm.getLine(), sm.getColumn());

native/src/main/java/io/ballerina/lib/data/yaml/parser/YamlParser.java

@@ -766,7 +766,7 @@ private static YamlEvent getNextYamlDocEvent(ComposerState state) throws Error.Y
     }
 
     private static boolean isBeginWithStream(ComposerState state, YamlEvent event) {
-        return event.getKind() == YamlEvent.EventKind.DOCUMENT_MARKER_EVENT
+        return event.getKind() == YamlEvent.EventKind.DOCUMENT_MARKER_EVENT && state.terminatedDocEvent != null
                 && ((YamlEvent.DocumentMarkerEvent) state.terminatedDocEvent).isExplicit();
     }