Chg: NewLine handling, ReadLine, SkipWhiteSpaes

Author: neolithos

NeoLua.Test/Parser.cs

@@ -40,7 +40,7 @@ private void TokenTest(ILuaLexer lex, params KeyValuePair<LuaToken, string>[] ex
 					lex.Next();
 				}
 				if (lex.Current.Typ != LuaToken.Eof)
-					Assert.Fail();
+					Assert.Fail("Eof expected.");
 			}
 		} // func TokenTest
 
@@ -247,6 +247,8 @@ public void TestConstants()
 
 		#endregion
 
+		#region -- Test Parser --------------------------------------------------------
+
 		[TestMethod]
 		public void TestParser01()
 		{
@@ -288,6 +290,8 @@ public void TestParserIssue92()
 			);
 		}
 
+		#endregion
+
 		#region -- Test Position ------------------------------------------------------
 
 		[TestMethod]
@@ -361,6 +365,36 @@ public void TestPosition02()
 			}
 		}
 
+		[TestMethod]
+		public void TestPosition03()
+		{
+			var t = "\r\nNull(";
+			using (var lex = CreateLuaLexer(t))
+			{
+				lex.Next();
+
+				Assert.AreEqual(2, lex.Current.Start.Line);
+				Assert.AreEqual(1, lex.Current.Start.Col);
+				Assert.AreEqual(2, lex.Current.Start.Index);
+
+				Assert.AreEqual(2, lex.Current.End.Line);
+				Assert.AreEqual(5, lex.Current.End.Col);
+				Assert.AreEqual(6, lex.Current.End.Index);
+				Assert.AreEqual("Null", t.Substring((int)lex.Current.Start.Index, (int)lex.Current.End.Index - (int)lex.Current.Start.Index));
+
+				lex.Next();
+
+				Assert.AreEqual(2, lex.Current.Start.Line);
+				Assert.AreEqual(5, lex.Current.Start.Col);
+				Assert.AreEqual(6, lex.Current.Start.Index);
+
+				Assert.AreEqual(2, lex.Current.End.Line);
+				Assert.AreEqual(6, lex.Current.End.Col);
+				Assert.AreEqual(7, lex.Current.End.Index);
+				Assert.AreEqual("(", t.Substring((int)lex.Current.Start.Index, (int)lex.Current.End.Index - (int)lex.Current.Start.Index));
+			}
+		}
+
 		#endregion
 
 		#region -- Html Lexer ---------------------------------------------------------
@@ -446,6 +480,5 @@ public void ParseVarTest()
 
 
 		#endregion
-
 	} // class Lexer
 }

NeoLua/LuaLexer.cs

@@ -368,17 +368,15 @@ public sealed class LuaCharLexer : IDisposable
 		private readonly bool leaveOpen;            // do not dispose the text reader at the end
 
 		private readonly char[] lookAheadBuffer;
+		private readonly byte[] moveIndex;
 		private int lookAheadOffset;
 		private int lookAheadEof;
 
 		private int currentLine;                    // Line in the source file
 		private int currentColumn;                  // Column in the source file
 		private readonly int firstColumnIndex;      // Index of the first char in line
 		private long currentIndex;                  // Index in the source file
-
-		private bool incrementLine = false;
-		private bool moveIndex = false;
-
+		
 		private Position startPosition;             // Start of the current token
 		private StringBuilder currentStringBuilder = new StringBuilder(); // Char buffer, for collected chars
 
@@ -404,31 +402,40 @@ public LuaCharLexer(string fileName, TextReader tr, int lookAheadLength, bool le
 			this.leaveOpen = leaveOpen;
 
 			this.lookAheadBuffer = new char[lookAheadLength];
+			this.moveIndex = new byte[lookAheadLength];
 			this.lookAheadOffset = lookAheadLength - 1;
 			this.lookAheadEof = -1;
 
 			this.currentLine = currentLine;
 			this.currentColumn = currentColumn - 1;
 			this.firstColumnIndex = firstColumnIndex;
-			this.currentIndex = -1;
+			this.currentIndex = 0;
 
 			// inital fill buffer
 			for (var i = 0; i < lookAheadLength - 1; i++)
 			{
 				if (lookAheadEof >= 0)
+				{
 					lookAheadBuffer[i] = '\0';
+					moveIndex[i] = 0;
+				}
 				else
 				{
-					var c = ReadCore();
-					if (c == -1)
+					var c = ReadCore(out var len);
+					if (len == 0)
 					{
 						lookAheadEof = i;
 						lookAheadBuffer[i] = '\0';
+						moveIndex[i] = 0;
 					}
 					else
+					{
 						lookAheadBuffer[i] = (char)c;
+						moveIndex[i] = len;
+					}
 				}
 			}
+			moveIndex[lookAheadOffset] = 0;
 
 			// read first char from the buffer
 			Next();
@@ -460,10 +467,13 @@ public void AppendValue(char value)
 		public void AppendValue(string value)
 			=> currentStringBuilder.Append(Cur);
 
-		private int ReadCore()
+		private char ReadCore(out byte len)
 		{
 			if (tr == null) // Source file is readed
-				return -1;
+			{
+				len = 0;
+				return '\0';
+			}
 			else
 			{
 				var i = tr.Read();
@@ -472,105 +482,93 @@ private int ReadCore()
 					if (!leaveOpen)
 						tr.Dispose();
 					tr = null;
+
+					len = 0;
+					return '\0';
+				}
+				else if (i == 10)
+				{
+					if (tr.Peek() == 13)
+					{
+						tr.Read();
+						len = 2;
+						return (char)i;
+					}
+					else
+					{
+						len = 1;
+						return (char)i;
+					}
+				}
+				else if (i == 13)
+				{
+					if (tr.Peek() == 10)
+					{
+						tr.Read();
+						len = 2;
+						return '\n';
+					}
+					else
+					{
+						len = 1;
+						return '\n';
+					}
 				}
-				return i;
+
+				len = 1;
+				return (char)i;
 			}
 		} // func ReadCore
 
 		private bool ReadCharToBuffer()
 		{
 			if (lookAheadEof < 0)
 			{
-				var c = ReadCore();
-				if (c == -1)
+				var c = ReadCore(out var len);
+				if (len == 0)
 				{
 					lookAheadEof = lookAheadOffset;
 					lookAheadBuffer[lookAheadOffset] = '\0';
+					moveIndex[lookAheadOffset] = 0;
 				}
 				else
+				{
 					lookAheadBuffer[lookAheadOffset] = (char)c;
+					moveIndex[lookAheadOffset] = len;
+				}
 
 				return true;
 			}
 			else if (lookAheadEof != lookAheadOffset)
 			{
 				lookAheadBuffer[lookAheadOffset] = '\0';
+				moveIndex[lookAheadOffset] = 0;
 				return true;
 			}
 			else
 				return false;
 		} // func ReadCharToBuffer
-
-		private bool MoveNextCore()
-		{
-			if (ReadCharToBuffer())
-			{
-				if (lookAheadBuffer.Length > 1)
-				{
-					lookAheadOffset++;
-					if (lookAheadOffset >= lookAheadBuffer.Length)
-						lookAheadOffset = 0;
-				}
-
-				return true;
-			}
-			else
-				return false;
-		} // func MoveNextCore
 
 		/// <summary>Move the char stream and discard the buffer.</summary>
 		public void Next()
 		{
-			if (moveIndex)
+			currentIndex += moveIndex[lookAheadOffset];
+			if (lookAheadBuffer[lookAheadOffset] == '\n')
 			{
-				moveIndex = false;
-				currentIndex++;
+				currentColumn = firstColumnIndex;
+				currentLine++;
 			}
+			else
+				currentColumn++;
 
-			if (MoveNextCore())
+			if (ReadCharToBuffer())
 			{
-				currentIndex++;
-
-				var c = lookAheadBuffer[lookAheadOffset];
-
-				// Normalize new line
-				if (c == '\n')
-				{
-					if (lookAheadBuffer.Length > 1 && IsLookAHead('\r', 1))
-					{
-						moveIndex = MoveNextCore();
-						lookAheadBuffer[lookAheadOffset] = '\n'; // replace with unique new line
-					}
-					else if (tr != null && tr.Peek() == 13)
-					{
-						moveIndex = MoveNextCore();
-						lookAheadBuffer[lookAheadOffset] = '\n'; // replace with unique new line
-					}
-
-					currentColumn++;
-					incrementLine = true;
-				}
-				else if (c == '\r')
-				{
-					if (lookAheadBuffer.Length > 1 && IsLookAHead('\n', 1))
-						moveIndex = MoveNextCore();
-					else if (tr != null && tr.Peek() == 10)
-						moveIndex = MoveNextCore();
-					else
-						lookAheadBuffer[lookAheadOffset] = '\n'; // replace with unique new line
-
-					currentColumn++;
-					incrementLine = true;
-				}
-				else if (incrementLine)
+				if (lookAheadBuffer.Length > 1)
 				{
-					currentColumn = firstColumnIndex;
-					currentLine++;
-
-					incrementLine = false;
+					lookAheadOffset++;
+					if (lookAheadOffset >= lookAheadBuffer.Length)
+						lookAheadOffset = 0;
 				}
-				else
-					currentColumn++;
 			}
 		} // proc Next
 
@@ -589,6 +587,25 @@ public void Eat()
 			Next();
 		} // proc Eat
 
+		/// <summary>Read a whole line</summary>
+		/// <returns></returns>
+		public string ReadLine()
+		{
+			while (!IsEof && Cur != '\n')
+				Eat();
+
+			var curValue = CurValue;
+			ResetCurValue();
+			return curValue;
+		} // func ReadLine
+
+		/// <summary>Skip white spaces</summary>
+		public void SkipWhiteSpaces()
+		{
+			while (!IsEof && Cur != '\n' && Char.IsWhiteSpace(Cur))
+				Next();
+		} // proc SkipWhiteSpaces
+
 		/// <summary>Create a new token with the current buffer.</summary>
 		/// <param name="kind">Token type</param>
 		/// <returns>Token</returns>
@@ -661,6 +678,17 @@ public bool IsLookAHead(string value, int offset = 0)
 			return true;
 		} // func LookAHead
 
+		/// <summary>Get look a head.</summary>
+		/// <param name="offset"></param>
+		/// <returns></returns>
+		public char GetLookAHead(int offset = 0)
+		{
+			if (offset > lookAheadBuffer.Length)
+				throw new ArgumentOutOfRangeException(nameof(offset), lookAheadBuffer.Length, "Look a head buffer is to small.");
+
+			return lookAheadBuffer[offset];
+		}// func GetLookAHead
+		
 		/// <summary></summary>
 		public void ResetCurValue()
 		{