[vscode] Fix column for non-BMP characters.

#64

ConsoleHost.fu

@@ -1,6 +1,6 @@
 // ConsoleHost.fu - emit errors to stderr
 //
-// Copyright (C) 2023  Piotr Fusik
+// Copyright (C) 2023-2024  Piotr Fusik
 //
 // This file is part of Fusion Transpiler,
 // see https://github.com/fusionlanguage/fut
@@ -22,7 +22,7 @@ public abstract class FuConsoleHost : GenHost
 {
 	internal bool HasErrors = false;
 
-	public override void ReportError!(string filename, int startLine, int startColumn, int endLine, int endColumn, string message)
+	public override void ReportError!(string filename, int startLine, int startUtf16Column, int endLine, int endUtf16Column, string message)
 	{
 		this.HasErrors = true;
 		Console.Error.WriteLine($"{filename}({startLine}): ERROR: {message}");

Lexer.fu

@@ -1,6 +1,6 @@
 // Lexer.fu - Fusion lexer
 //
-// Copyright (C) 2011-2023  Piotr Fusik
+// Copyright (C) 2011-2024  Piotr Fusik
 //
 // This file is part of Fusion Transpiler,
 // see https://github.com/fusionlanguage/fut
@@ -20,7 +20,7 @@
 
 public abstract class FuParserHost
 {
-	public abstract void ReportError!(string filename, int startLine, int startColumn, int endLine, int endColumn, string message);
+	public abstract void ReportError!(string filename, int startLine, int startUtf16Column, int endLine, int endUtf16Column, string message);
 }
 
 public enum FuToken
@@ -145,8 +145,8 @@ public abstract class FuLexer
 	FuParserHost! Host;
 	protected string() Filename;
 	protected int Line;
-	protected int Column;
-	protected int TokenColumn;
+	int Utf16Column;
+	int TokenUtf16Column;
 	protected int LexemeOffset;
 	protected FuToken CurrentToken;
 	protected long LongValue; // for FuToken.LiteralLong, FuToken.LiteralChar
@@ -175,7 +175,7 @@ public abstract class FuLexer
 		this.InputLength = inputLength;
 		this.NextOffset = 0;
 		this.Line = 1;
-		this.Column = 1;
+		this.Utf16Column = 0;
 		FillNextChar();
 		if (this.NextChar == 0xfeff) // BOM
 			FillNextChar();
@@ -184,7 +184,7 @@ public abstract class FuLexer
 
 	protected void ReportError(string message)
 	{
-		this.Host.ReportError(this.Filename, this.Line, this.TokenColumn, this.Line, this.Column, message);
+		this.Host.ReportError(this.Filename, this.Line, this.TokenUtf16Column, this.Line, this.Utf16Column, message);
 	}
 
 	int ReadByte!()
@@ -247,15 +247,15 @@ public abstract class FuLexer
 		switch (c) {
 		case '\t':
 		case ' ':
-			this.Column++;
+			this.Utf16Column++;
 			break;
 		case '\n':
 			this.Line++;
-			this.Column = 1;
+			this.Utf16Column = 0;
 			this.AtLineStart = true;
 			break;
 		default:
-			this.Column++;
+			this.Utf16Column += c < 0x10000 ? 1 : 2;
 			this.AtLineStart = false;
 			break;
 		}
@@ -485,7 +485,7 @@ public abstract class FuLexer
 	{
 		for (;;) {
 			bool atLineStart = this.AtLineStart;
-			this.TokenColumn = this.Column;
+			this.TokenUtf16Column = this.Utf16Column;
 			this.LexemeOffset = this.CharOffset;
 			int c = ReadChar();
 			switch (c) {

editors/vscode/src/extension.ts

@@ -1,3 +1,23 @@
+// extension.ts - Visual Studio Code extension
+//
+// Copyright (C) 2023-2024  Piotr Fusik
+//
+// This file is part of Fusion Transpiler,
+// see https://github.com/fusionlanguage/fut
+//
+// Fusion Transpiler is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Fusion Transpiler is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Fusion Transpiler.  If not, see http://www.gnu.org/licenses/
+
 import * as vscode from "vscode";
 import { FuParser, FuProgram, FuSystem, FuParserHost } from "./parser.js";
 
@@ -6,9 +26,9 @@ class VsCodeHost extends FuParserHost
 	#system = FuSystem.new();
 	#diagnostics: vscode.Diagnostic[] = [];
 
-	reportError(filename: string, startLine: number, startColumn: number, endLine: number, endColumn: number, message: string) : void
+	reportError(filename: string, startLine: number, startUtf16Column: number, endLine: number, endUtf16Column: number, message: string) : void
 	{
-		this.#diagnostics.push(new vscode.Diagnostic(new vscode.Range(startLine - 1, startColumn - 1, endLine - 1, endColumn - 1), message));
+		this.#diagnostics.push(new vscode.Diagnostic(new vscode.Range(startLine - 1, startUtf16Column, endLine - 1, endUtf16Column), message));
 	}
 
 	updateDiagnostics(document: vscode.TextDocument, diagnosticCollection: vscode.DiagnosticCollection): void

libfut.cpp

@@ -48,7 +48,7 @@ void FuLexer::open(std::string_view filename, uint8_t const * input, int inputLe
 	this->inputLength = inputLength;
 	this->nextOffset = 0;
 	this->line = 1;
-	this->column = 1;
+	this->utf16Column = 0;
 	fillNextChar();
 	if (this->nextChar == 65279)
 		fillNextChar();
@@ -57,7 +57,7 @@ void FuLexer::open(std::string_view filename, uint8_t const * input, int inputLe
 
 void FuLexer::reportError(std::string_view message) const
 {
-	this->host->reportError(this->filename, this->line, this->tokenColumn, this->line, this->column, message);
+	this->host->reportError(this->filename, this->line, this->tokenUtf16Column, this->line, this->utf16Column, message);
 }
 
 int FuLexer::readByte()
@@ -124,15 +124,15 @@ int FuLexer::readChar()
 	switch (c) {
 	case '\t':
 	case ' ':
-		this->column++;
+		this->utf16Column++;
 		break;
 	case '\n':
 		this->line++;
-		this->column = 1;
+		this->utf16Column = 0;
 		this->atLineStart = true;
 		break;
 	default:
-		this->column++;
+		this->utf16Column += c < 65536 ? 1 : 2;
 		this->atLineStart = false;
 		break;
 	}
@@ -370,7 +370,7 @@ FuToken FuLexer::readPreToken()
 {
 	for (;;) {
 		bool atLineStart = this->atLineStart;
-		this->tokenColumn = this->column;
+		this->tokenUtf16Column = this->utf16Column;
 		this->lexemeOffset = this->charOffset;
 		int c = readChar();
 		switch (c) {
@@ -4065,7 +4065,7 @@ void FuParser::parse(std::string_view filename, uint8_t const * input, int input
 	}
 }
 
-void FuConsoleHost::reportError(std::string_view filename, int startLine, int startColumn, int endLine, int endColumn, std::string_view message)
+void FuConsoleHost::reportError(std::string_view filename, int startLine, int startUtf16Column, int endLine, int endUtf16Column, std::string_view message)
 {
 	this->hasErrors = true;
 	std::cerr << filename << "(" << startLine << "): ERROR: " << message << '\n';

libfut.cs

@@ -11,7 +11,7 @@ namespace Fusion
 	public abstract class FuParserHost
 	{
 
-		public abstract void ReportError(string filename, int startLine, int startColumn, int endLine, int endColumn, string message);
+		public abstract void ReportError(string filename, int startLine, int startUtf16Column, int endLine, int endUtf16Column, string message);
 	}
 
 	public enum FuToken
@@ -145,9 +145,9 @@ public abstract class FuLexer
 
 		protected int Line;
 
-		protected int Column;
+		int Utf16Column;
 
-		protected int TokenColumn;
+		int TokenUtf16Column;
 
 		protected int LexemeOffset;
 
@@ -186,7 +186,7 @@ protected void Open(string filename, byte[] input, int inputLength)
 			this.InputLength = inputLength;
 			this.NextOffset = 0;
 			this.Line = 1;
-			this.Column = 1;
+			this.Utf16Column = 0;
 			FillNextChar();
 			if (this.NextChar == 65279)
 				FillNextChar();
@@ -195,7 +195,7 @@ protected void Open(string filename, byte[] input, int inputLength)
 
 		protected void ReportError(string message)
 		{
-			this.Host.ReportError(this.Filename, this.Line, this.TokenColumn, this.Line, this.Column, message);
+			this.Host.ReportError(this.Filename, this.Line, this.TokenUtf16Column, this.Line, this.Utf16Column, message);
 		}
 
 		int ReadByte()
@@ -261,15 +261,15 @@ protected int ReadChar()
 			switch (c) {
 			case '\t':
 			case ' ':
-				this.Column++;
+				this.Utf16Column++;
 				break;
 			case '\n':
 				this.Line++;
-				this.Column = 1;
+				this.Utf16Column = 0;
 				this.AtLineStart = true;
 				break;
 			default:
-				this.Column++;
+				this.Utf16Column += c < 65536 ? 1 : 2;
 				this.AtLineStart = false;
 				break;
 			}
@@ -501,7 +501,7 @@ FuToken ReadPreToken()
 		{
 			for (;;) {
 				bool atLineStart = this.AtLineStart;
-				this.TokenColumn = this.Column;
+				this.TokenUtf16Column = this.Utf16Column;
 				this.LexemeOffset = this.CharOffset;
 				int c = ReadChar();
 				switch (c) {
@@ -4381,7 +4381,7 @@ public abstract class FuConsoleHost : GenHost
 
 		internal bool HasErrors = false;
 
-		public override void ReportError(string filename, int startLine, int startColumn, int endLine, int endColumn, string message)
+		public override void ReportError(string filename, int startLine, int startUtf16Column, int endLine, int endUtf16Column, string message)
 		{
 			this.HasErrors = true;
 			Console.Error.WriteLine($"{filename}({startLine}): ERROR: {message}");

libfut.hpp

@@ -472,7 +472,7 @@ class FuParserHost
 {
 public:
 	virtual ~FuParserHost() = default;
-	virtual void reportError(std::string_view filename, int startLine, int startColumn, int endLine, int endColumn, std::string_view message) = 0;
+	virtual void reportError(std::string_view filename, int startLine, int startUtf16Column, int endLine, int endUtf16Column, std::string_view message) = 0;
 protected:
 	FuParserHost() = default;
 };
@@ -489,8 +489,6 @@ class FuLexer
 	int charOffset;
 	std::string filename;
 	int line;
-	int column;
-	int tokenColumn;
 	int lexemeOffset;
 	FuToken currentToken;
 	int64_t longValue;
@@ -517,6 +515,8 @@ class FuLexer
 	int nextOffset;
 	int nextChar;
 	FuParserHost * host;
+	int utf16Column;
+	int tokenUtf16Column;
 	std::unordered_set<std::string> preSymbols;
 	bool atLineStart = true;
 	bool lineMode = false;
@@ -1604,7 +1604,7 @@ class GenHost : public FuSemaHost
 class FuConsoleHost : public GenHost
 {
 public:
-	void reportError(std::string_view filename, int startLine, int startColumn, int endLine, int endColumn, std::string_view message) override;
+	void reportError(std::string_view filename, int startLine, int startUtf16Column, int endLine, int endUtf16Column, std::string_view message) override;
 protected:
 	FuConsoleHost() = default;
 public:

libfut.js

@@ -131,8 +131,8 @@ export class FuLexer
 	#host;
 	filename;
 	line;
-	column;
-	tokenColumn;
+	#utf16Column;
+	#tokenUtf16Column;
 	lexemeOffset;
 	currentToken;
 	longValue;
@@ -161,7 +161,7 @@ export class FuLexer
 		this.#inputLength = inputLength;
 		this.#nextOffset = 0;
 		this.line = 1;
-		this.column = 1;
+		this.#utf16Column = 0;
 		this.#fillNextChar();
 		if (this.#nextChar == 65279)
 			this.#fillNextChar();
@@ -170,7 +170,7 @@ export class FuLexer
 
 	reportError(message)
 	{
-		this.#host.reportError(this.filename, this.line, this.tokenColumn, this.line, this.column, message);
+		this.#host.reportError(this.filename, this.line, this.#tokenUtf16Column, this.line, this.#utf16Column, message);
 	}
 
 	#readByte()
@@ -237,15 +237,15 @@ export class FuLexer
 		switch (c) {
 		case 9:
 		case 32:
-			this.column++;
+			this.#utf16Column++;
 			break;
 		case 10:
 			this.line++;
-			this.column = 1;
+			this.#utf16Column = 0;
 			this.#atLineStart = true;
 			break;
 		default:
-			this.column++;
+			this.#utf16Column += c < 65536 ? 1 : 2;
 			this.#atLineStart = false;
 			break;
 		}
@@ -483,7 +483,7 @@ export class FuLexer
 	{
 		for (;;) {
 			let atLineStart = this.#atLineStart;
-			this.tokenColumn = this.column;
+			this.#tokenUtf16Column = this.#utf16Column;
 			this.lexemeOffset = this.charOffset;
 			let c = this.readChar();
 			switch (c) {
@@ -4527,7 +4527,7 @@ export class FuConsoleHost extends GenHost
 {
 	hasErrors = false;
 
-	reportError(filename, startLine, startColumn, endLine, endColumn, message)
+	reportError(filename, startLine, startUtf16Column, endLine, endUtf16Column, message)
 	{
 		this.hasErrors = true;
 		console.error(`${filename}(${startLine}): ERROR: ${message}`);