JavetExecutionException is reporting raw JS line numbers (does not use sourcemap)

[lukehutch]

I am using swc4j to transpile TS to JS, then I am executing the JS in Javet.

When a JavetExecutionException e is thrown, raw JS line numbers are reported, rather than the correct TS line numbers.

var params = e.getParameters();
var lineNumber = params.get("lineNumber");
var startColumn = params.get("startColumn");
var sourceLine = params.get("sourceLine");

This shows the correct TS line, but the line and column numbers are for JS, not TS.

The JS source code does correctly include a sourcemap in the last line. But it is not being used here.

I looked in swc4j and couldn't see a way to look up the correct line numbers using the source map. But anyway, Javet should be doing that work here automatically, whenever a sourcemap is present.

[caoccao]

Javet doesn't process source map. For now, you'll have to do that by yourself. But, that could be a feature in the future.

[lukehutch]

Please consider this as a feature request then.. Source map processing is not easy, especially if you're trying to do it in Java (which doesn't seem to have mature libraries to do it).

[lukehutch]

I asked in swc if they could provide a utility function for translating sourcemaps (still working on convincing them it would be useful):

swc-project/swc#10166

But then I found this Java library that can parse sourcemaps:

https://github.com/atlassian/sourcemap/tree/master

I assume that stacktraces generated in V8 will have the original TS line numbers, if an exception is thrown in JS and the last line of the JS source included a sourcemap. So the only thing that needs to include the correct line number would be a JavetException, which as I understand it is mainly thrown if lexing or parsing fails.

So in those cases, Javet could use this Atlassian library to parse the sourcemap (if present) and convert the line numbers.

[caoccao]

Thank you for the info. Unfortunately, Javet cannot reference any other libraries. It has to be an in-house solution.

[lukehutch]

Will you accept the incorporation of Apache-licensed code? Or would it have to be rewritten?

[caoccao]

The spec is open. I'm in favor of a standalone implementation.

[lukehutch]

I built this using Claude, so it should be fully copyright-free, and you are free to use it, change it, or license it however you want. It works for me.

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;

import com.fasterxml.jackson.jr.ob.JSON;

public class SourceMap {
    /** A parsed source map mapping. */
    public static class MappingEntry {
        public final int generatedLine;
        public final int generatedColumn;
        public final int sourceIndex;
        public final int sourceLine;
        public final int sourceColumn;
        public final int nameIndex;

        public MappingEntry(int generatedLine, int generatedColumn, int sourceIndex, int sourceLine, int sourceColumn,
                int nameIndex) {
            this.generatedLine = generatedLine;
            this.generatedColumn = generatedColumn;
            this.sourceIndex = sourceIndex;
            this.sourceLine = sourceLine;
            this.sourceColumn = sourceColumn;
            this.nameIndex = nameIndex;
        }

        @Override
        public String toString() {
            StringBuilder sb = new StringBuilder();
            sb.append("Generated: (").append(generatedLine).append(":").append(generatedColumn).append(")");
            if (sourceIndex >= 0) {
                sb.append(" -> Source[").append(sourceIndex).append("]: (").append(sourceLine).append(":")
                        .append(sourceColumn).append(")");
                if (nameIndex >= 0) {
                    sb.append(" Name[").append(nameIndex).append("]");
                }
            }
            return sb.toString();
        }
    }

    // Base64 character set for VLQ encoding used by source maps
    // Different from standard Base64 encoding
    private static final String BASE64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";

    // Constants for VLQ encoding
    private static final int VLQ_BASE_SHIFT = 5;
    private static final int VLQ_BASE = 1 << VLQ_BASE_SHIFT;
    private static final int VLQ_BASE_MASK = VLQ_BASE - 1;
    private static final int VLQ_CONTINUATION_BIT = VLQ_BASE;

    // Source mapping URL regexp
    public static final Pattern SOURCE_MAPPING_URL_PATTERN = Pattern.compile("^//[#@]\\s+sourceMappingURL=(.*)$",
            Pattern.MULTILINE);

    // The parsed mappings
    private List<MappingEntry> mappings;

    /**
     * Decodes a VLQ encoded value starting at the specified index in the string.
     * 
     * @param str   The string containing the VLQ encoded value
     * @param index The starting index for decoding
     * @return An array where [0] is the decoded value and [1] is the new index
     */
    private static int[] decode(String str, int index) {
        int result = 0;
        int shift = 0;
        int continuation;
        int digit;

        do {
            if (index >= str.length()) {
                throw new IllegalArgumentException("Expected more digits in VLQ value");
            }

            char c = str.charAt(index++);
            int digitIndex = BASE64_CHARS.indexOf(c);
            if (digitIndex == -1) {
                throw new IllegalArgumentException("Invalid Base64 character: " + c);
            }

            continuation = digitIndex & VLQ_CONTINUATION_BIT;
            digit = digitIndex & VLQ_BASE_MASK;

            result += (digit << shift);
            shift += VLQ_BASE_SHIFT;
        } while (continuation != 0);

        // Sign bit is encoded in the least significant bit of the value
        boolean negate = (result & 1) == 1;
        result >>= 1;

        if (negate) {
            result = -result;
        }

        return new int[] { result, index };
    }

    /**
     * Decodes a source map mappings string into a list of MappingEntry objects.
     * 
     * @param mappingsStr The mappings string from a source map
     * @return A list of MappingEntry objects
     */
    public SourceMap(String mappingsStr) {
        this.mappings = new ArrayList<>();

        int lineNumber = 0;
        int index = 0;
        int generatedColumn = 0;
        int sourceFileIndex = 0;
        int sourceLine = 0;
        int sourceColumn = 0;
        int nameIndex = 0;

        while (index < mappingsStr.length()) {
            if (mappingsStr.charAt(index) == ';') {
                lineNumber++;
                generatedColumn = 0;
                index++;
                continue;
            } else if (mappingsStr.charAt(index) == ',') {
                index++;
                continue;
            }

            // Read generated column offset
            int[] decoded = decode(mappingsStr, index);
            generatedColumn += decoded[0];
            index = decoded[1];

            int currentSourceFileIndex = -1;
            int currentSourceLine = -1;
            int currentSourceColumn = -1;
            int currentNameIndex = -1;

            // Check if we have a source and original position
            if (index < mappingsStr.length() && mappingsStr.charAt(index) != ',' && mappingsStr.charAt(index) != ';') {
                // Read source file index
                decoded = decode(mappingsStr, index);
                sourceFileIndex += decoded[0];
                index = decoded[1];
                currentSourceFileIndex = sourceFileIndex;

                // Read source line
                decoded = decode(mappingsStr, index);
                sourceLine += decoded[0];
                index = decoded[1];
                currentSourceLine = sourceLine;

                // Read source column
                decoded = decode(mappingsStr, index);
                sourceColumn += decoded[0];
                index = decoded[1];
                currentSourceColumn = sourceColumn;

                // Check if we have a name
                if (index < mappingsStr.length() && mappingsStr.charAt(index) != ','
                        && mappingsStr.charAt(index) != ';') {
                    // Read name index
                    decoded = decode(mappingsStr, index);
                    nameIndex += decoded[0];
                    index = decoded[1];
                    currentNameIndex = nameIndex;
                }
            }

            // Adjust column numbers and line numbers to be 1-based
            mappings.add(new MappingEntry(lineNumber + 1, generatedColumn + 1, currentSourceFileIndex,
                    currentSourceLine + 1, currentSourceColumn + 1, currentNameIndex + 1));
        }
    }

    /**
     * Extracts and decodes a source map from JS source.
     * 
     * @param jsSource The JavaScript source code.
     * @return A list of MappingEntry objects representing the source map.
     * @throws IllegalArgumentException if there is no sourceMappingURL found in the JS source, or if it points to an
     *                                  external file, or the source map cannot be parsed.
     */
    public static SourceMap decode(String jsSource) throws ParseException {
        // Extract sourceMappingURL from the JS file
        var matcher = SOURCE_MAPPING_URL_PATTERN.matcher(jsSource);
        if (!matcher.find()) {
            throw new IllegalArgumentException("No sourceMappingURL found in JS source");
        }
        var sourceMappingURL = matcher.group(1);

        // Handle different formats of sourceMappingURL
        if (sourceMappingURL.startsWith("data:application/json;base64,")) {
            // Inline base64-encoded source map
            var base64Data = sourceMappingURL.substring("data:application/json;base64,".length());
            var sourceMapJson = new String(Base64.getDecoder().decode(base64Data), StandardCharsets.UTF_8);
            Map<String, Object> sourceMap;
            try {
                // Parse the source map JSON
                sourceMap = JSON.std.mapFrom(sourceMapJson);
            } catch (IOException e) {
                throw new IllegalArgumentException("Failed to parse source map JSON");
            }
            // Extract the mappings
            var mappingsStr = sourceMap.get("mappings");
            if (mappingsStr == null) {
                throw new IllegalArgumentException("Invalid source map: missing 'mappings' field");
            }
            if (mappingsStr instanceof String mStr) {
                // Parse the mappings
                return new SourceMap(mStr);
            } else {
                throw new IllegalArgumentException("Invalid source map: 'mappings' field is not a string");
            }
        } else {
            // External source map file
            throw new IllegalArgumentException("External source map files not supported");
        }
    }

    /** Finds the original position for a given position in the generated code. */
    public MappingEntry findOriginalPosition(int generatedLineNumber, int generatedStartColumn) {
        MappingEntry bestMatch = null;
        var bestMatchLineDifference = Integer.MAX_VALUE;
        var bestMatchColumnDifference = Integer.MAX_VALUE;
        for (var mapping : mappings) {
            // Find the mapping that is closest to the requested position
            var lineDifference = Math.abs(generatedLineNumber - mapping.generatedLine);
            var columnDifference = Math.abs(generatedStartColumn - mapping.generatedColumn);
            if (lineDifference < bestMatchLineDifference
                    || (lineDifference == bestMatchLineDifference && columnDifference < bestMatchColumnDifference)) {
                bestMatch = mapping;
                bestMatchLineDifference = lineDifference;
                bestMatchColumnDifference = columnDifference;
            }
        }
        return bestMatch;
    }
}

[lukehutch]

I'm using it as follows: I am rewriting the line numbers in the JavetException, but I am also rewriting any line numbers in the Javascript exception, so that I can do catch (e) { throw new Error(e.stack) } to capture the JS stacktrace in the message:

    private static final String V8_RESOURCE_NAME = "<jsSource>";
    private static final Pattern SOURCE_LOCATION_PATTERN = Pattern.compile(V8_RESOURCE_NAME + ":(\\d+):(\\d+)");

// ...
           } catch (JavetException javetException) {
                // Add the line and column number to the exception message if it is available
                var params = javetException.getParameters();
                var sourceLine = params.get("sourceLine");
                var lineNumber = params.get("lineNumber");
                var startColumn = params.get("startColumn");
                var sourceLocation = "<jsSource>:" + lineNumber + ":" + startColumn;

                // Parse JS source map if present
                SourceMap sourceMap = null;
                try {
                    sourceMap = SourceMap.decode(jsSource);
                } catch (ParseException ex) {
                    // Fall through
                }
                if (sourceMap != null) {
                    // Look up TS location of exception from JS location using source map
                    if (lineNumber instanceof Integer && startColumn instanceof Integer) {
                        var mappingEntry = sourceMap.findOriginalPosition((Integer) lineNumber,
                                (Integer) startColumn);
                        if (mappingEntry != null) {
                            // Prepend TS source location to JS source location
                            sourceLocation = "<tsSource>" + ":" + mappingEntry.sourceLine + ":"
                                    + mappingEntry.sourceColumn + " / " + sourceLocation;
                        }
                    }
                    // If an exception was thrown in Javascript, then add the TS source location to each mention of
                    // JS source location, by finding source locations in the stacktrace in the exception message
                    var msg = javetException.getMessage();
                    var matcher = SOURCE_LOCATION_PATTERN.matcher(msg);
                    var transformedMsg = new StringBuilder();
                    int prevEnd = 0;
                    while (matcher.find()) {
                        try {
                            int lineNum = Integer.parseInt(matcher.group(1));
                            int columnNum = Integer.parseInt(matcher.group(2));
                            // Look up TS location of exception from JS location using source map,
                            // and prepend it to the JS location
                            var mappingEntry = sourceMap.findOriginalPosition(lineNum, columnNum);
                            if (mappingEntry != null) {
                                String tsLocation = "<tsSource>:" + mappingEntry.sourceLine + ":"
                                        + mappingEntry.sourceColumn;
                                String replacement = tsLocation + " / " + matcher.group();
                                transformedMsg.append(msg.substring(prevEnd, matcher.start()));
                                transformedMsg.append(replacement);
                                prevEnd = matcher.end();
                            }
                        } catch (NumberFormatException e2) {
                            // Skip this source location if line or column numbers can't be parsed
                            transformedMsg.append(msg.substring(prevEnd, matcher.end()));
                        }
                    }
                    transformedMsg.append(msg.substring(prevEnd));
                    throw new CodeExecutionException(tsSource, jsSourceWithoutSourceMap.get(), jsExecutionResult,
                            "Error in generated code at " + sourceLocation + ": " + sourceLine + "\nCaused by "
                                    + transformedMsg,
                            DEFAULT_USER_FRIENDLY_MESSAGE, javetException.getCause());
                } else {
                    throw new CodeExecutionException(tsSource, jsSourceWithoutSourceMap.get(), jsExecutionResult,
                            "Error in generated code at " + sourceLocation + ": " + sourceLine,
                            DEFAULT_USER_FRIENDLY_MESSAGE, javetException);
                }
                } else {
                    throw new CodeExecutionException(tsSource, jsSourceWithoutSourceMap.get(), jsExecutionResult,
                            "Exception while running generated code", DEFAULT_USER_FRIENDLY_MESSAGE, cause);
                }