Loader.java.erb

<%- string_type = Prism::Template::JAVA_STRING_TYPE -%>
package org.ruby_lang.prism;

import org.ruby_lang.prism.Nodes;

import java.lang.Short;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.Locale;

// GENERATED BY <%= File.basename(__FILE__) %>
// @formatter:off
public class Loader {

    public static ParseResult load(byte[] serialized, byte[] sourceBytes) {
        return new Loader(serialized, sourceBytes).load();
    }

    // Overridable methods

    public Charset getEncodingCharset(String encodingName) {
        encodingName = encodingName.toLowerCase(Locale.ROOT);
        if (encodingName.equals("ascii-8bit")) {
            return StandardCharsets.US_ASCII;
        }
        return Charset.forName(encodingName);
    }

    public <%= string_type %> bytesToName(byte[] bytes) {
        <%- if string_type == "String" -%>
        return new String(bytes, encodingCharset).intern();
        <%- else -%>
        return null; // Must be implemented by subclassing Loader
        <%- end -%>
    }

    private static final class ConstantPool {

        private final Loader loader;
        private final int bufferOffset;
        private final <%= string_type %>[] cache;

        ConstantPool(Loader loader, int bufferOffset, int length) {
            this.loader = loader;
            this.bufferOffset = bufferOffset;
            cache = new <%= string_type %>[length];
        }

        <%= string_type %> get(ByteBuffer buffer, int oneBasedIndex) {
            int index = oneBasedIndex - 1;
            <%= string_type %> constant = cache[index];

            if (constant == null) {
                int offset = bufferOffset + index * 8;
                int start = buffer.getInt(offset);
                int length = buffer.getInt(offset + 4);

                byte[] bytes = new byte[length];
                buffer.get(start, bytes);

                constant = loader.bytesToName(bytes);
                cache[index] = constant;
            }

            return constant;
        }

    }

    private final ByteBuffer buffer;
    private final Nodes.Source source;
    protected String encodingName;
    <%- if string_type == "String" -%>
    private Charset encodingCharset;
    <%- end -%>
    private ConstantPool constantPool;

    protected Loader(byte[] serialized, byte[] sourceBytes) {
        this.buffer = ByteBuffer.wrap(serialized).order(ByteOrder.nativeOrder());
        this.source = new Nodes.Source(sourceBytes);
    }

    protected ParseResult load() {
        expect((byte) 'P', "incorrect prism header");
        expect((byte) 'R', "incorrect prism header");
        expect((byte) 'I', "incorrect prism header");
        expect((byte) 'S', "incorrect prism header");
        expect((byte) 'M', "incorrect prism header");

        expect((byte) 1, "prism major version does not match");
        expect((byte) 9, "prism minor version does not match");
        expect((byte) 0, "prism patch version does not match");

        expect((byte) 1, "Loader.java requires no location fields in the serialized output");

        // This loads the name of the encoding.
        int encodingLength = loadVarUInt();
        byte[] encodingNameBytes = new byte[encodingLength];
        buffer.get(encodingNameBytes);
        this.encodingName = new String(encodingNameBytes, StandardCharsets.US_ASCII);
        <%- if string_type == "String" -%>
        this.encodingCharset = getEncodingCharset(this.encodingName);
        <%- end -%>

        source.setStartLine(loadVarSInt());
        source.setLineOffsets(loadLineOffsets());

        ParseResult.MagicComment[] magicComments = loadMagicComments();
        Nodes.Location dataLocation = loadOptionalLocation();
        ParseResult.Error[] errors = loadErrors();
        ParseResult.Warning[] warnings = loadWarnings();

        int constantPoolBufferOffset = buffer.getInt();
        int constantPoolLength = loadVarUInt();
        this.constantPool = new ConstantPool(this, constantPoolBufferOffset, constantPoolLength);

        Nodes.Node node;
        if (errors.length == 0) {
            node = loadNode();

            int left = constantPoolBufferOffset - buffer.position();
            if (left != 0) {
                throw new Error("Expected to consume all bytes while deserializing but there were " + left + " bytes left");
            }

            boolean[] newlineMarked = new boolean[1 + source.getLineCount()];
            MarkNewlinesVisitor visitor = new MarkNewlinesVisitor(source, newlineMarked);
            node.accept(visitor);
        } else {
            node = null;
        }

        return new ParseResult(node, magicComments, dataLocation, errors, warnings, source);
    }

    private byte[] loadString() {
        int length = loadVarUInt();
        byte[] bytes = new byte[length];
        buffer.get(bytes);
        return bytes;
    }

    private int[] loadLineOffsets() {
        int count = loadVarUInt();
        int[] lineOffsets = new int[count];
        for (int i = 0; i < count; i++) {
            lineOffsets[i] = loadVarUInt();
        }
        return lineOffsets;
    }

    private ParseResult.MagicComment[] loadMagicComments() {
        int count = loadVarUInt();
        ParseResult.MagicComment[] magicComments = new ParseResult.MagicComment[count];

        for (int i = 0; i < count; i++) {
            Nodes.Location keyLocation = loadLocation();
            Nodes.Location valueLocation = loadLocation();

            ParseResult.MagicComment magicComment = new ParseResult.MagicComment(keyLocation, valueLocation);
            magicComments[i] = magicComment;
        }

        return magicComments;
    }

    private ParseResult.Error[] loadErrors() {
        int count = loadVarUInt();
        ParseResult.Error[] errors = new ParseResult.Error[count];

        // error messages only contain ASCII characters
        for (int i = 0; i < count; i++) {
            Nodes.ErrorType type = Nodes.ERROR_TYPES[loadVarUInt()];
            byte[] bytes = loadString();
            String message = new String(bytes, StandardCharsets.US_ASCII);
            Nodes.Location location = loadLocation();
            ParseResult.ErrorLevel level = ParseResult.ERROR_LEVELS[buffer.get()];

            ParseResult.Error error = new ParseResult.Error(type, message, location, level);
            errors[i] = error;
        }

        return errors;
    }

    private ParseResult.Warning[] loadWarnings() {
        int count = loadVarUInt();
        ParseResult.Warning[] warnings = new ParseResult.Warning[count];

        // warning messages only contain ASCII characters
        for (int i = 0; i < count; i++) {
            Nodes.WarningType type = Nodes.WARNING_TYPES[loadVarUInt() - <%= errors.length %>];
            byte[] bytes = loadString();
            String message = new String(bytes, StandardCharsets.US_ASCII);
            Nodes.Location location = loadLocation();
            ParseResult.WarningLevel level = ParseResult.WARNING_LEVELS[buffer.get()];

            ParseResult.Warning warning = new ParseResult.Warning(type, message, location, level);
            warnings[i] = warning;
        }

        return warnings;
    }

    private Nodes.Node loadOptionalNode() {
        if (buffer.get(buffer.position()) != 0) {
            return loadNode();
        } else {
            buffer.position(buffer.position() + 1); // continue after the 0 byte
            return null;
        }
    }

    private <%= string_type %> loadConstant() {
        return constantPool.get(buffer, loadVarUInt());
    }

    private <%= string_type %> loadOptionalConstant() {
        if (buffer.get(buffer.position()) != 0) {
            return loadConstant();
        } else {
            buffer.position(buffer.position() + 1); // continue after the 0 byte
            return null;
        }
    }

    private <%= string_type %>[] loadConstants() {
        int length = loadVarUInt();
        if (length == 0) {
            return Nodes.EMPTY_STRING_ARRAY;
        }
        <%= string_type %>[] constants = new <%= string_type %>[length];
        for (int i = 0; i < length; i++) {
            constants[i] = constantPool.get(buffer, loadVarUInt());
        }
        return constants;
    }

    private Nodes.Location loadLocation() {
        return new Nodes.Location(loadVarUInt(), loadVarUInt());
    }

    private Nodes.Location loadOptionalLocation() {
        if (buffer.get() != 0) {
            return loadLocation();
        } else {
            return null;
        }
    }

    // From https://github.com/protocolbuffers/protobuf/blob/v23.1/java/core/src/main/java/com/google/protobuf/BinaryReader.java#L1507
    private int loadVarUInt() {
        int x;
        if ((x = buffer.get()) >= 0) {
            return x;
        } else if ((x ^= (buffer.get() << 7)) < 0) {
            x ^= (~0 << 7);
        } else if ((x ^= (buffer.get() << 14)) >= 0) {
            x ^= (~0 << 7) ^ (~0 << 14);
        } else if ((x ^= (buffer.get() << 21)) < 0) {
            x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21);
        } else {
            x ^= buffer.get() << 28;
            x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28);
        }
        return x;
    }

    // From https://github.com/protocolbuffers/protobuf/blob/v25.1/java/core/src/main/java/com/google/protobuf/CodedInputStream.java#L508-L510
    private int loadVarSInt() {
        int x = loadVarUInt();
        return (x >>> 1) ^ (-(x & 1));
    }

    private short loadFlags() {
        int flags = loadVarUInt();
        assert flags >= 0 && flags <= Short.MAX_VALUE;
        return (short) flags;
    }

    private static final BigInteger UNSIGNED_LONG_MASK = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE);

    private Object loadInteger() {
        boolean negative = buffer.get() != 0;

        int wordsLength = loadVarUInt();
        assert wordsLength > 0;

        // Load the first word. If it's the only word, then return an int if it
        // fits into one and a long otherwise.
        int firstWord = loadVarUInt();
        if (wordsLength == 1) {
            if (firstWord < 0) {
                if (negative && firstWord == Integer.MIN_VALUE) {
                    return Integer.MIN_VALUE;
                }

                long words = Integer.toUnsignedLong(firstWord);
                return negative ? -words : words;
            }
            return negative ? -firstWord : firstWord;
        }

        // Load the second word. If there are only two words, then return a long
        // if it fits into one and a BigInteger otherwise.
        int secondWord = loadVarUInt();
        if (wordsLength == 2) {
            long words = (((long) secondWord) << 32L) | Integer.toUnsignedLong(firstWord);
            if (words < 0L) {
                if (negative && words == Long.MIN_VALUE) {
                    return Long.MIN_VALUE;
                }

                BigInteger result = BigInteger.valueOf(words).and(UNSIGNED_LONG_MASK);
                return negative ? result.negate() : result;
            }
            return negative ? -words : words;
        }

        // Otherwise, load the remaining words and return a BigInt.
        BigInteger result = BigInteger.valueOf(Integer.toUnsignedLong(firstWord));
        result = result.or(BigInteger.valueOf(Integer.toUnsignedLong(secondWord)).shiftLeft(32));

        for (int wordsIndex = 2; wordsIndex < wordsLength; wordsIndex++) {
            result = result.or(BigInteger.valueOf(Integer.toUnsignedLong(loadVarUInt())).shiftLeft(wordsIndex * 32));
        }

        return negative ? result.negate() : result;
    }

    private Nodes.Node loadNode() {
        int type = buffer.get() & 0xFF;
        <%- if Prism::Template::INCLUDE_NODE_ID -%>
        int nodeId = loadVarUInt();
        <%- end -%>
        int startOffset = loadVarUInt();
        int length = loadVarUInt();

        switch (type) {
            <%- array_types = [] -%>
            <%- nodes.each_with_index do |node, index| -%>
            case <%= index + 1 %>:
            <%-
            params = []
            params << "nodeId" if Prism::Template::INCLUDE_NODE_ID
            params << "startOffset" << "length"
            params << "buffer.getInt()" if node.needs_serialized_length?
            params << "loadFlags()" if node.flags
            params.concat node.semantic_fields.map { |field|
              case field
              when Prism::Template::NodeField then "#{field.java_cast}loadNode()"
              when Prism::Template::OptionalNodeField then "#{field.java_cast}loadOptionalNode()"
              when Prism::Template::StringField then "loadString()"
              when Prism::Template::NodeListField then
                element_type = field.java_type.sub('[]', '')
                array_types << element_type
                "load#{element_type}s()"
              when Prism::Template::ConstantField then "loadConstant()"
              when Prism::Template::OptionalConstantField then "loadOptionalConstant()"
              when Prism::Template::ConstantListField then "loadConstants()"
              when Prism::Template::LocationField then "loadLocation()"
              when Prism::Template::OptionalLocationField then "loadOptionalLocation()"
              when Prism::Template::UInt8Field then "buffer.get()"
              when Prism::Template::UInt32Field then "loadVarUInt()"
              when Prism::Template::IntegerField then "loadInteger()"
              when Prism::Template::DoubleField then "buffer.getDouble()"
              else raise
              end
            }
            -%>
                return new Nodes.<%= node.name %>(<%= params.join(", ") -%>);
            <%- end -%>
            default:
                throw new Error("Unknown node type: " + type);
        }
    }
    <%- array_types.uniq.each do |type| -%>

    private static final Nodes.<%= type %>[] EMPTY_<%= type %>_ARRAY = {};

    private Nodes.<%= type %>[] load<%= type %>s() {
        int length = loadVarUInt();
        if (length == 0) {
            return EMPTY_<%= type %>_ARRAY;
        }
        Nodes.<%= type %>[] nodes = new Nodes.<%= type %>[length];
        for (int i = 0; i < length; i++) {
            <%- if type == 'Node' -%>
            nodes[i] = loadNode();
            <%- else -%>
            nodes[i] = (Nodes.<%= type %>) loadNode();
            <%- end -%>
        }
        return nodes;
    }
    <%- end -%>

    private void expect(byte value, String error) {
        byte b = buffer.get();
        if (b != value) {
            throw new Error("Deserialization error: " + error + " (expected " + value + " but was " + b + " at position " + buffer.position() + ")");
        }
    }

}
// @formatter:on