Merge branch 'master' into mb-simplify-getivar

Author: tekknolagi

.gitignore

@@ -264,14 +264,14 @@ lcov*.info
 /lib/prism/reflection.rb
 /lib/prism/serialize.rb
 /lib/prism/visitor.rb
+/prism/internal/diagnostic.h
 /prism/api_node.c
 /prism/ast.h
 /prism/diagnostic.c
-/prism/diagnostic.h
 /prism/node.c
 /prism/prettyprint.c
 /prism/serialize.c
-/prism/token_type.c
+/prism/tokens.c
 /prism/srcs.mk
 /dump_ast
 

box.c

@@ -22,6 +22,9 @@
 
 #include <stdio.h>
 
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
 #ifdef HAVE_SYS_SENDFILE_H
 # include <sys/sendfile.h>
 #endif

common.mk

@@ -90,34 +90,44 @@ MAKE_ENC      = -f $(ENC_MK) V="$(V)" UNICODE_HDR_DIR="$(UNICODE_HDR_DIR)" \
 
 PRISM_BUILD_DIR = prism
 
-LIBPRISM_OBJS = prism/diagnostic.$(OBJEXT) \
+LIBPRISM_OBJS = \
+		prism/arena.$(OBJEXT) \
+		prism/buffer.$(OBJEXT) \
+		prism/char.$(OBJEXT) \
+		prism/constant_pool.$(OBJEXT) \
+		prism/diagnostic.$(OBJEXT) \
 		prism/encoding.$(OBJEXT) \
+		prism/integer.$(OBJEXT) \
+		prism/json.$(OBJEXT) \
+		prism/line_offset_list.$(OBJEXT) \
+		prism/list.$(OBJEXT) \
+		prism/memchr.$(OBJEXT) \
 		prism/node.$(OBJEXT) \
 		prism/options.$(OBJEXT) \
+		prism/parser.$(OBJEXT) \
 		prism/prettyprint.$(OBJEXT) \
+		prism/prism.$(OBJEXT) \
 		prism/regexp.$(OBJEXT) \
 		prism/serialize.$(OBJEXT) \
+		prism/source.$(OBJEXT) \
 		prism/static_literals.$(OBJEXT) \
-		prism/token_type.$(OBJEXT) \
-		prism/util/pm_arena.$(OBJEXT) \
-		prism/util/pm_buffer.$(OBJEXT) \
-		prism/util/pm_char.$(OBJEXT) \
-		prism/util/pm_constant_pool.$(OBJEXT) \
-		prism/util/pm_integer.$(OBJEXT) \
-		prism/util/pm_line_offset_list.$(OBJEXT) \
-		prism/util/pm_list.$(OBJEXT) \
-		prism/util/pm_memchr.$(OBJEXT) \
-		prism/util/pm_string.$(OBJEXT) \
-		prism/util/pm_strncasecmp.$(OBJEXT) \
-		prism/util/pm_strpbrk.$(OBJEXT) \
-		prism/prism.$(OBJEXT)
+		prism/string_query.$(OBJEXT) \
+		prism/stringy.$(OBJEXT) \
+		prism/strncasecmp.$(OBJEXT) \
+		prism/strpbrk.$(OBJEXT) \
+		prism/tokens.$(OBJEXT)
 
 EXTPRISM_OBJS = prism/api_node.$(OBJEXT) \
 		prism/extension.$(OBJEXT) \
 		prism_init.$(OBJEXT)
 
 PRISM_OBJS = $(LIBPRISM_OBJS) $(EXTPRISM_OBJS)
 
+# Prism objects depend on generated headers that are created from templates.
+# This must be declared here to ensure parallel builds don't compile prism
+# sources before the generated headers exist.
+$(LIBPRISM_OBJS): $(srcdir)/prism/ast.h $(srcdir)/prism/internal/diagnostic.h
+
 COMMONOBJS    = \
 		array.$(OBJEXT) \
 		ast.$(OBJEXT) \

depend

@@ -982,7 +982,7 @@ rb_iseq_new_top(const VALUE ast_value, VALUE name, VALUE path, VALUE realpath, c
 rb_iseq_t *
 pm_iseq_new_top(pm_scope_node_t *node, VALUE name, VALUE path, VALUE realpath, const rb_iseq_t *parent, int *error_state)
 {
-    iseq_new_setup_coverage(path, (int) (node->parser->line_offsets.size - 1));
+    iseq_new_setup_coverage(path, (int) (pm_parser_line_offsets(node->parser)->size - 1));
 
     return pm_iseq_new_with_opt(node, name, path, realpath, 0, parent, 0,
                                 ISEQ_TYPE_TOP, &COMPILE_OPTION_DEFAULT, error_state);
@@ -1006,7 +1006,7 @@ rb_iseq_new_main(const VALUE ast_value, VALUE path, VALUE realpath, const rb_ise
 rb_iseq_t *
 pm_iseq_new_main(pm_scope_node_t *node, VALUE path, VALUE realpath, const rb_iseq_t *parent, int opt, int *error_state)
 {
-    iseq_new_setup_coverage(path, (int) (node->parser->line_offsets.size - 1));
+    iseq_new_setup_coverage(path, (int) (pm_parser_line_offsets(node->parser)->size - 1));
 
     return pm_iseq_new_with_opt(node, rb_fstring_lit("<main>"),
                                 path, realpath, 0,
@@ -1035,7 +1035,7 @@ pm_iseq_new_eval(pm_scope_node_t *node, VALUE name, VALUE path, VALUE realpath,
     if (rb_get_coverage_mode() & COVERAGE_TARGET_EVAL) {
         VALUE coverages = rb_get_coverages();
         if (RTEST(coverages) && RTEST(path) && !RTEST(rb_hash_has_key(coverages, path))) {
-            iseq_setup_coverage(coverages, path, ((int) (node->parser->line_offsets.size - 1)) + first_lineno - 1);
+            iseq_setup_coverage(coverages, path, ((int) (pm_parser_line_offsets(node->parser)->size - 1)) + first_lineno - 1);
         }
     }
 
@@ -1143,10 +1143,11 @@ pm_iseq_new_with_opt(pm_scope_node_t *node, VALUE name, VALUE path, VALUE realpa
     option = &next_option;
 
     pm_location_t *location = &node->base.location;
-    int32_t start_line = node->parser->start_line;
+    int32_t start_line = pm_parser_start_line(node->parser);
+    const pm_line_offset_list_t *line_offsets = pm_parser_line_offsets(node->parser);
 
-    pm_line_column_t start = pm_line_offset_list_line_column(&node->parser->line_offsets, location->start, start_line);
-    pm_line_column_t end = pm_line_offset_list_line_column(&node->parser->line_offsets, location->start + location->length, start_line);
+    pm_line_column_t start = pm_line_offset_list_line_column(line_offsets, location->start, start_line);
+    pm_line_column_t end = pm_line_offset_list_line_column(line_offsets, location->start + location->length, start_line);
 
     rb_code_location_t code_location = (rb_code_location_t) {
         .beg_pos = { .lineno = (int) start.line, .column = (int) start.column },
@@ -1411,19 +1412,20 @@ pm_iseq_compile_with_option(VALUE src, VALUE file, VALUE realpath, VALUE line, V
         src = StringValue(src);
     }
 
-    pm_parse_result_t result = { 0 };
-    pm_options_line_set(&result.options, NUM2INT(line));
-    pm_options_scopes_init(&result.options, 1);
+    pm_parse_result_t result;
+    pm_parse_result_init(&result);
+    pm_options_line_set(result.options, NUM2INT(line));
+    pm_options_scopes_init(result.options, 1);
     result.node.coverage_enabled = 1;
 
     switch (option.frozen_string_literal) {
       case ISEQ_FROZEN_STRING_LITERAL_UNSET:
         break;
       case ISEQ_FROZEN_STRING_LITERAL_DISABLED:
-        pm_options_frozen_string_literal_set(&result.options, false);
+        pm_options_frozen_string_literal_set(result.options, false);
         break;
       case ISEQ_FROZEN_STRING_LITERAL_ENABLED:
-        pm_options_frozen_string_literal_set(&result.options, true);
+        pm_options_frozen_string_literal_set(result.options, true);
         break;
       default:
         rb_bug("pm_iseq_compile_with_option: invalid frozen_string_literal=%d", option.frozen_string_literal);
@@ -1783,6 +1785,8 @@ iseqw_s_compile_prism(int argc, VALUE *argv, VALUE self)
     return iseqw_s_compile_parser(argc, argv, self, true);
 }
 
+static VALUE iseqw_s_compile_file_prism(int argc, VALUE *argv, VALUE self);
+
 /*
  *  call-seq:
  *      InstructionSequence.compile_file(file[, options]) -> iseq
@@ -1806,6 +1810,10 @@ iseqw_s_compile_prism(int argc, VALUE *argv, VALUE self)
 static VALUE
 iseqw_s_compile_file(int argc, VALUE *argv, VALUE self)
 {
+    if (rb_ruby_prism_p()) {
+        return iseqw_s_compile_file_prism(argc, argv, self);
+    }
+
     VALUE file, opt = Qnil;
     VALUE parser, f, exc = Qnil, ret;
     rb_ast_t *ast;
@@ -1892,8 +1900,8 @@ iseqw_s_compile_file_prism(int argc, VALUE *argv, VALUE self)
     rb_execution_context_t *ec = GET_EC();
     VALUE v = rb_vm_push_frame_fname(ec, file);
 
-    pm_parse_result_t result = { 0 };
-    result.options.line = 1;
+    pm_parse_result_t result;
+    pm_parse_result_init(&result);
     result.node.coverage_enabled = 1;
 
     VALUE script_lines;

iseq.c

@@ -20,6 +20,10 @@
 #include "internal/imemo.h"
 #include "ruby/internal/core/rtypeddata.h"
 
+#ifndef _WIN32
+#include <sys/mman.h>
+#endif
+
 enum jit_bindgen_constants {
     // Field offsets for the RObject struct
     ROBJECT_OFFSET_AS_HEAP_FIELDS = offsetof(struct RObject, as.heap.fields),