Merge branch 'improve-source-scanning-for-presym' of https://github.com/shuujii/mruby into shuujii-improve-source-scanning-for-presym

lib/mruby/build.rb

@@ -5,6 +5,7 @@ require "mruby/build/command"
 module MRuby
   autoload :Gem, "mruby/gem"
   autoload :Lockfile, "mruby/lockfile"
+  autoload :Presym, "mruby/presym"
 
   class << self
     def targets
@@ -51,6 +52,7 @@ module MRuby
   class Build
     class << self
       attr_accessor :current
+
       def mruby_config_path
         path = ENV['MRUBY_CONFIG'] || ENV['CONFIG']
         if path.nil? || path.empty?
@@ -61,11 +63,16 @@ module MRuby
         end
         path
       end
+
+      def install_dir
+        @install_dir ||= ENV['INSTALL_DIR'] || "#{MRUBY_ROOT}/bin"
+      end
     end
+
     include Rake::DSL
     include LoadGems
     attr_accessor :name, :bins, :exts, :file_separator, :build_dir, :gem_clone_dir
-    attr_reader :libmruby_core_objs, :libmruby_objs, :gems, :toolchains, :gem_dir_to_repo_url
+    attr_reader :products, :libmruby_core_objs, :libmruby_objs, :gems, :toolchains, :presym, :mrbc_build, :gem_dir_to_repo_url
 
     alias libmruby libmruby_objs
 
@@ -73,16 +80,16 @@ module MRuby
     COMMANDS = COMPILERS + %w(linker archiver yacc gperf git exts mrbc)
     attr_block MRuby::Build::COMMANDS
 
-    Exts = Struct.new(:object, :executable, :library)
+    Exts = Struct.new(:object, :executable, :library, :preprocessed)
 
-    def initialize(name='host', build_dir=nil, &block)
+    def initialize(name='host', build_dir=nil, internal: false, &block)
       @name = name.to_s
 
       unless current = MRuby.targets[@name]
         if ENV['OS'] == 'Windows_NT'
-          @exts = Exts.new('.o', '.exe', '.a')
+          @exts = Exts.new('.o', '.exe', '.a', '.i')
         else
-          @exts = Exts.new('.o', '', '.a')
+          @exts = Exts.new('.o', '', '.a', '.i')
         end
 
         build_dir = build_dir || ENV['MRUBY_BUILD_DIR'] || "#{MRUBY_ROOT}/build"
@@ -101,6 +108,7 @@ module MRuby
         @git = Command::Git.new(self)
         @mrbc = Command::Mrbc.new(self)
 
+        @products = []
         @bins = []
         @gems = MRuby::Gem::List.new
         @libmruby_core_objs = []
@@ -113,6 +121,9 @@ module MRuby
         @enable_bintest = false
         @enable_test = false
         @enable_lock = true
+        @enable_presym = true
+        @mrbcfile_external = false
+        @internal = internal
         @toolchains = []
         @gem_dir_to_repo_url = {}
 
@@ -121,7 +132,14 @@ module MRuby
 
       MRuby::Build.current = current
       current.instance_eval(&block)
-      current.build_mrbc_exec if current.libmruby_enabled? && @name == "host"
+      if current.libmruby_enabled? && !current.mrbcfile_external?
+        if current.presym_enabled?
+          current.create_mrbc_build if current.host? || current.gems["mruby-bin-mrbc"]
+        elsif current.host?
+          current.build_mrbc_exec
+        end
+      end
+      current.presym = Presym.new(current) if current.presym_enabled?
       current.build_mrbtest if current.test_enabled?
     end
 
@@ -149,6 +167,17 @@ module MRuby
       @enable_debug = true
     end
 
+    def presym_enabled?
+      @enable_presym
+    end
+
+    def disable_presym
+      if @enable_presym
+        @enable_presym = false
+        compilers.each{|c| c.defines << "MRB_NO_PRESYM"}
+      end
+    end
+
     def disable_lock
       @enable_lock = false
     end
@@ -200,8 +229,29 @@ module MRuby
       @cxx_abi_enabled = true
     end
 
-    def compile_as_cxx src, cxx_src, obj = nil, includes = []
-      obj = objfile(cxx_src) if obj.nil?
+    def compile_as_cxx(src, cxx_src = nil, obj = nil, includes = [])
+      #
+      # If `cxx_src` is specified, this method behaves the same as before as
+      # compatibility mode, but `.d` file is not read.
+      #
+      # If `cxx_src` is omitted, `.d` file is read by using mruby standard
+      # Rake rule (C++ source name is also changed).
+      #
+      if cxx_src
+        obj ||= cxx_src + @exts.object
+        dsts = [obj]
+        dsts << (cxx_src + @exts.preprocessed) if presym_enabled?
+        defines = []
+        include_paths = ["#{MRUBY_ROOT}/src", *includes]
+        dsts.each do |dst|
+          file dst => cxx_src do |t|
+            cxx.run t.name, t.prerequisites.first, defines, include_paths
+          end
+        end
+      else
+        cxx_src = "#{build_dir}/#{src.relative_path})".ext << "-cxx.cxx"
+        obj = cxx_src.ext(@exts.object)
+      end
 
       file cxx_src => [src, __FILE__] do |t|
         mkdir_p File.dirname t.name
@@ -219,10 +269,6 @@ extern "C" {
 EOS
       end
 
-      file obj => cxx_src do |t|
-        cxx.run t.name, t.prerequisites.first, [], ["#{MRUBY_ROOT}/src"] + includes
-      end
-
       obj
     end
 
@@ -263,11 +309,11 @@ EOS
     end
 
     def build_mrbtest
-      gem :core => 'mruby-test'
+      gem :core => 'mruby-test' unless @gems['mruby-test']
     end
 
     def build_mrbc_exec
-      gem :core => 'mruby-bin-mrbc'
+      gem :core => 'mruby-bin-mrbc' unless @gems['mruby-bin-mrbc']
     end
 
     def locks
@@ -278,10 +324,23 @@ EOS
       return @mrbcfile if @mrbcfile
 
       gem_name = "mruby-bin-mrbc"
-      gem = gems[gem_name] || MRuby.targets["host"].gems[gem_name]
+      gem = @gems[gem_name]
+      gem ||= (host = MRuby.targets["host"]) && host.gems[gem_name]
+      unless gem
+        fail "external mrbc or mruby-bin-mrbc gem in current('#{@name}') or 'host' build is required"
+      end
       @mrbcfile = exefile("#{gem.build.build_dir}/bin/mrbc")
     end
 
+    def mrbcfile=(path)
+      @mrbcfile = path
+      @mrbcfile_external = true
+    end
+
+    def mrbcfile_external?
+      @mrbcfile_external
+    end
+
     def compilers
       COMPILERS.map do |c|
         instance_variable_get("@#{c}")
@@ -289,7 +348,7 @@ EOS
     end
 
     def define_rules
-      use_mrdb = @gems.find{|g| g.name == "mruby-bin-debugger"}
+      use_mrdb = @gems["mruby-bin-debugger"]
       compilers.each do |compiler|
         if respond_to?(:enable_gems?) && enable_gems?
           compiler.defines -= %w(MRB_NO_GEMS)
@@ -298,7 +357,10 @@ EOS
         end
         compiler.defines |= %w(MRB_USE_DEBUG_HOOK) if use_mrdb
       end
-      cc.define_rules(build_dir, MRUBY_ROOT)
+      [@cc, *(@cxx if cxx_exception_enabled?)].each do |compiler|
+        compiler.define_rules(@build_dir, MRUBY_ROOT, @exts.object)
+        compiler.define_rules(@build_dir, MRUBY_ROOT, @exts.preprocessed) if presym_enabled?
+      end
     end
 
     def filename(name)
@@ -359,7 +421,8 @@ EOS
       puts ">>> Bintest #{name} <<<"
       targets = @gems.select { |v| File.directory? "#{v.dir}/bintest" }.map { |v| filename v.dir }
       targets << filename(".") if File.directory? "./bintest"
-      env = {"BUILD_DIR" => @build_dir}
+      mrbc = @gems["mruby-bin-mrbc"] ? exefile("#{@build_dir}/bin/mrbc") : mrbcfile
+      env = {"BUILD_DIR" => @build_dir, "MRBCFILE" => mrbc}
       sh env, "ruby test/bintest.rb#{verbose_flag} #{targets.join ' '}"
     end
 
@@ -393,6 +456,41 @@ EOS
     def libraries
       [libmruby_static]
     end
+
+    def host?
+      @name == "host"
+    end
+
+    def internal?
+      @internal
+    end
+
+    protected
+
+    attr_writer :presym
+
+    def create_mrbc_build
+      exclusions = %i[@name @build_dir @gems @enable_test @enable_bintest @internal]
+      name = "#{@name}/mrbc"
+      MRuby.targets.delete(name)
+      build = self.class.new(name, internal: true){}
+      instance_variables.each do |n|
+        next if exclusions.include?(n)
+        v = instance_variable_get(n)
+        v = case v
+            when nil, true, false, Numeric; v
+            when String, Command; v.clone
+            else Marshal.load(Marshal.dump(v))  # deep clone
+            end
+        build.instance_variable_set(n, v)
+      end
+      build.build_mrbc_exec
+      build.disable_libmruby
+      build.disable_presym
+      @mrbc_build = build
+      self.mrbcfile = build.mrbcfile
+      build
+    end
   end # Build
 
   class CrossBuild < Build
@@ -405,7 +503,7 @@ EOS
     def initialize(name, build_dir=nil, &block)
       @test_runner = Command::CrossTestRunner.new(self)
       super
-      unless MRuby.targets['host']
+      unless mrbcfile_external? || MRuby.targets['host']
         # add minimal 'host'
         MRuby::Build.new('host') do |conf|
           if ENV['VisualStudioVersion'] || ENV['VSINSTALLDIR']
@@ -413,16 +511,13 @@ EOS
           else
             toolchain :gcc
           end
-          conf.gem :core => 'mruby-bin-mrbc'
+          conf.build_mrbc_exec
           conf.disable_libmruby
+          conf.disable_presym
         end
       end
     end
 
-    def mrbcfile
-      MRuby.targets['host'].mrbcfile
-    end
-
     def run_test
       @test_runner.runner_options << verbose_flag
       mrbtest = exefile("#{build_dir}/bin/mrbtest")
@@ -433,5 +528,9 @@ EOS
         @test_runner.run(mrbtest)
       end
     end
+
+    protected
+
+    def create_mrbc_build; end
   end # CrossBuild
 end # MRuby

lib/mruby/build/command.rb

@@ -42,6 +42,7 @@ module MRuby
     attr_accessor :label, :flags, :include_paths, :defines, :source_exts
     attr_accessor :compile_options, :option_define, :option_include_path, :out_ext
     attr_accessor :cxx_compile_flag, :cxx_exception_flag, :cxx_invalid_flags
+    attr_writer :preprocess_options
 
     def initialize(build, source_exts=[], label: "CC")
       super(build)
@@ -55,9 +56,15 @@ module MRuby
       @option_define = %q[-D"%s"]
       @compile_options = %q[%{flags} -o "%{outfile}" -c "%{infile}"]
       @cxx_invalid_flags = []
+      @out_ext = build.exts.object
     end
 
     alias header_search_paths include_paths
+
+    def preprocess_options
+      @preprocess_options ||= @compile_options.sub(/(?:\A|\s)\K-c(?=\s)/, "-E -P")
+    end
+
     def search_header_path(name)
       header_search_paths.find do |v|
         File.exist? build.filename("#{v}/#{name}").sub(/^"(.*)"$/, '\1')
@@ -79,13 +86,20 @@ module MRuby
 
     def run(outfile, infile, _defines=[], _include_paths=[], _flags=[])
       mkdir_p File.dirname(outfile)
-      _pp @label, infile.relative_path, outfile.relative_path
-      _run compile_options, { :flags => all_flags(_defines, _include_paths, _flags),
-                              :infile => filename(infile), :outfile => filename(outfile) }
+      flags = all_flags(_defines, _include_paths, _flags)
+      if File.extname(outfile) == build.exts.object
+        label = @label
+        opts = compile_options
+      else
+        label = "CPP"
+        opts = preprocess_options
+        flags << " -DMRB_PRESYM_SCANNING"
+      end
+      _pp label, infile.relative_path, outfile.relative_path
+      _run opts, flags: flags, infile: filename(infile), outfile: filename(outfile)
     end
 
-    def define_rules(build_dir, source_dir='')
-      @out_ext = build.exts.object
+    def define_rules(build_dir, source_dir='', out_ext=build.exts.object)
       gemrake = File.join(source_dir, "mrbgem.rake")
       rakedep = File.exist?(gemrake) ? [ gemrake ] : []
 
@@ -143,10 +157,10 @@ module MRuby
     #   /src/value_array.h:
     #
     def get_dependencies(file)
-      file = file.ext('d') unless File.extname(file) == '.d'
-      return [MRUBY_CONFIG] unless File.exist?(file)
+      dep_file = "#{file}.d"
+      return [MRUBY_CONFIG] unless File.exist?(dep_file)
 
-      deps = File.read(file).gsub("\\\n ", "").split("\n").map do |dep_line|
+      deps = File.read(dep_file).gsub("\\\n ", "").split("\n").map do |dep_line|
         # dep_line:
         # - "/build/host/src/array.o:   /src/array.c   /include/mruby/common.h ..."
         # - ""
@@ -187,6 +201,10 @@ module MRuby
       [libraries, _libraries].flatten.map{ |d| option_library % d }.join(' ')
     end
 
+    def run_attrs
+      [@libraries, @library_paths, @flags, @flags_before_libraries, @flags_after_libraries]
+    end
+
     def run(outfile, objfiles, _libraries=[], _library_paths=[], _flags=[], _flags_before_libraries=[], _flags_after_libraries=[])
       mkdir_p File.dirname(outfile)
       library_flags = [libraries, _libraries].flatten.map { |d| option_library % d }

lib/mruby/gem.rb

@@ -7,7 +7,6 @@ module MRuby
     class << self
       attr_accessor :current
     end
-    LinkerConfig = Struct.new(:libraries, :library_paths, :flags, :flags_before_libraries, :flags_after_libraries)
 
     class Specification
       include Rake::DSL
@@ -50,13 +49,13 @@ module MRuby
       end
 
       def setup
-        return if defined?(@linker)  # return if already set up
+        return if defined?(@bins)  # return if already set up
 
         MRuby::Gem.current = self
         MRuby::Build::COMMANDS.each do |command|
           instance_variable_set("@#{command}", @build.send(command).clone)
         end
-        @linker = LinkerConfig.new([], [], [], [], [])
+        @linker.run_attrs.each(&:clear)
 
         @rbfiles = Dir.glob("#{@dir}/mrblib/**/*.rb").sort
         @objs = Dir.glob("#{@dir}/src/*.{c,cpp,cxx,cc,m,asm,s,S}").map do |f|
@@ -72,6 +71,7 @@ module MRuby
         @test_args = {}
 
         @bins = []
+        @cdump = true
 
         @requirements = []
         @export_include_paths = []
@@ -81,7 +81,6 @@ module MRuby
 
         @generate_functions = !(@rbfiles.empty? && @objs.empty?)
         @objs << objfile("#{build_dir}/gem_init") if @generate_functions
-        @cdump = core?  # by default core gems use cdump and others use mrb dump
 
         if !name || !licenses || !authors
           fail "#{name || dir} required to set name, license(s) and author(s)"
@@ -96,10 +95,11 @@ module MRuby
       end
 
       def setup_compilers
-        (core? ? [cc] : compilers).each do |compiler|
-          compiler.define_rules build_dir, "#{dir}"
+        (core? ? [@cc, *(@cxx if build.cxx_exception_enabled?)] : compilers).each do |compiler|
+          compiler.define_rules build_dir, @dir, @build.exts.preprocessed if build.presym_enabled?
+          compiler.define_rules build_dir, @dir, @build.exts.object
           compiler.defines << %Q[MRBGEM_#{funcname.upcase}_VERSION=#{version}]
-          compiler.include_paths << "#{dir}/include" if File.directory? "#{dir}/include"
+          compiler.include_paths << "#{@dir}/include" if File.directory? "#{@dir}/include"
         end
 
         define_gem_init_builder if @generate_functions
@@ -114,12 +114,12 @@ module MRuby
         return false
       end
 
-      def enable_cdump
-        @cdump = true
+      def disable_cdump
+        @cdump = false
       end
 
       def cdump?
-        @cdump
+        build.presym_enabled? && @cdump
       end
 
       def core?
@@ -175,7 +175,6 @@ module MRuby
       end
 
       def define_gem_init_builder
-        file objfile("#{build_dir}/gem_init") => [ "#{build_dir}/gem_init.c", File.join(dir, "mrbgem.rake") ]
         file "#{build_dir}/gem_init.c" => [build.mrbcfile, __FILE__] + [rbfiles].flatten do |t|
           mkdir_p build_dir
           generate_gem_init("#{build_dir}/gem_init.c")
@@ -187,7 +186,7 @@ module MRuby
         open(fname, 'w') do |f|
           print_gem_init_header f
           unless rbfiles.empty?
-            if @cdump
+            if cdump?
               build.mrbc.run f, rbfiles, "gem_mrblib_#{funcname}_proc"
             else
               build.mrbc.run f, rbfiles, "gem_mrblib_irep_#{funcname}", false
@@ -198,10 +197,10 @@ module MRuby
           f.puts %Q[]
           f.puts %Q[void GENERATED_TMP_mrb_#{funcname}_gem_init(mrb_state *mrb) {]
           f.puts %Q[  int ai = mrb_gc_arena_save(mrb);]
-          f.puts %Q[  struct REnv *e;] unless rbfiles.empty?
+          f.puts %Q[  gem_mrblib_#{funcname}_proc_init_syms(mrb);] if !rbfiles.empty? && cdump?
           f.puts %Q[  mrb_#{funcname}_gem_init(mrb);] if objs != [objfile("#{build_dir}/gem_init")]
           unless rbfiles.empty?
-            if @cdump
+            if cdump?
               f.puts %Q[  mrb_load_proc(mrb, gem_mrblib_#{funcname}_proc);]
             else
               f.puts %Q[  mrb_load_irep(mrb, gem_mrblib_irep_#{funcname});]
@@ -211,7 +210,7 @@ module MRuby
             f.puts %Q[    mrb_close(mrb);]
             f.puts %Q[    exit(EXIT_FAILURE);]
             f.puts %Q[  }]
-            f.puts %Q[  e = mrb->c->cibase->env;]
+            f.puts %Q[  struct REnv *e = mrb->c->cibase->env;]
             f.puts %Q[  mrb->c->cibase->env = NULL;]
             f.puts %Q[  mrb_env_unshare(mrb, e);]
           end
@@ -241,8 +240,6 @@ module MRuby
           f.puts %Q[#include <mruby.h>]
         else
           f.puts %Q[#include <stdlib.h>]
-          f.puts %Q[#include <mruby.h>]
-          f.puts %Q[#include <mruby/proc.h>]
         end
       end
 
@@ -251,7 +248,7 @@ module MRuby
         f.puts %Q[#include <stdio.h>]
         f.puts %Q[#include <stdlib.h>]
         f.puts %Q[#include <mruby.h>]
-        f.puts %Q[#include <mruby/proc.h>]
+        f.puts %Q[#include <mruby/irep.h>]
         f.puts %Q[#include <mruby/variable.h>]
         f.puts %Q[#include <mruby/hash.h>] unless test_args.empty?
       end

lib/mruby/presym.rb

@@ -0,0 +1,113 @@
+module MRuby
+  class Presym
+    include Rake::DSL
+
+    OPERATORS = {
+      "!" => "not",
+      "%" => "mod",
+      "&" => "and",
+      "*" => "mul",
+      "+" => "add",
+      "-" => "sub",
+      "/" => "div",
+      "<" => "lt",
+      ">" => "gt",
+      "^" => "xor",
+      "`" => "tick",
+      "|" => "or",
+      "~" => "neg",
+      "!=" => "neq",
+      "!~" => "nmatch",
+      "&&" => "andand",
+      "**" => "pow",
+      "+@" => "plus",
+      "-@" => "minus",
+      "<<" => "lshift",
+      "<=" => "le",
+      "==" => "eq",
+      "=~" => "match",
+      ">=" => "ge",
+      ">>" => "rshift",
+      "[]" => "aref",
+      "||" => "oror",
+      "<=>" => "cmp",
+      "===" => "eqq",
+      "[]=" => "aset",
+    }.freeze
+
+    SYMBOL_TO_MACRO = {
+    #      Symbol      =>      Macro
+    # [prefix, suffix] => [prefix, suffix]
+      ["@@"  , ""    ] => ["CV"  , ""    ],
+      ["@"   , ""    ] => ["IV"  , ""    ],
+      [""    , "!"   ] => [""    , "_B"  ],
+      [""    , "?"   ] => [""    , "_Q"  ],
+      [""    , "="   ] => [""    , "_E"  ],
+      [""    , ""    ] => [""    , ""    ],
+    }.freeze
+
+    C_STR_LITERAL_RE = /"(?:[^\\\"]|\\.)*"/
+
+    def initialize(build)
+      @build = build
+    end
+
+    def scan(paths)
+      presym_hash = {}
+      paths.each {|path| read_preprocessed(presym_hash, path)}
+      presym_hash.keys.sort_by!{|sym| [c_literal_size(sym), sym]}
+    end
+
+    def read_list
+      File.readlines(list_path, mode: "r:binary").each(&:chomp!)
+    end
+
+    def write_list(presyms)
+      _pp "GEN", list_path.relative_path
+      File.binwrite(list_path, presyms.join("\n") << "\n")
+    end
+
+    def write_header(presyms)
+      prefix_re = Regexp.union(*SYMBOL_TO_MACRO.keys.map(&:first).uniq)
+      suffix_re = Regexp.union(*SYMBOL_TO_MACRO.keys.map(&:last).uniq)
+      sym_re = /\A(#{prefix_re})?([\w&&\D]\w*)(#{suffix_re})?\z/o
+      _pp "GEN", header_path.relative_path
+      mkdir_p(File.dirname(header_path))
+      File.open(header_path, "w:binary") do |f|
+        f.puts "/* MRB_PRESYM_NAMED(lit, num, type, name) */"
+        f.puts "/* MRB_PRESYM_UNNAMED(lit, num) */"
+        presyms.each.with_index(1) do |sym, num|
+          if sym_re =~ sym && (affixes = SYMBOL_TO_MACRO[[$1, $3]])
+            f.puts %|MRB_PRESYM_NAMED("#{sym}", #{num}, #{affixes * 'SYM'}, #{$2})|
+          elsif name = OPERATORS[sym]
+            f.puts %|MRB_PRESYM_NAMED("#{sym}", #{num}, OPSYM, #{name})|
+          elsif
+            f.puts %|MRB_PRESYM_UNNAMED("#{sym}", #{num})|
+          end
+        end
+        f.puts "#define MRB_PRESYM_MAX #{presyms.size}"
+      end
+    end
+
+    def list_path
+      @list_pat ||= "#{@build.build_dir}/presym".freeze
+    end
+
+    def header_path
+      @header_path ||= "#{@build.build_dir}/include/mruby/presym.inc".freeze
+    end
+
+    private
+
+    def read_preprocessed(presym_hash, path)
+      File.binread(path).scan(/<@! (.*?) !@>/) do |part,|
+        literals = part.scan(C_STR_LITERAL_RE)
+        presym_hash[literals.map{|l| l[1..-2]}.join] = true unless literals.empty?
+      end
+    end
+
+    def c_literal_size(literal_without_quote)
+      literal_without_quote.size  # TODO: consider escape sequence
+    end
+  end
+end