diff --git a/doc/guides/mrbgems.md b/doc/guides/mrbgems.md index 7649f40d2..64a12cff4 100644 --- a/doc/guides/mrbgems.md +++ b/doc/guides/mrbgems.md @@ -33,6 +33,9 @@ conf.gem :github => 'masuidrive/mrbgems-example', :branch => 'master' conf.gem :bitbucket => 'mruby/mrbgems-example', :branch => 'master' ``` +NOTE: `:bitbucket` option supports only git. Hg is unsupported in this +version. + You can specify the subdirectory of the repository with `:path` option: ```ruby @@ -55,10 +58,24 @@ conf.gem mgem: 'mruby-redis', checksum_hash: '3446d19fc4a3f9697b5ddbf2a904f301c4 If there are missing dependencies, mrbgem dependencies solver will reference mrbgem from the core or mgem-list. -To pull all gems from remote GIT repository on build, call `rake -p`, -or `rake --pull-gems`. +Note that if more than one git-based gem has the same base name +(i.e. the default checkout directory name), it is (now) an error +**UNLESS** they are have the same repository URL, branch name and +commit-id (i.e. checksum hash). You can bypass this by explicitly +importing your preferred version **first** and setting the +`canonical:` option to `true`: -NOTE: `:bitbucket` option supports only git. Hg is unsupported in this version. +```ruby +conf.gem github: 'me/mruby-yaml', branch: 'my-hacked-branch', canonical: true +``` + +If you do this, the system will (mostly) silently ignore other +attempts to clone a gem with this name. + +Note that this only affects cloning the gem from git. It does not +resolve version conflicts. If the version as specified in the gem's +rakefile is incompatible with a dependency, your build will still +fail. ## GemBox diff --git a/lib/mruby/build/load_gems.rb b/lib/mruby/build/load_gems.rb index 6609a5c8e..15331dd6f 100644 --- a/lib/mruby/build/load_gems.rb +++ b/lib/mruby/build/load_gems.rb @@ -1,5 +1,6 @@ module MRuby module LoadGems + def gembox(gemboxfile) gembox = File.expand_path("#{gemboxfile}.gembox", "#{MRUBY_ROOT}/mrbgems") fail "Can't find gembox '#{gembox}'" unless File.exist?(gembox) @@ -12,28 +13,30 @@ module MRuby GemBox.path = nil end - def gem(gemdir, &block) - if gemdir.is_a?(Hash) - gemdir = load_special_path_gem(gemdir) - elsif GemBox.path - gemdir = File.expand_path(gemdir, File.dirname(GemBox.path)) - else - caller_dir = File.expand_path(File.dirname(caller(1,1)[0][/^(.*?):\d/,1])) - if caller_dir == "#{MRUBY_ROOT}/build_config" - caller_dir = MRUBY_ROOT - end - gemdir = File.expand_path(gemdir, caller_dir) - end + def gem(gem_src, &block) - gemrake = File.join(gemdir, "mrbgem.rake") + caller_dir = File.expand_path(File.dirname(caller(1,1)[0][/^(.*?):\d/,1])) + gem_src = {gemdir: gem_src} if gem_src.is_a? String + + @gem_checkouts ||= {} + checkout = GemLoader + .new(self, caller_dir, @gem_checkouts, **gem_src) + .fetch! + return nil unless checkout + @gem_checkouts[checkout.gemdir] = checkout + + # Load the gem's rakefile + gemrake = File.join(checkout.gemdir, "mrbgem.rake") fail "Can't find #{gemrake}" unless File.exist?(gemrake) + Gem.current = nil load gemrake return nil unless Gem.current current = Gem.current - current.dir = gemdir + # Add it to gems + current.dir = checkout.gemdir current.build = self.is_a?(MRuby::Build) ? self : MRuby::Build.current current.build_config_initializer = block gems << current @@ -44,92 +47,329 @@ module MRuby current end - def load_special_path_gem(params) - if params[:github] - params[:git] = "https://github.com/#{params[:github]}.git" - elsif params[:bitbucket] - if params[:method] == "ssh" - params[:git] = "git@bitbucket.org:#{params[:bitbucket]}.git" - else - params[:git] = "https://bitbucket.org/#{params[:bitbucket]}.git" - end - elsif params[:mgem] - mgem_list_dir = "#{gem_clone_dir}/mgem-list" - mgem_list_url = 'https://github.com/mruby/mgem-list.git' - if File.exist? mgem_list_dir - git.run_pull mgem_list_dir, mgem_list_url if $pull_gems - else - mkdir_p mgem_list_dir - git.run_clone mgem_list_dir, mgem_list_url, "--depth 1" - end - require 'yaml' + # Class to represent the relationship between a gem dependency and + # its remote repository (if any). + class GemCheckout + attr_reader :gemdir, :repo, :branch, :commit - conf_path = "#{mgem_list_dir}/#{params[:mgem]}.gem" - conf_path = "#{mgem_list_dir}/mruby-#{params[:mgem]}.gem" unless File.exist? conf_path - fail "mgem not found: #{params[:mgem]}" unless File.exist? conf_path - conf = YAML.load File.read conf_path + def initialize(gemdir, repo, branch, commit, canonical) + @gemdir = gemdir # Working copy of the gem - fail "unknown mgem protocol: #{conf['protocol']}" if conf['protocol'] != 'git' - params[:git] = conf['repository'] - params[:branch] = conf['branch'] if conf['branch'] + @repo = repo # Remote gem repo + @branch = branch # Branch to check out + @commit = commit # Commit-id to use + + @canonical = canonical # This is the One True checkout end - if params[:core] - gemdir = "#{root}/mrbgems/#{params[:core]}" - elsif params[:git] - url = params[:git] - gemdir = "#{gem_clone_dir}/#{url.match(/([-\w]+)(\.[-\w]+|)$/).to_a[1]}" + def canonical?() return @canonical; end + def git?() return !!@repo; end + def gemname() return File.basename(@gemdir); end - # by default the 'master' branch is used - branch = params[:branch] ? params[:branch] : 'master' + def hash() + return [@gemdir, @repo, @branch, @commit, @canonical].hash + end - lock = locks[url] if lock_enabled? + def ==(other) + return @gemdir == other.gemdir && @repo == other.repo && + @branch == other.branch && @commit == other.commit && + @canonical == other.canonical? + end + alias_method :eql?, :== - if File.exist?(gemdir) - if $pull_gems - # Jump to the top of the branch - git.run_checkout gemdir, branch - git.run_pull gemdir, url - elsif params[:checksum_hash] - git.run_checkout_detach gemdir, params[:checksum_hash] - elsif lock - git.run_checkout_detach gemdir, lock['commit'] - end + def to_s + desc = @gemdir + desc += " -> #{@repo}/#{@branch}" if git? + desc += "/#{commit}" if commit + return desc + end + end + + # Class to decode the argument set given to 'MRuby::Build::gem', + # and git-clone+git-checkout the sources if needed. + class GemLoader + def initialize(build, + build_config_dir, # Parent dir. of build_config + gem_checkouts, # Hash of existing checkouts + + # Git repo: + git: nil, + branch: "master", + checksum_hash: nil, + options: [], + + # Git repo on GitHub + github: nil, + + # Git repo on BitBucket + bitbucket: nil, + method: nil, + + # mgem entry + mgem: nil, + + # Core package + core: nil, + + # Path relative to the mruby checkout; probably wrong! + path: nil, + + # Local file(s) + gemdir: nil, + + # Related flags: + canonical: false # Ignore subsequent checkout of this gem + ) + @build = build + @build_config_dir = build_config_dir + @gem_checkouts = gem_checkouts + @canonical = canonical + + @git = git + @branch = branch + @checksum_hash = checksum_hash + @options = options + @canonical = canonical + + @github = github + + @bitbucket = bitbucket + @method = method + + @mgem = mgem + @core = core + @path = path + @gemdir = gemdir + + + actions = [git, github, bitbucket, mgem, core, path, gemdir] + fail("Need to set exactly ONE of git, github, bitbucket, mgem, core, " + + "path, or gemdir") unless actions.compact.size == 1 + end + + # Retrieve the repo and return the details in a GemCheckout + # object or nil if nothing needed to be done. + def fetch! + return fromGemdir! if @gemdir + return fromCore! if @core + return fromPath! if @path + return fromGitHub! if @github + return fromBitBucket! if @bitbucket + + return fromMGem! if @mgem + + return fromGit!(@git, @branch) if @git + + # Shouldn't be reachable, but... + fail "Invalid gem configuration!" + end + + private + + # + # Local Paths + # + + def fromGemdir! + gem_src = @gemdir + + # If @gemdir is a relative path, we first convert it to an + # absolute path; this depends on circumstances. + if MRuby::GemBox.path + # If GemBox.path is set, it means that this fetch operation is + # happening as part of a gembox evaluation and we use the + # gembox's path as the starting point. + gem_src = File.expand_path(gem_src, File.dirname(MRuby::GemBox.path)) else - options = [params[:options]] || [] - options << "--recursive" - options << "--branch \"#{branch}\"" - options << "--depth 1" unless params[:checksum_hash] || lock - mkdir_p "#{gem_clone_dir}" - git.run_clone gemdir, url, options + # Otherwise, we use the path to the build_config.rb file that + # requested this gem. This path was extracted earlier and + # stored in @build_config_dir via the second argument of + # 'initialize'. + root_dir = @build_config_dir - # Jump to the specified commit - if params[:checksum_hash] - git.run_checkout_detach gemdir, params[:checksum_hash] - elsif lock - git.run_checkout_detach gemdir, lock['commit'] - end + # And we default to the repo root if the file is one of the + # stock configs in build_config/. + root_dir = MRUBY_ROOT if root_dir == "#{MRUBY_ROOT}/build_config" + + gem_src = File.expand_path(gem_src, root_dir) end - if lock_enabled? - @gem_dir_to_repo_url[gemdir] = url unless params[:path] - locks[url] = { + return GemCheckout.new(gem_src, nil, nil, nil, @canonical) + end + + def fromCore! + return GemCheckout.new("#{@build.root}/mrbgems/#{@core}", nil, nil, + nil, @canonical) + end + + # This is probably incorrect. + # + # According to doc/guides/mrbgems.md, this should specify a + # subdirectory of a git checkout (i.e. for the case where the gem + # itself is not in the repository's root.) + # + # However, the code treats this as simple path to a local + # directory (just like :gemdir) with relative paths resolved + # relative to the project root. + # + # I'm pretty sure that the correct thing is what guide says but + # I'm going to keep the initial semantics for now. + def fromPath! + p = Pathname.new(@path).absolute? ? @path : "#{@build.root}/#{@path}" + return GemCheckout.new(p, nil, nil, nil, @canonical) + end + + + # + # Git forges + # + + def fromGitHub! + url = "https://github.com/#{@github}.git" + return fromGit!(url, @branch) + end + + def fromBitBucket! + if @method == "ssh" + url = "git@bitbucket.org:#{@bitbucket}.git" + else + url = "https://bitbucket.org/#{@bitbucket}.git" + end + + return fromGit!(url, @branch) + end + + + # + # mgem file + # + + def fromMGem! + mgem = fetchMGem(@mgem) + + url = mgem['repository'] + branch = mgem['branch'] || @branch + + return fromGit!(url, branch) + end + + # Fetch the contents of the named mgem item. Will clone the + # mgem-list repo if not present + def fetchMGem(mgem) + list_dir = "#{@build.gem_clone_dir}/mgem-list" + url = 'https://github.com/mruby/mgem-list.git' + + git_clone_dependency(url, list_dir, nil, 'master') + + conf_path = "#{list_dir}/#{mgem}.gem" + conf_path = "#{list_dir}/mruby-#{mgem}.gem" unless + File.exist? conf_path + fail "mgem not found: #{mgem}" unless File.exist? conf_path + + conf = YAML.load File.read conf_path + fail "unknown mgem protocol: #{conf['protocol']}" if + conf['protocol'] != 'git' + + return conf + end + + + # + # Git checkouts + # + + def fromGit!(url, branch) + repo_dir = "#{@build.gem_clone_dir}/" + + "#{url.match(/([-\w]+)(\.[-\w]+|)$/).to_a[1]}" + commit = @checksum_hash + + return nil if skip_this?(url, repo_dir, branch, commit) + + # If there's a lockfile entry for this repo AND the user hasn't + # specified a specific commit ID, we use the locked branch and + # commit. + lock = @build.locks[url] if @build.lock_enabled? + if !commit && lock + branch = lock['branch'] + commit = lock['commit'] + end + + # Clone the dependency (if needed) and checkout the expected + # revision. + git_clone_dependency(url, repo_dir, commit, branch) + git_checkout_dependency(repo_dir, commit, branch) + + # Set the lockfile entry if enabled + if @build.lock_enabled? + @build.gem_dir_to_repo_url[repo_dir] = url + @build.locks[url] = { 'url' => url, - 'branch' => git.current_branch(gemdir), - 'commit' => git.commit_hash(gemdir), + 'branch' => @build.git.current_branch(repo_dir), + 'commit' => @build.git.commit_hash(repo_dir), } end - gemdir << "/#{params[:path]}" if params[:path] - elsif params[:path] - require 'pathname' - gemdir = Pathname.new(params[:path]).absolute? ? params[:path] : "#{root}/#{params[:path]}" - else - fail "unknown gem option #{params}" + return GemCheckout.new(repo_dir, url, branch, commit, @canonical) end - gemdir + + # Test if this repo can be skipped. This will happen if it's + # already in @gem_checkouts and EITHER it is identical (same + # url, branch and commit-ID) as the current checkout OR it's + # "canonical" flag is true. If it's in @gem_checkouts and + # neither of these conditions is true, that's a fatal error; it + # means there are multiple incompatible versions of this gem to + # be checked out into this directory. + # + # Otherwise, returns false. + def skip_this?(url, repo_dir, branch, commit) + prev = @gem_checkouts[repo_dir] + return false unless prev + + # Canonical declarations must precede all others. + fail("Attempted to re-declare #{prev.gemname} as canonical!\n" + + "('canonical' can only be used on its first declaration.)") if + prev && @canonical + + # If prev is canonical, we can ignore this + if prev.canonical? + puts "Found canonical #{prev.gemname}; skipping this one." + return true + end + + # If this checkout is identical to the current one, we can skip it. + candidate = GemCheckout.new(repo_dir, url, branch, commit, @canonical) + if prev == candidate + puts "Found duplicate checkout for #{repo_dir}; ignoring." + return true + end + + # Otherwise, we have a checkout conflict. This is an error. + fail "Conflicting gem definitions for '#{repo_dir}':\n" + + " #{candidate}\n" + + " #{prev}\n" + end + + + # Retrieve a git repo if it's not present. Return + # [path_to_checkout, did_clone] + def git_clone_dependency(url, repo_dir, commit, branch) + return if File.exist?(repo_dir) + + FileUtils.mkdir_p repo_dir + + options = @options.dup + options << "--recursive" + options << "--branch \"#{branch}\"" + options << "--depth 1" unless commit + + @build.git.run_clone repo_dir, url, options + end + + def git_checkout_dependency(repo_dir, commit, branch) + @build.git.run_checkout_detach(repo_dir, commit) + end end def enable_gems?