[Bug #21388] Make snapshots of gems

If the revision of bundled gems is specified for ruby master (and
`git` is usable), checkout that revision and build a snapshot gem, and
use it for `test-spec` instead of the downloaded release version.

Author: nobu

common.mk

@@ -1533,46 +1533,44 @@ extract-gems: $(HAVE_BASERUBY:yes=update-gems)
 update-gems$(sequential): PHONY
 	$(ECHO) Downloading bundled gem files...
 	$(Q) $(BASERUBY) -C "$(srcdir)" \
-	    -I./tool -rdownloader -answ \
-	    -e 'gem, ver = *$$F' \
-	    -e 'next if !ver or /^#/=~gem' \
-	    -e 'old = Dir.glob("gems/#{gem}-*.gem")' \
-	    -e 'gem = "#{gem}-#{ver}.gem"' \
-	    -e 'Downloader::RubyGems.download(gem, "gems", nil) and' \
-	    -e '(old.delete("gems/#{gem}"); !old.empty?) and' \
-	    -e 'File.unlink(*old) and' \
-	    -e 'FileUtils.rm_rf(old.map{'"|n|"'n.chomp(".gem")})' \
-	    gems/bundled_gems
+	    -I./tool/lib -r./tool/downloader -rbundled_gem  \
+	    -e "BundledGem.each(snapshot: %[$(HAVE_GIT)]==%[yes]) do |gem, ver, _, rev|" \
+	    -e   "old = Dir.glob(%[gems/#{gem}-*.gem])" \
+	    -e   "gem = %[#{gem}-#{ver}.gem]" \
+	    -e   "(rev || Downloader::RubyGems.download(gem, %[gems], nil)) and" \
+	    -e   "(old.delete(%[gems/#{gem}]); !old.empty?) and" \
+	    -e   "File.unlink(*old) and" \
+	    -e   "FileUtils.rm_rf(old.map{|n|n.chomp(%[.gem])})" \
+	    -e "end"
 
 extract-gems$(sequential): PHONY
 	$(ECHO) Extracting bundled gem files...
 	$(Q) $(BASERUBY) -C "$(srcdir)" \
-	    -Itool/lib -rfileutils -rbundled_gem -answ \
-	    -e 'BEGIN {d = ".bundle/gems"}' \
-	    -e 'gem, ver, _, rev = *$$F' \
-	    -e 'next if !ver or /^#/=~gem' \
-	    -e 'g = "#{gem}-#{ver}"' \
-	    -e 'unless File.directory?("#{d}/#{g}")' \
-	    -e   'if rev and File.exist?(gs = "gems/src/#{gem}/#{gem}.gemspec")' \
-	    -e     'BundledGem.build(gs, ver, "gems")' \
-	    -e   'end' \
-	    -e   'BundledGem.unpack("gems/#{g}.gem", ".bundle")' \
-	    -e 'end' \
-	    gems/bundled_gems
+	    -Itool/lib -rfileutils -rbundled_gem \
+	    -e "d = ARGV.shift" \
+	    -e "BundledGem.each(snapshot: %[$(HAVE_GIT)]==%[yes]) do |gem, ver, _, rev|" \
+	    -e   "g = %[#{gem}-#{ver}]" \
+	    -e   "unless File.directory?(%[#{d}/#{g}])" \
+	    -e     "if rev and File.exist?(gs = %[gems/src/#{gem}/#{gem}.gemspec])" \
+	    -e       "BundledGem.build(gs, ver, %[gems])" \
+	    -e     "end" \
+	    -e     "BundledGem.unpack(%[gems/#{g}.gem], %[.bundle])" \
+	    -e   "end" \
+	    -e "end"
+	    -- .bundle/gems
 
 extract-gems$(sequential): $(HAVE_GIT:yes=clone-bundled-gems-src)
 
 clone-bundled-gems-src: PHONY
 	$(Q) $(BASERUBY) -C "$(srcdir)" \
-	    -Itool/lib -rbundled_gem -answ \
-	    -e 'BEGIN {git = $$git}' \
-	    -e 'gem, _, repo, rev = *$$F' \
-	    -e 'next if !rev or /^#/=~gem' \
-	    -e 'gemdir = "gems/src/#{gem}"' \
-	    -e 'BundledGem.checkout(gemdir, repo, rev, git: git)' \
-	    -e 'BundledGem.dummy_gemspec("#{gemdir}/#{gem}.gemspec")' \
-	    -- -git="$(GIT)" \
-	    gems/bundled_gems
+	    -Itool/lib -rbundled_gem \
+	    -e "git = $$git" \
+	    -e "BundledGem.each do |gem, _, repo, rev|" \
+	    -e   "gemdir = %[gems/src/#{gem}]" \
+	    -e   "BundledGem.checkout(gemdir, repo, rev, git: git)" \
+	    -e   "BundledGem.dummy_gemspec(%[#{gemdir}/#{gem}.gemspec])" \
+	    -e "end" \
+	    -- -git="$(GIT)"
 
 outdate-bundled-gems: PHONY
 	$(Q) $(BASERUBY) $(tooldir)/$@.rb --make="$(MAKE)" --mflags="$(MFLAGS)" \

defs/gmake.mk

@@ -305,13 +305,23 @@ HELP_EXTRA_TASKS = \
 
 # 1. squeeze spaces
 # 2. strip and skip comment/empty lines
-# 3. "gem x.y.z URL xxxxxx" -> "gem|x.y.z|xxxxxx|URL"
+# 3. "gem x.y.z URL xxxxxx" -> "gem|x.y.z(+1).snapshot|xxxxxx|URL"
 # 4. "gem x.y.z URL" -> "gem-x.y.z"
 bundled-gems := $(shell sed \
 	-e 's/[ 	][ 	]*/ /g' \
 	-e 's/^ //;/\#/d;s/ *$$//;/^$$/d' \
 	$(if $(filter yes,$(HAVE_GIT)), \
-	-e 's/^\(.*\) \(.*\) \(.*\) \(.*\)/\1|\2|\4|\3/' \
+	-e '/^\(.*\) \(.*\) \(.*\) \(.*\)/{' \
+	  -e 's//\1|\3|\4|\2/' \
+	  -e 's/|[0-9][0-9]*\.[0-9][0-9]*$$/&.0/' \
+	  -e '/\([0-9]9*\)$$/{' \
+	    -e 's//\n\1/;h;s/^.*\n//' \
+	    -e 'y/0123456789/1234567890/;s/^0/10/' \
+	    -e 'x;G;s/\n.*\n//;s/$$/.snapshot/' \
+	  -e '}' \
+	  -e 's/^\(.*\)|\(.*\)|\(.*\)|\(.*\)/\1|\4|\3|\2/' \
+	\
+	-e '}' \
 	) \
 	-e 's/ /-/;s/ .*//' \
 	 $(srcdir)/gems/bundled_gems)
@@ -385,11 +395,12 @@ $(bundled-gem-gemspec): $(bundled-gem-revision) \
 	| $(srcdir)/gems/src/$(1)/.git
 	$(Q) $(BASERUBY) -I$(tooldir)/lib -rbundled_gem -e 'BundledGem.dummy_gemspec(*ARGV)' $$(@)
 
-$(bundled-gem-gemfile): $(bundled-gem-gemspec) $(bundled-gem-revision)
+$(bundled-gem-gemfile): $(bundled-gem-revision)
 	$(ECHO) Building $(1)@$(3) to $$(@)
 	$(Q) $(BASERUBY) -C "$(srcdir)" \
 	    -Itool/lib -rbundled_gem \
-	    -e 'BundledGem.build("gems/src/$(1)/$(1).gemspec", "$(2)", "gems", validation: false)'
+	    -e 'BundledGem.build(*ARGV, validation: false)' \
+	    gems/src/$(1)/$(1).gemspec $(2) gems
 
 endef
 define build-gem-0

tool/lib/bundled_gem.rb

@@ -16,6 +16,24 @@ module BundledGem
     "psych" # rdoc
   ]
 
+  def self.each(release: true, snapshot: false)
+    File.foreach(File.join(__dir__, "../../gems/bundled_gems")) do |line|
+      line.chomp!
+      next if /^\s*(?:#|$)/ =~ line
+      gem, ver, uri, rev = line.split
+      if !rev
+        next unless release
+      elsif snapshot
+        # Assume a version ending with digits only segment is a release
+        # version, and append suffix to make prerelase version.
+        # Bump up because "X.Y.Z.snapshot" < "X.Y.Z" as versions.
+        ver = ver.succ if /\.\d+\z/.match?(ver)
+        ver += ".snapshot"
+      end
+      yield gem, ver, uri, rev
+    end
+  end
+
   module_function
 
   def unpack(file, *rest)
@@ -36,6 +54,15 @@ def build(gemspec, version, outdir = ".", validation: true)
     Dir.chdir(gemdir) do
       spec = Gem::Specification.load(gemfile)
       abort "Failed to load #{gemspec}" unless spec
+      spec.version = version
+      spec.files.delete_if do |f|
+        case f
+        when 'Gemfile', 'Rakefile', gemfile
+          true
+        else
+          f.start_with?('bin/', 'test/', '.git')
+        end
+      end
       output = File.join(outdir, spec.file_name)
       FileUtils.rm_rf(output)
       package = Gem::Package.new(output)