Sync latest development version of bundler & rubygems

Merged: https://github.com/ruby/ruby/pull/4143

lib/rubygems/command.rb

@@ -634,6 +634,7 @@ RubyGems is a package manager for Ruby.
     gem install rake
     gem list --local
     gem build package.gemspec
+    gem push package-0.0.1.gem
     gem help install
 
   Further help:

lib/rubygems/config_file.rb

@@ -45,6 +45,7 @@ class Gem::ConfigFile
   DEFAULT_UPDATE_SOURCES = true
   DEFAULT_CONCURRENT_DOWNLOADS = 8
   DEFAULT_CERT_EXPIRATION_LENGTH_DAYS = 365
+  DEFAULT_IPV4_FALLBACK_ENABLED = false
 
   ##
   # For Ruby packagers to set configuration defaults.  Set in
@@ -140,6 +141,12 @@ class Gem::ConfigFile
 
   attr_accessor :cert_expiration_length_days
 
+  ##
+  # == Experimental ==
+  # Fallback to IPv4 when IPv6 is not reachable or slow (default: false)
+
+  attr_accessor :ipv4_fallback_enabled
+
   ##
   # Path name of directory or file of openssl client certificate, used for remote https connection with client authentication
 
@@ -175,6 +182,7 @@ class Gem::ConfigFile
     @update_sources = DEFAULT_UPDATE_SOURCES
     @concurrent_downloads = DEFAULT_CONCURRENT_DOWNLOADS
     @cert_expiration_length_days = DEFAULT_CERT_EXPIRATION_LENGTH_DAYS
+    @ipv4_fallback_enabled = ENV['IPV4_FALLBACK_ENABLED'] == 'true' || DEFAULT_IPV4_FALLBACK_ENABLED
 
     operating_system_config = Marshal.load Marshal.dump(OPERATING_SYSTEM_DEFAULTS)
     platform_config = Marshal.load Marshal.dump(PLATFORM_DEFAULTS)
@@ -203,6 +211,7 @@ class Gem::ConfigFile
     @disable_default_gem_server  = @hash[:disable_default_gem_server]  if @hash.key? :disable_default_gem_server
     @sources                     = @hash[:sources]                     if @hash.key? :sources
     @cert_expiration_length_days = @hash[:cert_expiration_length_days] if @hash.key? :cert_expiration_length_days
+    @ipv4_fallback_enabled       = @hash[:ipv4_fallback_enabled]       if @hash.key? :ipv4_fallback_enabled
 
     @ssl_verify_mode  = @hash[:ssl_verify_mode]  if @hash.key? :ssl_verify_mode
     @ssl_ca_cert      = @hash[:ssl_ca_cert]      if @hash.key? :ssl_ca_cert

lib/rubygems/core_ext/tcpsocket_init.rb

@@ -0,0 +1,52 @@
+require 'socket'
+
+module CoreExtensions
+  module TCPSocketExt
+    def self.prepended(base)
+      base.prepend Initializer
+    end
+
+    module Initializer
+      CONNECTION_TIMEOUT = 5
+      IPV4_DELAY_SECONDS = 0.1
+
+      def initialize(host, serv, *rest)
+        mutex = Mutex.new
+        addrs = []
+        threads = []
+        cond_var = ConditionVariable.new
+
+        Addrinfo.foreach(host, serv, nil, :STREAM) do |addr|
+          Thread.report_on_exception = false if defined? Thread.report_on_exception = ()
+
+          threads << Thread.new(addr) do
+            # give head start to ipv6 addresses
+            sleep IPV4_DELAY_SECONDS if addr.ipv4?
+
+            # raises Errno::ECONNREFUSED when ip:port is unreachable
+            Socket.tcp(addr.ip_address, serv, connect_timeout: CONNECTION_TIMEOUT).close
+            mutex.synchronize do
+              addrs << addr.ip_address
+              cond_var.signal
+            end
+          end
+        end
+
+        mutex.synchronize do
+          timeout_time = CONNECTION_TIMEOUT + Time.now.to_f
+          while addrs.empty? && (remaining_time = timeout_time - Time.now.to_f) > 0
+            cond_var.wait(mutex, remaining_time)
+          end
+
+          host = addrs.shift unless addrs.empty?
+        end
+
+        threads.each {|t| t.kill.join if t.alive? }
+
+        super(host, serv, *rest)
+      end
+    end
+  end
+end
+
+TCPSocket.prepend CoreExtensions::TCPSocketExt

lib/rubygems/dependency.rb

@@ -277,7 +277,7 @@ class Gem::Dependency
       requirement.satisfied_by?(spec.version) && env_req.satisfied_by?(spec.version)
     end.map(&:to_spec)
 
-    Gem::BundlerVersionFinder.filter!(matches) if name == "bundler".freeze && !requirement.specific?
+    Gem::BundlerVersionFinder.filter!(matches) if filters_bundler?
 
     if platform_only
       matches.reject! do |spec|
@@ -295,6 +295,10 @@ class Gem::Dependency
     @requirement.specific?
   end
 
+  def filters_bundler?
+    name == "bundler".freeze && !specific?
+  end
+
   def to_specs
     matches = matching_specs true
 

lib/rubygems/ext/builder.rb

@@ -28,13 +28,14 @@ class Gem::Ext::Builder
     unless make_program
       make_program = (/mswin/ =~ RUBY_PLATFORM) ? 'nmake' : 'make'
     end
+    make_program = Shellwords.split(make_program)
 
     destdir = 'DESTDIR=%s' % ENV['DESTDIR']
 
     ['clean', '', 'install'].each do |target|
       # Pass DESTDIR via command line to override what's in MAKEFLAGS
       cmd = [
-        make_program,
+        *make_program,
         destdir,
         target,
       ].reject(&:empty?)

lib/rubygems/installer_uninstaller_utils.rb

@@ -6,11 +6,16 @@
 module Gem::InstallerUninstallerUtils
 
   def regenerate_plugins_for(spec, plugins_dir)
+    plugins = spec.plugins
+    return if plugins.empty?
+
+    require 'pathname'
+
     spec.plugins.each do |plugin|
       plugin_script_path = File.join plugins_dir, "#{spec.name}_plugin#{File.extname(plugin)}"
 
       File.open plugin_script_path, 'wb' do |file|
-        file.puts "require '#{plugin}'"
+        file.puts "require_relative '#{Pathname.new(plugin).relative_path_from(Pathname.new(plugins_dir))}'"
       end
 
       verbose plugin_script_path

lib/rubygems/platform.rb

@@ -66,7 +66,7 @@ class Gem::Platform
     when String then
       arch = arch.split '-'
 
-      if arch.length > 2 and arch.last !~ /\d/ # reassemble x86-linux-gnu
+      if arch.length > 2 and arch.last !~ /\d+(\.\d+)?$/ # reassemble x86-linux-{libc}
         extra = arch.pop
         arch.last << "-#{extra}"
       end
@@ -121,10 +121,6 @@ class Gem::Platform
     end
   end
 
-  def inspect
-    "%s @cpu=%p, @os=%p, @version=%p>" % [super[0..-2], *to_a]
-  end
-
   def to_a
     [@cpu, @os, @version]
   end
@@ -150,7 +146,8 @@ class Gem::Platform
   ##
   # Does +other+ match this platform?  Two platforms match if they have the
   # same CPU, or either has a CPU of 'universal', they have the same OS, and
-  # they have the same version, or either has no version.
+  # they have the same version, or either has no version (except for 'linux'
+  # where the version is the libc name, with no version standing for 'gnu')
   #
   # Additionally, the platform will match if the local CPU is 'arm' and the
   # other CPU starts with "arm" (for generic ARM family support).
@@ -166,7 +163,10 @@ class Gem::Platform
     @os == other.os and
 
     # version
-    (@version.nil? or other.version.nil? or @version == other.version)
+    (
+      (@os != 'linux' and (@version.nil? or other.version.nil?)) or
+      @version == other.version
+    )
   end
 
   ##

lib/rubygems/remote_fetcher.rb

@@ -78,6 +78,7 @@ class Gem::RemoteFetcher
   #            fetching the gem.
 
   def initialize(proxy=nil, dns=nil, headers={})
+    require 'rubygems/core_ext/tcpsocket_init' if Gem.configuration.ipv4_fallback_enabled
     require 'net/http'
     require 'stringio'
     require 'uri'
@@ -295,7 +296,7 @@ class Gem::RemoteFetcher
 
     data = fetch_path(uri, mtime)
 
-    if data == nil # indicates the server returned 304 Not Modified
+    if data.nil? # indicates the server returned 304 Not Modified
       return Gem.read_binary(path)
     end
 

lib/rubygems/requirement.rb

@@ -190,7 +190,7 @@ class Gem::Requirement
   end
 
   def hash # :nodoc:
-    requirements.sort.hash
+    requirements.map {|r| r.first == "~>" ? [r[0], r[1].to_s] : r }.sort.hash
   end
 
   def marshal_dump # :nodoc:

lib/rubygems/resolver/index_specification.rb

@@ -35,9 +35,12 @@ class Gem::Resolver::IndexSpecification < Gem::Resolver::Specification
 
   ##
   # The required_ruby_version constraint for this specification
+  #
+  # A fallback is included because when generated, some marshalled specs have it
+  # set to +nil+.
 
   def required_ruby_version
-    spec.required_ruby_version
+    spec.required_ruby_version || Gem::Requirement.default
   end
 
   ##

lib/rubygems/resolver/molinillo/lib/molinillo/delegates/specification_provider.rb

@@ -26,6 +26,13 @@ module Gem::Resolver::Molinillo
         end
       end
 
+      # (see Gem::Resolver::Molinillo::SpecificationProvider#dependencies_equal?)
+      def dependencies_equal?(dependencies, other_dependencies)
+        with_no_such_dependency_error_handling do
+          specification_provider.dependencies_equal?(dependencies, other_dependencies)
+        end
+      end
+
       # (see Gem::Resolver::Molinillo::SpecificationProvider#name_for)
       def name_for(dependency)
         with_no_such_dependency_error_handling do

lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'set'
 require 'tsort'
 
 require_relative 'dependency_graph/log'

lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/vertex.rb

@@ -59,7 +59,7 @@ module Gem::Resolver::Molinillo
       # @param [Set<Vertex>] vertices the set to add the predecessors to
       # @return [Set<Vertex>] the vertices of {#graph} where `self` is a
       #   {#descendent?}
-      def _recursive_predecessors(vertices = Set.new)
+      def _recursive_predecessors(vertices = new_vertex_set)
         incoming_edges.each do |edge|
           vertex = edge.origin
           next unless vertices.add?(vertex)
@@ -85,7 +85,7 @@ module Gem::Resolver::Molinillo
       # @param [Set<Vertex>] vertices the set to add the successors to
       # @return [Set<Vertex>] the vertices of {#graph} where `self` is an
       #   {#ancestor?}
-      def _recursive_successors(vertices = Set.new)
+      def _recursive_successors(vertices = new_vertex_set)
         outgoing_edges.each do |edge|
           vertex = edge.destination
           next unless vertices.add?(vertex)
@@ -138,7 +138,7 @@ module Gem::Resolver::Molinillo
       # @param [Vertex] other the vertex to check if there's a path to
       # @param [Set<Vertex>] visited the vertices of {#graph} that have been visited
       # @return [Boolean] whether there is a path to `other` from `self`
-      def _path_to?(other, visited = Set.new)
+      def _path_to?(other, visited = new_vertex_set)
         return false unless visited.add?(self)
         return true if equal?(other)
         successors.any? { |v| v._path_to?(other, visited) }
@@ -147,12 +147,18 @@ module Gem::Resolver::Molinillo
 
       # Is there a path from `other` to `self` following edges in the
       # dependency graph?
-      # @return true iff there is a path following edges within this {#graph}
+      # @return whether there is a path following edges within this {#graph}
       def ancestor?(other)
         other.path_to?(self)
       end
 
       alias is_reachable_from? ancestor?
+
+      def new_vertex_set
+        require 'set'
+        Set.new
+      end
+      private :new_vertex_set
     end
   end
 end

lib/rubygems/resolver/molinillo/lib/molinillo/errors.rb

@@ -121,7 +121,7 @@ module Gem::Resolver::Molinillo
           t = ''.dup
           depth = 2
           tree.each do |req|
-            t << '  ' * depth << req.to_s
+            t << '  ' * depth << printable_requirement.call(req)
             unless tree.last == req
               if spec = conflict.activated_by_name[name_for(req)]
                 t << %( was resolved to #{version_for_spec.call(spec)}, which)

lib/rubygems/resolver/molinillo/lib/molinillo/modules/specification_provider.rb

@@ -45,6 +45,17 @@ module Gem::Resolver::Molinillo
       true
     end
 
+    # Determines whether two arrays of dependencies are equal, and thus can be
+    # grouped.
+    #
+    # @param [Array<Object>] dependencies
+    # @param [Array<Object>] other_dependencies
+    # @return [Boolean] whether `dependencies` and `other_dependencies` should
+    #   be considered equal.
+    def dependencies_equal?(dependencies, other_dependencies)
+      dependencies == other_dependencies
+    end
+
     # Returns the name for the given `dependency`.
     # @note This method should be 'pure', i.e. the return value should depend
     #   only on the `dependency` parameter.

lib/rubygems/resolver/molinillo/lib/molinillo/resolution.rb

@@ -329,11 +329,11 @@ module Gem::Resolver::Molinillo
 
         # Look for past conflicts that could be unwound to affect the
         # requirement tree for the current conflict
+        all_reqs = last_detail_for_current_unwind.all_requirements
+        all_reqs_size = all_reqs.size
         relevant_unused_unwinds = unused_unwind_options.select do |alternative|
-          intersecting_requirements =
-            last_detail_for_current_unwind.all_requirements &
-            alternative.requirements_unwound_to_instead
-          next if intersecting_requirements.empty?
+          diff_reqs = all_reqs - alternative.requirements_unwound_to_instead
+          next if diff_reqs.size == all_reqs_size
           # Find the highest index unwind whilst looping through
           current_detail = alternative if alternative > current_detail
           alternative
@@ -344,8 +344,12 @@ module Gem::Resolver::Molinillo
         state.unused_unwind_options += unwind_details.reject { |detail| detail.state_index == -1 }
 
         # Update the requirements_unwound_to_instead on any relevant unused unwinds
-        relevant_unused_unwinds.each { |d| d.requirements_unwound_to_instead << current_detail.state_requirement }
-        unwind_details.each { |d| d.requirements_unwound_to_instead << current_detail.state_requirement }
+        relevant_unused_unwinds.each do |d|
+          (d.requirements_unwound_to_instead << current_detail.state_requirement).uniq!
+        end
+        unwind_details.each do |d|
+          (d.requirements_unwound_to_instead << current_detail.state_requirement).uniq!
+        end
 
         current_detail
       end
@@ -803,7 +807,7 @@ module Gem::Resolver::Molinillo
 
         possibilities.reverse_each do |possibility|
           dependencies = dependencies_for(possibility)
-          if current_possibility_set && current_possibility_set.dependencies == dependencies
+          if current_possibility_set && dependencies_equal?(current_possibility_set.dependencies, dependencies)
             current_possibility_set.possibilities.unshift(possibility)
           else
             possibility_sets.unshift(PossibilitySet.new(dependencies, [possibility]))

lib/rubygems/specification.rb

@@ -182,6 +182,7 @@ class Gem::Specification < Gem::BasicSpecification
     @@default_value[k].nil?
   end
 
+  @@stubs = nil
   @@stubs_by_name = {}
 
   # Sentinel object to represent "not found" stubs
@@ -665,6 +666,9 @@ class Gem::Specification < Gem::BasicSpecification
   #
   #  # Only prereleases or final releases after 2.6.0.preview2
   #  spec.required_ruby_version = '> 2.6.0.preview2'
+  #
+  #  # This gem will work with 2.3.0 or greater, including major version 3, but lesser than 4.0.0
+  #  spec.required_ruby_version = '>= 2.3', '< 4'
 
   def required_ruby_version=(req)
     @required_ruby_version = Gem::Requirement.create req
@@ -800,10 +804,8 @@ class Gem::Specification < Gem::BasicSpecification
   def self.stubs
     @@stubs ||= begin
       pattern = "*.gemspec"
-      stubs = installed_stubs(dirs, pattern) + default_stubs(pattern)
-      stubs = stubs.uniq {|stub| stub.full_name }
+      stubs = stubs_for_pattern(pattern, false)
 
-      _resort!(stubs)
       @@stubs_by_name = stubs.select {|s| Gem::Platform.match_spec? s }.group_by(&:name)
       stubs
     end
@@ -820,31 +822,40 @@ class Gem::Specification < Gem::BasicSpecification
     end
   end
 
-  EMPTY = [].freeze # :nodoc:
-
   ##
   # Returns a Gem::StubSpecification for installed gem named +name+
   # only returns stubs that match Gem.platforms
 
   def self.stubs_for(name)
-    if @@stubs_by_name[name]
-      @@stubs_by_name[name]
+    if @@stubs
+      @@stubs_by_name[name] || []
     else
-      pattern = "#{name}-*.gemspec"
-      stubs = installed_stubs(dirs, pattern).select {|s| Gem::Platform.match_spec? s } + default_stubs(pattern)
-      stubs = stubs.uniq {|stub| stub.full_name }.group_by(&:name)
-      stubs.each_value {|v| _resort!(v) }
-
-      @@stubs_by_name.merge! stubs
-      @@stubs_by_name[name] ||= EMPTY
+      @@stubs_by_name[name] ||= stubs_for_pattern("#{name}-*.gemspec").select do |s|
+        s.name == name
+      end
     end
   end
 
+  ##
+  # Finds stub specifications matching a pattern from the standard locations,
+  # optionally filtering out specs not matching the current platform
+  #
+  def self.stubs_for_pattern(pattern, match_platform = true) # :nodoc:
+    installed_stubs = installed_stubs(Gem::Specification.dirs, pattern)
+    installed_stubs.select! {|s| Gem::Platform.match_spec? s } if match_platform
+    stubs = installed_stubs + default_stubs(pattern)
+    stubs = stubs.uniq {|stub| stub.full_name }
+    _resort!(stubs)
+    stubs
+  end
+
   def self._resort!(specs) # :nodoc:
     specs.sort! do |a, b|
       names = a.name <=> b.name
       next names if names.nonzero?
-      b.version <=> a.version
+      versions = b.version <=> a.version
+      next versions if versions.nonzero?
+      b.platform == Gem::Platform::RUBY ? -1 : 1
     end
   end
 
@@ -1080,20 +1091,15 @@ class Gem::Specification < Gem::BasicSpecification
   end
 
   def self._latest_specs(specs, prerelease = false) # :nodoc:
-    result = Hash.new {|h,k| h[k] = {} }
-    native = {}
+    result = {}
 
     specs.reverse_each do |spec|
       next if spec.version.prerelease? unless prerelease
 
-      native[spec.name] = spec.version if spec.platform == Gem::Platform::RUBY
-      result[spec.name][spec.platform] = spec
+      result[spec.name] = spec
     end
 
-    result.map(&:last).map(&:values).flatten.reject do |spec|
-      minimum = native[spec.name]
-      minimum && spec.version < minimum
-    end.sort_by{|tup| tup.name }
+    result.map(&:last).flatten.sort_by{|tup| tup.name }
   end
 
   ##
@@ -1552,7 +1558,6 @@ class Gem::Specification < Gem::BasicSpecification
   def build_extensions # :nodoc:
     return if default_gem?
     return if extensions.empty?
-    return if installed_by_version < Gem::Version.new('2.2.0.preview.2')
     return if File.exist? gem_build_complete_path
     return if !File.writable?(base_dir)
     return if !File.exist?(File.join(base_dir, 'extensions'))
@@ -2123,7 +2128,6 @@ class Gem::Specification < Gem::BasicSpecification
   def missing_extensions?
     return false if default_gem?
     return false if extensions.empty?
-    return false if installed_by_version < Gem::Version.new('2.2.0.preview.2')
     return false if File.exist? gem_build_complete_path
 
     true
@@ -2548,7 +2552,7 @@ class Gem::Specification < Gem::BasicSpecification
     begin
       dependencies.each do |dep|
         next unless dep.runtime?
-        dep.to_specs.each do |dep_spec|
+        dep.matching_specs(true).each do |dep_spec|
           next if visited.has_key?(dep_spec)
           visited[dep_spec] = true
           trail.push(dep_spec)

lib/rubygems/test_case.rb

@@ -301,7 +301,9 @@ class Gem::TestCase < Minitest::Test
 
   def setup
     @orig_env = ENV.to_hash
-    @tmp = Dir.mktmpdir("tmp", Dir.pwd)
+    @tmp = File.expand_path("tmp")
+
+    FileUtils.mkdir_p @tmp
 
     ENV['GEM_VENDOR'] = nil
     ENV['GEMRC'] = nil
@@ -310,7 +312,6 @@ class Gem::TestCase < Minitest::Test
     ENV['XDG_DATA_HOME'] = nil
     ENV['SOURCE_DATE_EPOCH'] = nil
     ENV['BUNDLER_VERSION'] = nil
-    ENV["TMPDIR"] = @tmp
 
     @current_dir = Dir.pwd
     @fetcher     = nil
@@ -321,13 +322,10 @@ class Gem::TestCase < Minitest::Test
     # capture output
     Gem::DefaultUserInteraction.ui = Gem::MockGemUi.new
 
-    tmpdir = File.realpath Dir.tmpdir
-    tmpdir.tap(&Gem::UNTAINT)
-
-    @tempdir = File.join(tmpdir, "test_rubygems_#{$$}")
+    @tempdir = Dir.mktmpdir("test_rubygems_", @tmp)
     @tempdir.tap(&Gem::UNTAINT)
 
-    FileUtils.mkdir_p @tempdir
+    ENV["TMPDIR"] = @tempdir
 
     @orig_SYSTEM_WIDE_CONFIG_FILE = Gem::ConfigFile::SYSTEM_WIDE_CONFIG_FILE
     Gem::ConfigFile.send :remove_const, :SYSTEM_WIDE_CONFIG_FILE
@@ -366,7 +364,9 @@ class Gem::TestCase < Minitest::Test
     Dir.chdir @tempdir
 
     ENV['HOME'] = @userhome
+    Gem.instance_variable_set :@config_file, nil
     Gem.instance_variable_set :@user_home, nil
+    Gem.instance_variable_set :@config_home, nil
     Gem.instance_variable_set :@data_home, nil
     Gem.instance_variable_set :@gemdeps, nil
     Gem.instance_variable_set :@env_requirements_by_name, nil
@@ -449,7 +449,6 @@ class Gem::TestCase < Minitest::Test
     Dir.chdir @current_dir
 
     FileUtils.rm_rf @tempdir
-    FileUtils.rm_rf @tmp
 
     ENV.replace(@orig_env)