Index: lib/open-uri.rb =================================================================== --- lib/open-uri.rb (revision 36240) +++ lib/open-uri.rb (working copy) @@ -692,84 +692,6 @@ module OpenURI end module URI - class Generic - # returns a proxy URI. - # The proxy URI is obtained from environment variables such as http_proxy, - # ftp_proxy, no_proxy, etc. - # If there is no proper proxy, nil is returned. - # - # Note that capitalized variables (HTTP_PROXY, FTP_PROXY, NO_PROXY, etc.) - # are examined too. - # - # But http_proxy and HTTP_PROXY is treated specially under CGI environment. - # It's because HTTP_PROXY may be set by Proxy: header. - # So HTTP_PROXY is not used. - # http_proxy is not used too if the variable is case insensitive. - # CGI_HTTP_PROXY can be used instead. - def find_proxy - name = self.scheme.downcase + '_proxy' - proxy_uri = nil - if name == 'http_proxy' && ENV.include?('REQUEST_METHOD') # CGI? - # HTTP_PROXY conflicts with *_proxy for proxy settings and - # HTTP_* for header information in CGI. - # So it should be careful to use it. - pairs = ENV.reject {|k, v| /\Ahttp_proxy\z/i !~ k } - case pairs.length - when 0 # no proxy setting anyway. - proxy_uri = nil - when 1 - k, _ = pairs.shift - if k == 'http_proxy' && ENV[k.upcase] == nil - # http_proxy is safe to use because ENV is case sensitive. - proxy_uri = ENV[name] - else - proxy_uri = nil - end - else # http_proxy is safe to use because ENV is case sensitive. - proxy_uri = ENV.to_hash[name] - end - if !proxy_uri - # Use CGI_HTTP_PROXY. cf. libwww-perl. - proxy_uri = ENV["CGI_#{name.upcase}"] - end - elsif name == 'http_proxy' - unless proxy_uri = ENV[name] - if proxy_uri = ENV[name.upcase] - warn 'The environment variable HTTP_PROXY is discouraged. Use http_proxy.' - end - end - else - proxy_uri = ENV[name] || ENV[name.upcase] - end - - if proxy_uri && self.hostname - require 'socket' - begin - addr = IPSocket.getaddress(self.hostname) - proxy_uri = nil if /\A127\.|\A::1\z/ =~ addr - rescue SocketError - end - end - - if proxy_uri - proxy_uri = URI.parse(proxy_uri) - name = 'no_proxy' - if no_proxy = ENV[name] || ENV[name.upcase] - no_proxy.scan(/([^:,]*)(?::(\d+))?/) {|host, port| - if /(\A|\.)#{Regexp.quote host}\z/i =~ self.host && - (!port || self.port == port.to_i) - proxy_uri = nil - break - end - } - end - proxy_uri - else - nil - end - end - end - class HTTP def buffer_open(buf, proxy, options) # :nodoc: OpenURI.open_http(buf, self, proxy, options) Index: lib/uri/generic.rb =================================================================== --- lib/uri/generic.rb (revision 36240) +++ lib/uri/generic.rb (working copy) @@ -1596,5 +1596,81 @@ module URI return oth, self end + + # returns a proxy URI. + # The proxy URI is obtained from environment variables such as http_proxy, + # ftp_proxy, no_proxy, etc. + # If there is no proper proxy, nil is returned. + # + # Note that capitalized variables (HTTP_PROXY, FTP_PROXY, NO_PROXY, etc.) + # are examined too. + # + # But http_proxy and HTTP_PROXY is treated specially under CGI environment. + # It's because HTTP_PROXY may be set by Proxy: header. + # So HTTP_PROXY is not used. + # http_proxy is not used too if the variable is case insensitive. + # CGI_HTTP_PROXY can be used instead. + def find_proxy + name = self.scheme.downcase + '_proxy' + proxy_uri = nil + if name == 'http_proxy' && ENV.include?('REQUEST_METHOD') # CGI? + # HTTP_PROXY conflicts with *_proxy for proxy settings and + # HTTP_* for header information in CGI. + # So it should be careful to use it. + pairs = ENV.reject {|k, v| /\Ahttp_proxy\z/i !~ k } + case pairs.length + when 0 # no proxy setting anyway. + proxy_uri = nil + when 1 + k, _ = pairs.shift + if k == 'http_proxy' && ENV[k.upcase] == nil + # http_proxy is safe to use because ENV is case sensitive. + proxy_uri = ENV[name] + else + proxy_uri = nil + end + else # http_proxy is safe to use because ENV is case sensitive. + proxy_uri = ENV.to_hash[name] + end + if !proxy_uri + # Use CGI_HTTP_PROXY. cf. libwww-perl. + proxy_uri = ENV["CGI_#{name.upcase}"] + end + elsif name == 'http_proxy' + unless proxy_uri = ENV[name] + if proxy_uri = ENV[name.upcase] + warn 'The environment variable HTTP_PROXY is discouraged. Use http_proxy.' + end + end + else + proxy_uri = ENV[name] || ENV[name.upcase] + end + + if proxy_uri && self.hostname + require 'socket' + begin + addr = IPSocket.getaddress(self.hostname) + proxy_uri = nil if /\A127\.|\A::1\z/ =~ addr + rescue SocketError + end + end + + if proxy_uri + proxy_uri = URI.parse(proxy_uri) + name = 'no_proxy' + if no_proxy = ENV[name] || ENV[name.upcase] + no_proxy.scan(/([^:,]*)(?::(\d+))?/) {|host, port| + if /(\A|\.)#{Regexp.quote host}\z/i =~ self.host && + (!port || self.port == port.to_i) + proxy_uri = nil + break + end + } + end + proxy_uri + else + nil + end + end end end Index: lib/net/http.rb =================================================================== --- lib/net/http.rb (revision 36240) +++ lib/net/http.rb (working copy) @@ -267,20 +267,20 @@ module Net #:nodoc: # # === Proxies # - # Net::HTTP::Proxy has the same methods as Net::HTTP but its instances always - # connect via the proxy instead of directly to the given host. + # Net::HTTP will automatically create a proxy from the +http_proxy+ + # environment variable if it is present. To disable use of +http_proxy+, + # pass +nil+ for the proxy address. + # + # You may also create a custom proxy: # # proxy_addr = 'your.proxy.host' # proxy_port = 8080 # - # Net::HTTP::Proxy(proxy_addr, proxy_port).start('www.example.com') {|http| - # # always connect to your.proxy.addr:8080 + # Net::HTTP.new('example.com', nil, proxy_addr, proxy_port).start { |http| + # # always proxy via your.proxy.addr:8080 # } # - # Net::HTTP::Proxy returns a Net::HTTP instance when proxy_addr is nil so - # there is no need for conditional code. - # - # See Net::HTTP::Proxy for further details and examples such as proxies that + # See Net::HTTP.new for further details and examples such as proxies that # require a username and password. # # == HTTP Request Classes @@ -569,16 +569,44 @@ module Net #:nodoc: http.start(&block) end - class << HTTP - alias newobj new - end - # Creates a new Net::HTTP object without opening a TCP connection or # HTTP session. - # The +address+ should be a DNS hostname or IP address. - # If +p_addr+ is given, creates a Net::HTTP object with proxy support. - def HTTP.new(address, port = nil, p_addr = nil, p_port = nil, p_user = nil, p_pass = nil) - Proxy(p_addr, p_port, p_user, p_pass).newobj(address, port) + # + # The +address+ should be a DNS hostname or IP address, the +port+ is the + # port the server operates on. If no +port+ is given the default port for + # HTTP or HTTPS is used. + # + # If none of the +p_+ arguments are given, the proxy host and port are + # taken from the +http_proxy+ environment variable (or its uppercase + # equivalent) if present. If the proxy requires authentication you must + # supply it by hand. See URI::Generic#find_proxy for details of proxy + # detection from the environment. To disable proxy detection set +p_addr+ + # to nil. + # + # If you are connecting to a custom proxy, +p_addr+ the DNS name or IP + # address of the proxy host, +p_port+ the port to use to access the proxy, + # and +p_user+ and +p_pass+ the username and password if authorization is + # required to use the proxy. + # + def HTTP.new(address, port = nil, p_addr = :ENV, p_port = nil, p_user = nil, p_pass = nil) + http = super address, port + + if proxy_class? then # from Net::HTTP::Proxy() + http.proxy_from_env = @proxy_from_env + http.proxy_address = @proxy_address + http.proxy_port = @proxy_port + http.proxy_user = @proxy_user + http.proxy_pass = @proxy_pass + elsif p_addr == :ENV then + http.proxy_from_env = true + else + http.proxy_address = p_addr + http.proxy_port = p_port + http.proxy_user = p_user + http.proxy_pass = p_pass + end + + http end # Creates a new Net::HTTP object for the specified server address, @@ -597,6 +625,14 @@ module Net #:nodoc: @read_timeout = 60 @continue_timeout = nil @debug_output = nil + + @proxy_from_env = false + @proxy_uri = nil + @proxy_address = nil + @proxy_port = nil + @proxy_user = nil + @proxy_pass = nil + @use_ssl = false @ssl_context = nil @enable_post_connection_check = true @@ -631,6 +667,12 @@ module Net #:nodoc: # The port number to connect to. attr_reader :port + attr_writer :proxy_from_env + attr_writer :proxy_address + attr_writer :proxy_port + attr_writer :proxy_user + attr_writer :proxy_pass + # Number of seconds to wait for the connection to open. Any number # may be used, including Floats for fractional seconds. If the HTTP # object cannot open a connection in this many seconds, it raises a @@ -797,9 +839,17 @@ module Net #:nodoc: private :do_start def connect - D "opening connection to #{conn_address()}..." + if proxy? then + conn_address = proxy_address + conn_port = proxy_port + else + conn_address = address + conn_port = port + end + + D "opening connection to #{conn_address}..." s = Timeout.timeout(@open_timeout, Net::OpenTimeout) { - TCPSocket.open(conn_address(), conn_port()) + TCPSocket.open(conn_address, conn_port) } D "opened" if use_ssl? @@ -876,6 +926,7 @@ module Net #:nodoc: # no proxy @is_proxy_class = false + @proxy_from_env = false @proxy_addr = nil @proxy_port = nil @proxy_user = nil @@ -884,52 +935,26 @@ module Net #:nodoc: # Creates an HTTP proxy class which behaves like Net::HTTP, but # performs all access via the specified proxy. # - # The arguments are the DNS name or IP address of the proxy host, - # the port to use to access the proxy, and a username and password - # if authorization is required to use the proxy. - # - # You can replace any use of the Net::HTTP class with use of the - # proxy class created. - # - # If +p_addr+ is nil, this method returns self (a Net::HTTP object). - # - # # Example - # proxy_class = Net::HTTP::Proxy('proxy.example.com', 8080) - # - # proxy_class.start('www.ruby-lang.org') {|http| - # # connecting proxy.foo.org:8080 - # } - # - # You may use them to work with authorization-enabled proxies: - # - # proxy_host = 'your.proxy.example' - # proxy_port = 8080 - # proxy_user = 'user' - # proxy_pass = 'pass' - # - # proxy = Net::HTTP::Proxy(proxy_host, proxy_port, proxy_user, proxy_pass) - # proxy.start('www.example.com') { |http| - # # always connect to your.proxy.example:8080 using specified username - # # and password - # } - # - # Note that net/http does not use the HTTP_PROXY environment variable. - # If you want to use a proxy, you must set it explicitly. - # - def HTTP.Proxy(p_addr, p_port = nil, p_user = nil, p_pass = nil) + # This class is obsolete. You may pass these same parameters directly to + # Net::HTTP.new. See Net::HTTP.new for details of the arguments. + def HTTP.Proxy(p_addr = :ENV, p_port = nil, p_user = nil, p_pass = nil) return self unless p_addr - delta = ProxyDelta - proxyclass = Class.new(self) - proxyclass.module_eval { - include delta - # with proxy + + Class.new(self) { @is_proxy_class = true - @proxy_address = p_addr - @proxy_port = p_port || default_port() - @proxy_user = p_user - @proxy_pass = p_pass + + if p_addr == :ENV then + @proxy_from_env = true + @proxy_address = nil + @proxy_port = nil + else + @proxy_address = p_addr + @proxy_port = p_port + end + + @proxy_user = p_user + @proxy_pass = p_pass } - proxyclass end class << HTTP @@ -952,29 +977,51 @@ module Net #:nodoc: attr_reader :proxy_pass end - # True if self is a HTTP proxy class. + # True if requests for this connection will be proxied def proxy? - self.class.proxy_class? + if @proxy_from_env then + proxy_uri + else + @proxy_address + end end - # A convenience method for accessing value of proxy_address from Net::HTTP. + # True if the proxy for this connection is determined from the environment + def proxy_from_env? + @proxy_from_env + end + + # The proxy URI determined from the environment for this connection. + def proxy_uri # :nodoc: + @proxy_uri ||= URI("http://#{address}:#{port}").find_proxy + end + + # The address of the proxy server, if one is configured. def proxy_address - self.class.proxy_address + if @proxy_from_env then + proxy_uri.hostname + else + @proxy_address + end end - # A convenience method for accessing value of proxy_port from Net::HTTP. + # The port of the proxy server, if one is configured. def proxy_port - self.class.proxy_port + if @proxy_from_env then + proxy_uri.port + else + @proxy_port + end end - # A convenience method for accessing value of proxy_user from Net::HTTP. + # The proxy username, if one is configured def proxy_user - self.class.proxy_user + @proxy_user end - # A convenience method for accessing value of proxy_pass from Net::HTTP. + # The proxy password, if one is configured def proxy_pass - self.class.proxy_pass + @proxy_pass end alias proxyaddr proxy_address #:nodoc: obsolete @@ -982,18 +1029,22 @@ module Net #:nodoc: private - # without proxy + # without proxy, obsolete - def conn_address + def conn_address # :nodoc: address() end - def conn_port + def conn_port # :nodoc: port() end def edit_path(path) - path + if proxy? and not use_ssl? then + "http://#{addr_port}#{path}" + else + path + end end # Index: test/open-uri/test_open-uri.rb =================================================================== --- test/open-uri/test_open-uri.rb (revision 36240) +++ test/open-uri/test_open-uri.rb (working copy) @@ -504,41 +504,6 @@ class TestOpenURI < Test::Unit::TestCase # 192.0.2.0/24 is TEST-NET. [RFC3330] - def test_find_proxy - assert_nil(URI("http://192.0.2.1/").find_proxy) - assert_nil(URI("ftp://192.0.2.1/").find_proxy) - with_env('http_proxy'=>'http://127.0.0.1:8080') { - assert_equal(URI('http://127.0.0.1:8080'), URI("http://192.0.2.1/").find_proxy) - assert_nil(URI("ftp://192.0.2.1/").find_proxy) - } - with_env('ftp_proxy'=>'http://127.0.0.1:8080') { - assert_nil(URI("http://192.0.2.1/").find_proxy) - assert_equal(URI('http://127.0.0.1:8080'), URI("ftp://192.0.2.1/").find_proxy) - } - with_env('REQUEST_METHOD'=>'GET') { - assert_nil(URI("http://192.0.2.1/").find_proxy) - } - with_env('CGI_HTTP_PROXY'=>'http://127.0.0.1:8080', 'REQUEST_METHOD'=>'GET') { - assert_equal(URI('http://127.0.0.1:8080'), URI("http://192.0.2.1/").find_proxy) - } - with_env('http_proxy'=>'http://127.0.0.1:8080', 'no_proxy'=>'192.0.2.2') { - assert_equal(URI('http://127.0.0.1:8080'), URI("http://192.0.2.1/").find_proxy) - assert_nil(URI("http://192.0.2.2/").find_proxy) - } - end - - def test_find_proxy_case_sensitive_env - with_env('http_proxy'=>'http://127.0.0.1:8080', 'REQUEST_METHOD'=>'GET') { - assert_equal(URI('http://127.0.0.1:8080'), URI("http://192.0.2.1/").find_proxy) - } - with_env('HTTP_PROXY'=>'http://127.0.0.1:8081', 'REQUEST_METHOD'=>'GET') { - assert_nil(nil, URI("http://192.0.2.1/").find_proxy) - } - with_env('http_proxy'=>'http://127.0.0.1:8080', 'HTTP_PROXY'=>'http://127.0.0.1:8081', 'REQUEST_METHOD'=>'GET') { - assert_equal(URI('http://127.0.0.1:8080'), URI("http://192.0.2.1/").find_proxy) - } - end unless RUBY_PLATFORM =~ /mswin|mingw/ - def test_ftp_invalid_request assert_raise(ArgumentError) { URI("ftp://127.0.0.1/").read } assert_raise(ArgumentError) { URI("ftp://127.0.0.1/a%0Db").read } Index: test/uri/test_generic.rb =================================================================== --- test/uri/test_generic.rb (revision 36240) +++ test/uri/test_generic.rb (working copy) @@ -732,4 +732,53 @@ class URI::TestGeneric < Test::Unit::Tes URI::Generic.build2(path: "/foo bar/baz") URI::Generic.build2(['http', nil, 'example.com', 80, nil, '/foo bar' , nil, nil, nil]) end + + # 192.0.2.0/24 is TEST-NET. [RFC3330] + + def test_find_proxy + assert_nil(URI("http://192.0.2.1/").find_proxy) + assert_nil(URI("ftp://192.0.2.1/").find_proxy) + with_env('http_proxy'=>'http://127.0.0.1:8080') { + assert_equal(URI('http://127.0.0.1:8080'), URI("http://192.0.2.1/").find_proxy) + assert_nil(URI("ftp://192.0.2.1/").find_proxy) + } + with_env('ftp_proxy'=>'http://127.0.0.1:8080') { + assert_nil(URI("http://192.0.2.1/").find_proxy) + assert_equal(URI('http://127.0.0.1:8080'), URI("ftp://192.0.2.1/").find_proxy) + } + with_env('REQUEST_METHOD'=>'GET') { + assert_nil(URI("http://192.0.2.1/").find_proxy) + } + with_env('CGI_HTTP_PROXY'=>'http://127.0.0.1:8080', 'REQUEST_METHOD'=>'GET') { + assert_equal(URI('http://127.0.0.1:8080'), URI("http://192.0.2.1/").find_proxy) + } + with_env('http_proxy'=>'http://127.0.0.1:8080', 'no_proxy'=>'192.0.2.2') { + assert_equal(URI('http://127.0.0.1:8080'), URI("http://192.0.2.1/").find_proxy) + assert_nil(URI("http://192.0.2.2/").find_proxy) + } + end + + def test_find_proxy_case_sensitive_env + with_env('http_proxy'=>'http://127.0.0.1:8080', 'REQUEST_METHOD'=>'GET') { + assert_equal(URI('http://127.0.0.1:8080'), URI("http://192.0.2.1/").find_proxy) + } + with_env('HTTP_PROXY'=>'http://127.0.0.1:8081', 'REQUEST_METHOD'=>'GET') { + assert_nil(nil, URI("http://192.0.2.1/").find_proxy) + } + with_env('http_proxy'=>'http://127.0.0.1:8080', 'HTTP_PROXY'=>'http://127.0.0.1:8081', 'REQUEST_METHOD'=>'GET') { + assert_equal(URI('http://127.0.0.1:8080'), URI("http://192.0.2.1/").find_proxy) + } + end unless RUBY_PLATFORM =~ /mswin|mingw/ + + def with_env(h) + begin + old = {} + h.each_key {|k| old[k] = ENV[k] } + h.each {|k, v| ENV[k] = v } + yield + ensure + h.each_key {|k| ENV[k] = old[k] } + end + end + end Index: test/net/http/test_http.rb =================================================================== --- test/net/http/test_http.rb (revision 36240) +++ test/net/http/test_http.rb (working copy) @@ -5,6 +5,167 @@ require 'net/http' require 'stringio' require_relative 'utils' +class TestNetHTTP < Test::Unit::TestCase + + def test_class_Proxy + no_proxy_class = Net::HTTP.Proxy nil + + assert_equal Net::HTTP, no_proxy_class + + proxy_class = Net::HTTP.Proxy 'proxy.example', 8000, 'user', 'pass' + + refute_equal Net::HTTP, proxy_class + + assert_operator proxy_class, :<, Net::HTTP + + assert_equal 'proxy.example', proxy_class.proxy_address + assert_equal 8000, proxy_class.proxy_port + assert_equal 'user', proxy_class.proxy_user + assert_equal 'pass', proxy_class.proxy_pass + + http = proxy_class.new 'example' + + refute http.proxy_from_env? + end + + def test_class_Proxy_from_ENV + clean_http_proxy_env do + ENV['http_proxy'] = 'http://proxy.example:8000' + + # These are ignored on purpose. See Bug 4388 and Feature 6546 + ENV['http_proxy_user'] = 'user' + ENV['http_proxy_pass'] = 'pass' + + proxy_class = Net::HTTP.Proxy :ENV + + refute_equal Net::HTTP, proxy_class + + assert_operator proxy_class, :<, Net::HTTP + + assert_nil proxy_class.proxy_address + assert_nil proxy_class.proxy_user + assert_nil proxy_class.proxy_pass + + refute_equal 8000, proxy_class.proxy_port + + http = proxy_class.new 'example' + + assert http.proxy_from_env? + end + end + + def test_edit_path + http = Net::HTTP.new 'example', nil, nil + + edited = http.send :edit_path, '/path' + + assert_equal '/path', edited + + http.use_ssl = true + + edited = http.send :edit_path, '/path' + + assert_equal '/path', edited + end + + def test_edit_path_proxy + http = Net::HTTP.new 'example', nil, 'proxy.example' + + edited = http.send :edit_path, '/path' + + assert_equal 'http://example/path', edited + + http.use_ssl = true + + edited = http.send :edit_path, '/path' + + assert_equal '/path', edited + end + + def test_proxy_address + http = Net::HTTP.new 'example', nil, 'proxy.example' + + assert_equal 'proxy.example', http.proxy_address + end + + def test_proxy_address_ENV + clean_http_proxy_env do + ENV['http_proxy'] = 'http://proxy.example:8000' + + http = Net::HTTP.new 'example' + + assert_equal 'proxy.example', http.proxy_address + end + end + + def test_proxy_eh_no_proxy + clean_http_proxy_env do + refute Net::HTTP.new('example', nil, nil).proxy? + end + end + + def test_proxy_eh_ENV + clean_http_proxy_env do + ENV['http_proxy'] = 'http://proxy.example:8000' + + http = Net::HTTP.new 'example' + + assert http.proxy? + end + end + + def test_proxy_eh_ENV_none_set + clean_http_proxy_env do + refute Net::HTTP.new('example').proxy? + end + end + + def test_proxy_eh_ENV_no_proxy + clean_http_proxy_env do + ENV['http_proxy'] = 'http://proxy.example:8000' + ENV['no_proxy'] = 'example' + + refute Net::HTTP.new('example').proxy? + end + end + + def test_proxy_port + http = Net::HTTP.new 'exmaple', nil, 'proxy.example', 8000 + + assert_equal 8000, http.proxy_port + end + + def test_proxy_port_ENV + clean_http_proxy_env do + ENV['http_proxy'] = 'http://proxy.example:8000' + + http = Net::HTTP.new 'example' + + assert_equal 8000, http.proxy_port + end + end + + def clean_http_proxy_env + orig = { + 'http_proxy' => ENV['http_proxy'], + 'http_proxy_user' => ENV['http_proxy_user'], + 'http_proxy_pass' => ENV['http_proxy_pass'], + 'no_proxy' => ENV['no_proxy'], + } + + orig.each_key do |key| + ENV.delete key + end + + yield + ensure + orig.each do |key, value| + ENV[key] = value + end + end + +end + module TestNetHTTP_version_1_1_methods def test_s_get