Feature #6492 » net.http.inflate_by_default.3.patch
| lib/net/http/response.rb (working copy) | ||
|---|---|---|
|
private
|
||
|
def read_body_0(dest)
|
||
|
if chunked?
|
||
|
read_chunked dest
|
||
|
return
|
||
|
end
|
||
|
clen = content_length()
|
||
|
if clen
|
||
|
@socket.read clen, dest, true # ignore EOF
|
||
|
return
|
||
|
##
|
||
|
# Checks for a supported Content-Encoding header and yields an Inflate
|
||
|
# wrapper for this response's socket when zlib is present. If the
|
||
|
# Content-Encoding is unsupported or zlib is missing the plain socket is
|
||
|
# yielded.
|
||
|
#
|
||
|
# If a Content-Range header is present a plain socket is yielded as the
|
||
|
# bytes in the range may not be a complete deflate block.
|
||
|
def inflater # :nodoc:
|
||
|
return yield @socket unless Net::HTTP::HAVE_ZLIB
|
||
|
return yield @socket if self['content-range']
|
||
|
case self['content-encoding']
|
||
|
when 'deflate', 'gzip', 'x-gzip' then
|
||
|
self.delete 'content-encoding'
|
||
|
inflate_body_io = Inflater.new(@socket)
|
||
|
begin
|
||
|
yield inflate_body_io
|
||
|
ensure
|
||
|
inflate_body_io.finish
|
||
|
end
|
||
|
when 'none', 'identity' then
|
||
|
self.delete 'content-encoding'
|
||
|
yield @socket
|
||
|
else
|
||
|
yield @socket
|
||
|
end
|
||
|
clen = range_length()
|
||
|
if clen
|
||
|
@socket.read clen, dest
|
||
|
return
|
||
|
end
|
||
|
def read_body_0(dest)
|
||
|
inflater do |inflate_body_io|
|
||
|
if chunked?
|
||
|
read_chunked dest, inflate_body_io
|
||
|
return
|
||
|
end
|
||
|
@socket = inflate_body_io
|
||
|
clen = content_length()
|
||
|
if clen
|
||
|
@socket.read clen, dest, true # ignore EOF
|
||
|
return
|
||
|
end
|
||
|
clen = range_length()
|
||
|
if clen
|
||
|
@socket.read clen, dest
|
||
|
return
|
||
|
end
|
||
|
@socket.read_all dest
|
||
|
end
|
||
|
@socket.read_all dest
|
||
|
end
|
||
|
def read_chunked(dest)
|
||
|
##
|
||
|
# read_chunked reads from +@socket+ for chunk-size, chunk-extension, CRLF,
|
||
|
# etc. and +chunk_data_io+ for chunk-data which may be deflate or gzip
|
||
|
# encoded.
|
||
|
#
|
||
|
# See RFC 2616 section 3.6.1 for definitions
|
||
|
def read_chunked(dest, chunk_data_io) # :nodoc:
|
||
|
len = nil
|
||
|
total = 0
|
||
|
while true
|
||
| ... | ... | |
|
len = hexlen.hex
|
||
|
break if len == 0
|
||
|
begin
|
||
|
@socket.read len, dest
|
||
|
chunk_data_io.read len, dest
|
||
|
ensure
|
||
|
total += len
|
||
|
@socket.read 2 # \r\n
|
||
| ... | ... | |
|
end
|
||
|
def procdest(dest, block)
|
||
|
raise ArgumentError, 'both arg and block given for HTTP method' \
|
||
|
if dest and block
|
||
|
raise ArgumentError, 'both arg and block given for HTTP method' if
|
||
|
dest and block
|
||
|
if block
|
||
|
Net::ReadAdapter.new(block)
|
||
|
else
|
||
| ... | ... | |
|
end
|
||
|
end
|
||
|
##
|
||
|
# Inflater is a wrapper around Net::BufferedIO that transparently inflates
|
||
|
# zlib and gzip streams.
|
||
|
class Inflater # :nodoc:
|
||
|
##
|
||
|
# Creates a new Inflater wrapping +socket+
|
||
|
def initialize socket
|
||
|
@socket = socket
|
||
|
# zlib with automatic gzip detection
|
||
|
@inflate = Zlib::Inflate.new(32 + Zlib::MAX_WBITS)
|
||
|
end
|
||
|
##
|
||
|
# Finishes the inflate stream.
|
||
|
def finish
|
||
|
@inflate.finish
|
||
|
end
|
||
|
##
|
||
|
# Returns a Net::ReadAdapter that inflates each read chunk into +dest+.
|
||
|
#
|
||
|
# This allows a large response body to be inflated without storing the
|
||
|
# entire body in memory.
|
||
|
def inflate_adapter(dest)
|
||
|
block = proc do |chunk|
|
||
|
dest << @inflate.inflate(chunk)
|
||
|
end
|
||
|
Net::ReadAdapter.new(block)
|
||
|
end
|
||
|
##
|
||
|
# Reads +clen+ bytes from the socket, inflates them, then writes them to
|
||
|
# +dest+. +ignore_eof+ is passed down to Net::BufferedIO#read
|
||
|
#
|
||
|
# Unlike Net::BufferedIO#read, this method returns more than +clen+ bytes.
|
||
|
# At this time there is no way for a user of Net::HTTPResponse to read a
|
||
|
# specific number of bytes from the HTTP response body, so this internal
|
||
|
# API does not return the same number of bytes as were requested.
|
||
|
#
|
||
|
# See https://bugs.ruby-lang.org/issues/6492 for further discussion.
|
||
|
def read clen, dest, ignore_eof = false
|
||
|
temp_dest = inflate_adapter(dest)
|
||
|
data = @socket.read clen, temp_dest, ignore_eof
|
||
|
end
|
||
|
##
|
||
|
# Reads the rest of the socket, inflates it, then writes it to +dest+.
|
||
|
def read_all dest
|
||
|
temp_dest = inflate_adapter(dest)
|
||
|
@socket.read_all temp_dest
|
||
|
end
|
||
|
end
|
||
|
end
|
||
| lib/net/http.rb (working copy) | ||
|---|---|---|
|
@use_ssl = false
|
||
|
@ssl_context = nil
|
||
|
@enable_post_connection_check = true
|
||
|
@compression = nil
|
||
|
@sspi_enabled = false
|
||
|
SSL_IVNAMES.each do |ivname|
|
||
|
instance_variable_set ivname, nil
|
||
| ... | ... | |
|
initheader = initheader.merge({
|
||
|
"accept-encoding" => "gzip;q=1.0,deflate;q=0.6,identity;q=0.3"
|
||
|
})
|
||
|
@compression = true
|
||
|
end
|
||
|
end
|
||
|
request(Get.new(path, initheader)) {|r|
|
||
|
if r.key?("content-encoding") and @compression
|
||
|
@compression = nil # Clear it till next set.
|
||
|
the_body = r.read_body dest, &block
|
||
|
case r["content-encoding"]
|
||
|
when "gzip"
|
||
|
r.body= Zlib::GzipReader.new(StringIO.new(the_body), encoding: "ASCII-8BIT").read
|
||
|
r.delete("content-encoding")
|
||
|
when "deflate"
|
||
|
r.body= Zlib::Inflate.inflate(the_body);
|
||
|
r.delete("content-encoding")
|
||
|
when "identity"
|
||
|
; # nothing needed
|
||
|
else
|
||
|
; # Don't do anything dramatic, unless we need to later
|
||
|
end
|
||
|
else
|
||
|
r.read_body dest, &block
|
||
|
end
|
||
|
r.read_body dest, &block
|
||
|
res = r
|
||
|
}
|
||
|
res
|
||
| test/net/http/test_httpresponse.rb (working copy) | ||
|---|---|---|
|
class HTTPResponseTest < Test::Unit::TestCase
|
||
|
def test_singleline_header
|
||
|
io = dummy_io(<<EOS.gsub(/\n/, "\r\n"))
|
||
|
io = dummy_io(<<EOS)
|
||
|
HTTP/1.1 200 OK
|
||
|
Content-Length: 5
|
||
|
Connection: close
|
||
| ... | ... | |
|
end
|
||
|
def test_multiline_header
|
||
|
io = dummy_io(<<EOS.gsub(/\n/, "\r\n"))
|
||
|
io = dummy_io(<<EOS)
|
||
|
HTTP/1.1 200 OK
|
||
|
X-Foo: XXX
|
||
|
YYY
|
||
| ... | ... | |
|
assert_equal('XXX YYY', res.header['x-bar'])
|
||
|
end
|
||
|
def test_read_body
|
||
|
io = dummy_io(<<EOS)
|
||
|
HTTP/1.1 200 OK
|
||
|
Connection: close
|
||
|
Content-Length: 5
|
||
|
hello
|
||
|
EOS
|
||
|
res = Net::HTTPResponse.read_new(io)
|
||
|
body = nil
|
||
|
res.reading_body io, true do
|
||
|
body = res.read_body
|
||
|
end
|
||
|
assert_equal 'hello', body
|
||
|
end
|
||
|
def test_read_body_block
|
||
|
io = dummy_io(<<EOS)
|
||
|
HTTP/1.1 200 OK
|
||
|
Connection: close
|
||
|
Content-Length: 5
|
||
|
hello
|
||
|
EOS
|
||
|
res = Net::HTTPResponse.read_new(io)
|
||
|
body = ''
|
||
|
res.reading_body io, true do
|
||
|
res.read_body do |chunk|
|
||
|
body << chunk
|
||
|
end
|
||
|
end
|
||
|
assert_equal 'hello', body
|
||
|
end
|
||
|
def test_read_body_content_encoding_deflate
|
||
|
io = dummy_io(<<EOS)
|
||
|
HTTP/1.1 200 OK
|
||
|
Connection: close
|
||
|
Content-Encoding: deflate
|
||
|
Content-Length: 13
|
||
|
x\x9C\xCBH\xCD\xC9\xC9\a\x00\x06,\x02\x15
|
||
|
EOS
|
||
|
res = Net::HTTPResponse.read_new(io)
|
||
|
body = nil
|
||
|
res.reading_body io, true do
|
||
|
body = res.read_body
|
||
|
end
|
||
|
assert_equal 'hello', body
|
||
|
end
|
||
|
def test_read_body_content_encoding_deflate_chunked
|
||
|
io = dummy_io(<<EOS)
|
||
|
HTTP/1.1 200 OK
|
||
|
Connection: close
|
||
|
Content-Encoding: deflate
|
||
|
Transfer-Encoding: chunked
|
||
|
6
|
||
|
x\x9C\xCBH\xCD\xC9
|
||
|
7
|
||
|
\xC9\a\x00\x06,\x02\x15
|
||
|
0
|
||
|
EOS
|
||
|
res = Net::HTTPResponse.read_new(io)
|
||
|
body = nil
|
||
|
res.reading_body io, true do
|
||
|
body = res.read_body
|
||
|
end
|
||
|
assert_equal 'hello', body
|
||
|
end
|
||
|
def test_read_body_content_encoding_deflate_no_length
|
||
|
io = dummy_io(<<EOS)
|
||
|
HTTP/1.1 200 OK
|
||
|
Connection: close
|
||
|
Content-Encoding: deflate
|
||
|
x\x9C\xCBH\xCD\xC9\xC9\a\x00\x06,\x02\x15
|
||
|
EOS
|
||
|
res = Net::HTTPResponse.read_new(io)
|
||
|
body = nil
|
||
|
res.reading_body io, true do
|
||
|
body = res.read_body
|
||
|
end
|
||
|
assert_equal 'hello', body
|
||
|
end
|
||
|
def test_read_body_content_encoding_deflate_content_range
|
||
|
io = dummy_io(<<EOS)
|
||
|
HTTP/1.1 200 OK
|
||
|
Accept-Ranges: bytes
|
||
|
Connection: close
|
||
|
Content-Encoding: gzip
|
||
|
Content-Length: 10
|
||
|
Content-Range: bytes 0-9/55
|
||
|
\x1F\x8B\b\x00\x00\x00\x00\x00\x00\x03
|
||
|
EOS
|
||
|
res = Net::HTTPResponse.read_new(io)
|
||
|
body = nil
|
||
|
res.reading_body io, true do
|
||
|
body = res.read_body
|
||
|
end
|
||
|
assert_equal "\x1F\x8B\b\x00\x00\x00\x00\x00\x00\x03", body
|
||
|
end
|
||
|
def test_read_body_string
|
||
|
io = dummy_io(<<EOS)
|
||
|
HTTP/1.1 200 OK
|
||
|
Connection: close
|
||
|
Content-Length: 5
|
||
|
hello
|
||
|
EOS
|
||
|
res = Net::HTTPResponse.read_new(io)
|
||
|
body = ''
|
||
|
res.reading_body io, true do
|
||
|
res.read_body body
|
||
|
end
|
||
|
assert_equal 'hello', body
|
||
|
end
|
||
|
private
|
||
|
def dummy_io(str)
|
||
|
str = str.gsub(/\n/, "\r\n")
|
||
|
Net::BufferedIO.new(StringIO.new(str))
|
||
|
end
|
||
|
end
|
||