Project

General

Profile

Download (9.42 KB)

Feature #6492 » net.http.inflate_by_default.4.patch

Updated to use streaming inflate - drbrain (Eric Hodel), 07/11/2012 05:29 AM

View differences:

lib/net/http/response.rb (working copy)
private
def read_body_0(dest)
if chunked?
read_chunked dest
return
end
clen = content_length()
if clen
@socket.read clen, dest, true # ignore EOF
return
##
# Checks for a supported Content-Encoding header and yields an Inflate
# wrapper for this response's socket when zlib is present. If the
# Content-Encoding is unsupported or zlib is missing the plain socket is
# yielded.
#
# If a Content-Range header is present a plain socket is yielded as the
# bytes in the range may not be a complete deflate block.
def inflater # :nodoc:
return yield @socket unless Net::HTTP::HAVE_ZLIB
return yield @socket if self['content-range']
case self['content-encoding']
when 'deflate', 'gzip', 'x-gzip' then
self.delete 'content-encoding'
inflate_body_io = Inflater.new(@socket)
begin
yield inflate_body_io
ensure
inflate_body_io.finish
end
when 'none', 'identity' then
self.delete 'content-encoding'
yield @socket
else
yield @socket
end
clen = range_length()
if clen
@socket.read clen, dest
return
end
def read_body_0(dest)
inflater do |inflate_body_io|
if chunked?
read_chunked dest, inflate_body_io
return
end
@socket = inflate_body_io
clen = content_length()
if clen
@socket.read clen, dest, true # ignore EOF
return
end
clen = range_length()
if clen
@socket.read clen, dest
return
end
@socket.read_all dest
end
@socket.read_all dest
end
def read_chunked(dest)
##
# read_chunked reads from +@socket+ for chunk-size, chunk-extension, CRLF,
# etc. and +chunk_data_io+ for chunk-data which may be deflate or gzip
# encoded.
#
# See RFC 2616 section 3.6.1 for definitions
def read_chunked(dest, chunk_data_io) # :nodoc:
len = nil
total = 0
while true
......
len = hexlen.hex
break if len == 0
begin
@socket.read len, dest
chunk_data_io.read len, dest
ensure
total += len
@socket.read 2 # \r\n
......
end
def procdest(dest, block)
raise ArgumentError, 'both arg and block given for HTTP method' \
if dest and block
raise ArgumentError, 'both arg and block given for HTTP method' if
dest and block
if block
Net::ReadAdapter.new(block)
else
......
end
end
##
# Inflater is a wrapper around Net::BufferedIO that transparently inflates
# zlib and gzip streams.
class Inflater # :nodoc:
##
# Creates a new Inflater wrapping +socket+
def initialize socket
@socket = socket
# zlib with automatic gzip detection
@inflate = Zlib::Inflate.new(32 + Zlib::MAX_WBITS)
end
##
# Finishes the inflate stream.
def finish
@inflate.finish
end
##
# Returns a Net::ReadAdapter that inflates each read chunk into +dest+.
#
# This allows a large response body to be inflated without storing the
# entire body in memory.
def inflate_adapter(dest)
block = proc do |compressed_chunk|
@inflate.inflate(compressed_chunk) do |chunk|
dest << chunk
end
end
Net::ReadAdapter.new(block)
end
##
# Reads +clen+ bytes from the socket, inflates them, then writes them to
# +dest+. +ignore_eof+ is passed down to Net::BufferedIO#read
#
# Unlike Net::BufferedIO#read, this method returns more than +clen+ bytes.
# At this time there is no way for a user of Net::HTTPResponse to read a
# specific number of bytes from the HTTP response body, so this internal
# API does not return the same number of bytes as were requested.
#
# See https://bugs.ruby-lang.org/issues/6492 for further discussion.
def read clen, dest, ignore_eof = false
temp_dest = inflate_adapter(dest)
data = @socket.read clen, temp_dest, ignore_eof
end
##
# Reads the rest of the socket, inflates it, then writes it to +dest+.
def read_all dest
temp_dest = inflate_adapter(dest)
@socket.read_all temp_dest
end
end
end
lib/net/http.rb (working copy)
@use_ssl = false
@ssl_context = nil
@enable_post_connection_check = true
@compression = nil
@sspi_enabled = false
SSL_IVNAMES.each do |ivname|
instance_variable_set ivname, nil
......
initheader = initheader.merge({
"accept-encoding" => "gzip;q=1.0,deflate;q=0.6,identity;q=0.3"
})
@compression = true
end
end
request(Get.new(path, initheader)) {|r|
if r.key?("content-encoding") and @compression
@compression = nil # Clear it till next set.
the_body = r.read_body dest, &block
case r["content-encoding"]
when "gzip"
r.body= Zlib::GzipReader.new(StringIO.new(the_body), encoding: "ASCII-8BIT").read
r.delete("content-encoding")
when "deflate"
r.body= Zlib::Inflate.inflate(the_body);
r.delete("content-encoding")
when "identity"
; # nothing needed
else
; # Don't do anything dramatic, unless we need to later
end
else
r.read_body dest, &block
end
r.read_body dest, &block
res = r
}
res
test/net/http/test_httpresponse.rb (working copy)
class HTTPResponseTest < Test::Unit::TestCase
def test_singleline_header
io = dummy_io(<<EOS.gsub(/\n/, "\r\n"))
io = dummy_io(<<EOS)
HTTP/1.1 200 OK
Content-Length: 5
Connection: close
......
end
def test_multiline_header
io = dummy_io(<<EOS.gsub(/\n/, "\r\n"))
io = dummy_io(<<EOS)
HTTP/1.1 200 OK
X-Foo: XXX
YYY
......
assert_equal('XXX YYY', res.header['x-bar'])
end
def test_read_body
io = dummy_io(<<EOS)
HTTP/1.1 200 OK
Connection: close
Content-Length: 5
hello
EOS
res = Net::HTTPResponse.read_new(io)
body = nil
res.reading_body io, true do
body = res.read_body
end
assert_equal 'hello', body
end
def test_read_body_block
io = dummy_io(<<EOS)
HTTP/1.1 200 OK
Connection: close
Content-Length: 5
hello
EOS
res = Net::HTTPResponse.read_new(io)
body = ''
res.reading_body io, true do
res.read_body do |chunk|
body << chunk
end
end
assert_equal 'hello', body
end
def test_read_body_content_encoding_deflate
io = dummy_io(<<EOS)
HTTP/1.1 200 OK
Connection: close
Content-Encoding: deflate
Content-Length: 13
x\x9C\xCBH\xCD\xC9\xC9\a\x00\x06,\x02\x15
EOS
res = Net::HTTPResponse.read_new(io)
body = nil
res.reading_body io, true do
body = res.read_body
end
assert_equal 'hello', body
end
def test_read_body_content_encoding_deflate_chunked
io = dummy_io(<<EOS)
HTTP/1.1 200 OK
Connection: close
Content-Encoding: deflate
Transfer-Encoding: chunked
6
x\x9C\xCBH\xCD\xC9
7
\xC9\a\x00\x06,\x02\x15
0
EOS
res = Net::HTTPResponse.read_new(io)
body = nil
res.reading_body io, true do
body = res.read_body
end
assert_equal 'hello', body
end
def test_read_body_content_encoding_deflate_no_length
io = dummy_io(<<EOS)
HTTP/1.1 200 OK
Connection: close
Content-Encoding: deflate
x\x9C\xCBH\xCD\xC9\xC9\a\x00\x06,\x02\x15
EOS
res = Net::HTTPResponse.read_new(io)
body = nil
res.reading_body io, true do
body = res.read_body
end
assert_equal 'hello', body
end
def test_read_body_content_encoding_deflate_content_range
io = dummy_io(<<EOS)
HTTP/1.1 200 OK
Accept-Ranges: bytes
Connection: close
Content-Encoding: gzip
Content-Length: 10
Content-Range: bytes 0-9/55
\x1F\x8B\b\x00\x00\x00\x00\x00\x00\x03
EOS
res = Net::HTTPResponse.read_new(io)
body = nil
res.reading_body io, true do
body = res.read_body
end
assert_equal "\x1F\x8B\b\x00\x00\x00\x00\x00\x00\x03", body
end
def test_read_body_string
io = dummy_io(<<EOS)
HTTP/1.1 200 OK
Connection: close
Content-Length: 5
hello
EOS
res = Net::HTTPResponse.read_new(io)
body = ''
res.reading_body io, true do
res.read_body body
end
assert_equal 'hello', body
end
private
def dummy_io(str)
str = str.gsub(/\n/, "\r\n")
Net::BufferedIO.new(StringIO.new(str))
end
end
(4-4/4)