net.http.inflate_by_default.4.patch

Updated to use streaming inflate - Eric Hodel, 07/11/2012 05:29 AM

Download (9.42 KB)

View differences:

lib/net/http/response.rb (working copy)
222 222

  
223 223
  private
224 224

  
225
  def read_body_0(dest)
226
    if chunked?
227
      read_chunked dest
228
      return
229
    end
230
    clen = content_length()
231
    if clen
232
      @socket.read clen, dest, true   # ignore EOF
233
      return
225
  ##
226
  # Checks for a supported Content-Encoding header and yields an Inflate
227
  # wrapper for this response's socket when zlib is present.  If the
228
  # Content-Encoding is unsupported or zlib is missing the plain socket is
229
  # yielded.
230
  #
231
  # If a Content-Range header is present a plain socket is yielded as the
232
  # bytes in the range may not be a complete deflate block.
233

  
234
  def inflater # :nodoc:
235
    return yield @socket unless Net::HTTP::HAVE_ZLIB
236
    return yield @socket if self['content-range']
237

  
238
    case self['content-encoding']
239
    when 'deflate', 'gzip', 'x-gzip' then
240
      self.delete 'content-encoding'
241

  
242
      inflate_body_io = Inflater.new(@socket)
243

  
244
      begin
245
        yield inflate_body_io
246
      ensure
247
        inflate_body_io.finish
248
      end
249
    when 'none', 'identity' then
250
      self.delete 'content-encoding'
251

  
252
      yield @socket
253
    else
254
      yield @socket
234 255
    end
235
    clen = range_length()
236
    if clen
237
      @socket.read clen, dest
238
      return
256
  end
257

  
258
  def read_body_0(dest)
259
    inflater do |inflate_body_io|
260
      if chunked?
261
        read_chunked dest, inflate_body_io
262
        return
263
      end
264

  
265
      @socket = inflate_body_io
266

  
267
      clen = content_length()
268
      if clen
269
        @socket.read clen, dest, true   # ignore EOF
270
        return
271
      end
272
      clen = range_length()
273
      if clen
274
        @socket.read clen, dest
275
        return
276
      end
277
      @socket.read_all dest
239 278
    end
240
    @socket.read_all dest
241 279
  end
242 280

  
243
  def read_chunked(dest)
281
  ##
282
  # read_chunked reads from +@socket+ for chunk-size, chunk-extension, CRLF,
283
  # etc. and +chunk_data_io+ for chunk-data which may be deflate or gzip
284
  # encoded.
285
  #
286
  # See RFC 2616 section 3.6.1 for definitions
287

  
288
  def read_chunked(dest, chunk_data_io) # :nodoc:
244 289
    len = nil
245 290
    total = 0
246 291
    while true
......
250 295
      len = hexlen.hex
251 296
      break if len == 0
252 297
      begin
253
        @socket.read len, dest
298
        chunk_data_io.read len, dest
254 299
      ensure
255 300
        total += len
256 301
        @socket.read 2   # \r\n
......
266 311
  end
267 312

  
268 313
  def procdest(dest, block)
269
    raise ArgumentError, 'both arg and block given for HTTP method' \
270
        if dest and block
314
    raise ArgumentError, 'both arg and block given for HTTP method' if
315
      dest and block
271 316
    if block
272 317
      Net::ReadAdapter.new(block)
273 318
    else
......
275 320
    end
276 321
  end
277 322

  
323
  ##
324
  # Inflater is a wrapper around Net::BufferedIO that transparently inflates
325
  # zlib and gzip streams.
326

  
327
  class Inflater # :nodoc:
328

  
329
    ##
330
    # Creates a new Inflater wrapping +socket+
331

  
332
    def initialize socket
333
      @socket = socket
334
      # zlib with automatic gzip detection
335
      @inflate = Zlib::Inflate.new(32 + Zlib::MAX_WBITS)
336
    end
337

  
338
    ##
339
    # Finishes the inflate stream.
340

  
341
    def finish
342
      @inflate.finish
343
    end
344

  
345
    ##
346
    # Returns a Net::ReadAdapter that inflates each read chunk into +dest+.
347
    #
348
    # This allows a large response body to be inflated without storing the
349
    # entire body in memory.
350

  
351
    def inflate_adapter(dest)
352
      block = proc do |compressed_chunk|
353
        @inflate.inflate(compressed_chunk) do |chunk|
354
          dest << chunk
355
        end
356
      end
357

  
358
      Net::ReadAdapter.new(block)
359
    end
360

  
361
    ##
362
    # Reads +clen+ bytes from the socket, inflates them, then writes them to
363
    # +dest+.  +ignore_eof+ is passed down to Net::BufferedIO#read
364
    #
365
    # Unlike Net::BufferedIO#read, this method returns more than +clen+ bytes.
366
    # At this time there is no way for a user of Net::HTTPResponse to read a
367
    # specific number of bytes from the HTTP response body, so this internal
368
    # API does not return the same number of bytes as were requested.
369
    #
370
    # See https://bugs.ruby-lang.org/issues/6492 for further discussion.
371

  
372
    def read clen, dest, ignore_eof = false
373
      temp_dest = inflate_adapter(dest)
374

  
375
      data = @socket.read clen, temp_dest, ignore_eof
376
    end
377

  
378
    ##
379
    # Reads the rest of the socket, inflates it, then writes it to +dest+.
380

  
381
    def read_all dest
382
      temp_dest = inflate_adapter(dest)
383

  
384
      @socket.read_all temp_dest
385
    end
386

  
387
  end
388

  
278 389
end
279 390

  
lib/net/http.rb (working copy)
600 600
      @use_ssl = false
601 601
      @ssl_context = nil
602 602
      @enable_post_connection_check = true
603
      @compression = nil
604 603
      @sspi_enabled = false
605 604
      SSL_IVNAMES.each do |ivname|
606 605
        instance_variable_set ivname, nil
......
1044 1043
          initheader = initheader.merge({
1045 1044
            "accept-encoding" => "gzip;q=1.0,deflate;q=0.6,identity;q=0.3"
1046 1045
          })
1047
          @compression = true
1048 1046
        end
1049 1047
      end
1050 1048
      request(Get.new(path, initheader)) {|r|
1051
        if r.key?("content-encoding") and @compression
1052
          @compression = nil # Clear it till next set.
1053
          the_body = r.read_body dest, &block
1054
          case r["content-encoding"]
1055
          when "gzip"
1056
            r.body= Zlib::GzipReader.new(StringIO.new(the_body), encoding: "ASCII-8BIT").read
1057
            r.delete("content-encoding")
1058
          when "deflate"
1059
            r.body= Zlib::Inflate.inflate(the_body);
1060
            r.delete("content-encoding")
1061
          when "identity"
1062
            ; # nothing needed
1063
          else
1064
            ; # Don't do anything dramatic, unless we need to later
1065
          end
1066
        else
1067
          r.read_body dest, &block
1068
        end
1049
        r.read_body dest, &block
1069 1050
        res = r
1070 1051
      }
1071 1052
      res
test/net/http/test_httpresponse.rb (working copy)
4 4

  
5 5
class HTTPResponseTest < Test::Unit::TestCase
6 6
  def test_singleline_header
7
    io = dummy_io(<<EOS.gsub(/\n/, "\r\n"))
7
    io = dummy_io(<<EOS)
8 8
HTTP/1.1 200 OK
9 9
Content-Length: 5
10 10
Connection: close
......
17 17
  end
18 18

  
19 19
  def test_multiline_header
20
    io = dummy_io(<<EOS.gsub(/\n/, "\r\n"))
20
    io = dummy_io(<<EOS)
21 21
HTTP/1.1 200 OK
22 22
X-Foo: XXX
23 23
   YYY
......
32 32
    assert_equal('XXX YYY', res.header['x-bar'])
33 33
  end
34 34

  
35
  def test_read_body
36
    io = dummy_io(<<EOS)
37
HTTP/1.1 200 OK
38
Connection: close
39
Content-Length: 5
40

  
41
hello
42
EOS
43

  
44
    res = Net::HTTPResponse.read_new(io)
45

  
46
    body = nil
47

  
48
    res.reading_body io, true do
49
      body = res.read_body
50
    end
51

  
52
    assert_equal 'hello', body
53
  end
54

  
55
  def test_read_body_block
56
    io = dummy_io(<<EOS)
57
HTTP/1.1 200 OK
58
Connection: close
59
Content-Length: 5
60

  
61
hello
62
EOS
63

  
64
    res = Net::HTTPResponse.read_new(io)
65

  
66
    body = ''
67

  
68
    res.reading_body io, true do
69
      res.read_body do |chunk|
70
        body << chunk
71
      end
72
    end
73

  
74
    assert_equal 'hello', body
75
  end
76

  
77
  def test_read_body_content_encoding_deflate
78
    io = dummy_io(<<EOS)
79
HTTP/1.1 200 OK
80
Connection: close
81
Content-Encoding: deflate
82
Content-Length: 13
83

  
84
x\x9C\xCBH\xCD\xC9\xC9\a\x00\x06,\x02\x15
85
EOS
86

  
87
    res = Net::HTTPResponse.read_new(io)
88

  
89
    body = nil
90

  
91
    res.reading_body io, true do
92
      body = res.read_body
93
    end
94

  
95
    assert_equal 'hello', body
96
  end
97

  
98
  def test_read_body_content_encoding_deflate_chunked
99
    io = dummy_io(<<EOS)
100
HTTP/1.1 200 OK
101
Connection: close
102
Content-Encoding: deflate
103
Transfer-Encoding: chunked
104

  
105
6
106
x\x9C\xCBH\xCD\xC9
107
7
108
\xC9\a\x00\x06,\x02\x15
109
0
110

  
111
EOS
112

  
113
    res = Net::HTTPResponse.read_new(io)
114

  
115
    body = nil
116

  
117
    res.reading_body io, true do
118
      body = res.read_body
119
    end
120

  
121
    assert_equal 'hello', body
122
  end
123

  
124
  def test_read_body_content_encoding_deflate_no_length
125
    io = dummy_io(<<EOS)
126
HTTP/1.1 200 OK
127
Connection: close
128
Content-Encoding: deflate
129

  
130
x\x9C\xCBH\xCD\xC9\xC9\a\x00\x06,\x02\x15
131
EOS
132

  
133
    res = Net::HTTPResponse.read_new(io)
134

  
135
    body = nil
136

  
137
    res.reading_body io, true do
138
      body = res.read_body
139
    end
140

  
141
    assert_equal 'hello', body
142
  end
143

  
144
  def test_read_body_content_encoding_deflate_content_range
145
    io = dummy_io(<<EOS)
146
HTTP/1.1 200 OK
147
Accept-Ranges: bytes
148
Connection: close
149
Content-Encoding: gzip
150
Content-Length: 10
151
Content-Range: bytes 0-9/55
152

  
153
\x1F\x8B\b\x00\x00\x00\x00\x00\x00\x03
154
EOS
155

  
156
    res = Net::HTTPResponse.read_new(io)
157

  
158
    body = nil
159

  
160
    res.reading_body io, true do
161
      body = res.read_body
162
    end
163

  
164
    assert_equal "\x1F\x8B\b\x00\x00\x00\x00\x00\x00\x03", body
165
  end
166

  
167
  def test_read_body_string
168
    io = dummy_io(<<EOS)
169
HTTP/1.1 200 OK
170
Connection: close
171
Content-Length: 5
172

  
173
hello
174
EOS
175

  
176
    res = Net::HTTPResponse.read_new(io)
177

  
178
    body = ''
179

  
180
    res.reading_body io, true do
181
      res.read_body body
182
    end
183

  
184
    assert_equal 'hello', body
185
  end
186

  
35 187
private
36 188

  
37 189
  def dummy_io(str)
190
    str = str.gsub(/\n/, "\r\n")
191

  
38 192
    Net::BufferedIO.new(StringIO.new(str))
39 193
  end
40 194
end