net.http.inflate_by_default.3.patch

Eric Hodel, 06/09/2012 06:47 AM

Download (9.36 KB)

View differences:

lib/net/http/response.rb (working copy)
275 275

  
276 276
  private
277 277

  
278
  def read_body_0(dest)
279
    if chunked?
280
      read_chunked dest
281
      return
282
    end
283
    clen = content_length()
284
    if clen
285
      @socket.read clen, dest, true   # ignore EOF
286
      return
278
  ##
279
  # Checks for a supported Content-Encoding header and yields an Inflate
280
  # wrapper for this response's socket when zlib is present.  If the
281
  # Content-Encoding is unsupported or zlib is missing the plain socket is
282
  # yielded.
283
  #
284
  # If a Content-Range header is present a plain socket is yielded as the
285
  # bytes in the range may not be a complete deflate block.
286

  
287
  def inflater # :nodoc:
288
    return yield @socket unless Net::HTTP::HAVE_ZLIB
289
    return yield @socket if self['content-range']
290

  
291
    case self['content-encoding']
292
    when 'deflate', 'gzip', 'x-gzip' then
293
      self.delete 'content-encoding'
294

  
295
      inflate_body_io = Inflater.new(@socket)
296

  
297
      begin
298
        yield inflate_body_io
299
      ensure
300
        inflate_body_io.finish
301
      end
302
    when 'none', 'identity' then
303
      self.delete 'content-encoding'
304

  
305
      yield @socket
306
    else
307
      yield @socket
287 308
    end
288
    clen = range_length()
289
    if clen
290
      @socket.read clen, dest
291
      return
309
  end
310

  
311
  def read_body_0(dest)
312
    inflater do |inflate_body_io|
313
      if chunked?
314
        read_chunked dest, inflate_body_io
315
        return
316
      end
317

  
318
      @socket = inflate_body_io
319

  
320
      clen = content_length()
321
      if clen
322
        @socket.read clen, dest, true   # ignore EOF
323
        return
324
      end
325
      clen = range_length()
326
      if clen
327
        @socket.read clen, dest
328
        return
329
      end
330
      @socket.read_all dest
292 331
    end
293
    @socket.read_all dest
294 332
  end
295 333

  
296
  def read_chunked(dest)
334
  ##
335
  # read_chunked reads from +@socket+ for chunk-size, chunk-extension, CRLF,
336
  # etc. and +chunk_data_io+ for chunk-data which may be deflate or gzip
337
  # encoded.
338
  #
339
  # See RFC 2616 section 3.6.1 for definitions
340

  
341
  def read_chunked(dest, chunk_data_io) # :nodoc:
297 342
    len = nil
298 343
    total = 0
299 344
    while true
......
303 348
      len = hexlen.hex
304 349
      break if len == 0
305 350
      begin
306
        @socket.read len, dest
351
        chunk_data_io.read len, dest
307 352
      ensure
308 353
        total += len
309 354
        @socket.read 2   # \r\n
......
319 364
  end
320 365

  
321 366
  def procdest(dest, block)
322
    raise ArgumentError, 'both arg and block given for HTTP method' \
323
        if dest and block
367
    raise ArgumentError, 'both arg and block given for HTTP method' if
368
      dest and block
324 369
    if block
325 370
      Net::ReadAdapter.new(block)
326 371
    else
......
328 373
    end
329 374
  end
330 375

  
376
  ##
377
  # Inflater is a wrapper around Net::BufferedIO that transparently inflates
378
  # zlib and gzip streams.
379

  
380
  class Inflater # :nodoc:
381

  
382
    ##
383
    # Creates a new Inflater wrapping +socket+
384

  
385
    def initialize socket
386
      @socket = socket
387
      # zlib with automatic gzip detection
388
      @inflate = Zlib::Inflate.new(32 + Zlib::MAX_WBITS)
389
    end
390

  
391
    ##
392
    # Finishes the inflate stream.
393

  
394
    def finish
395
      @inflate.finish
396
    end
397

  
398
    ##
399
    # Returns a Net::ReadAdapter that inflates each read chunk into +dest+.
400
    #
401
    # This allows a large response body to be inflated without storing the
402
    # entire body in memory.
403

  
404
    def inflate_adapter(dest)
405
      block = proc do |chunk|
406
        dest << @inflate.inflate(chunk)
407
      end
408

  
409
      Net::ReadAdapter.new(block)
410
    end
411

  
412
    ##
413
    # Reads +clen+ bytes from the socket, inflates them, then writes them to
414
    # +dest+.  +ignore_eof+ is passed down to Net::BufferedIO#read
415
    #
416
    # Unlike Net::BufferedIO#read, this method returns more than +clen+ bytes.
417
    # At this time there is no way for a user of Net::HTTPResponse to read a
418
    # specific number of bytes from the HTTP response body, so this internal
419
    # API does not return the same number of bytes as were requested.
420
    #
421
    # See https://bugs.ruby-lang.org/issues/6492 for further discussion.
422

  
423
    def read clen, dest, ignore_eof = false
424
      temp_dest = inflate_adapter(dest)
425

  
426
      data = @socket.read clen, temp_dest, ignore_eof
427
    end
428

  
429
    ##
430
    # Reads the rest of the socket, inflates it, then writes it to +dest+.
431

  
432
    def read_all dest
433
      temp_dest = inflate_adapter(dest)
434

  
435
      @socket.read_all temp_dest
436
    end
437

  
438
  end
439

  
331 440
end
332 441

  
lib/net/http.rb (working copy)
590 590
      @use_ssl = false
591 591
      @ssl_context = nil
592 592
      @enable_post_connection_check = true
593
      @compression = nil
594 593
      @sspi_enabled = false
595 594
      SSL_IVNAMES.each do |ivname|
596 595
        instance_variable_set ivname, nil
......
1034 1033
          initheader = initheader.merge({
1035 1034
            "accept-encoding" => "gzip;q=1.0,deflate;q=0.6,identity;q=0.3"
1036 1035
          })
1037
          @compression = true
1038 1036
        end
1039 1037
      end
1040 1038
      request(Get.new(path, initheader)) {|r|
1041
        if r.key?("content-encoding") and @compression
1042
          @compression = nil # Clear it till next set.
1043
          the_body = r.read_body dest, &block
1044
          case r["content-encoding"]
1045
          when "gzip"
1046
            r.body= Zlib::GzipReader.new(StringIO.new(the_body), encoding: "ASCII-8BIT").read
1047
            r.delete("content-encoding")
1048
          when "deflate"
1049
            r.body= Zlib::Inflate.inflate(the_body);
1050
            r.delete("content-encoding")
1051
          when "identity"
1052
            ; # nothing needed
1053
          else
1054
            ; # Don't do anything dramatic, unless we need to later
1055
          end
1056
        else
1057
          r.read_body dest, &block
1058
        end
1039
        r.read_body dest, &block
1059 1040
        res = r
1060 1041
      }
1061 1042
      res
test/net/http/test_httpresponse.rb (working copy)
4 4

  
5 5
class HTTPResponseTest < Test::Unit::TestCase
6 6
  def test_singleline_header
7
    io = dummy_io(<<EOS.gsub(/\n/, "\r\n"))
7
    io = dummy_io(<<EOS)
8 8
HTTP/1.1 200 OK
9 9
Content-Length: 5
10 10
Connection: close
......
17 17
  end
18 18

  
19 19
  def test_multiline_header
20
    io = dummy_io(<<EOS.gsub(/\n/, "\r\n"))
20
    io = dummy_io(<<EOS)
21 21
HTTP/1.1 200 OK
22 22
X-Foo: XXX
23 23
   YYY
......
32 32
    assert_equal('XXX YYY', res.header['x-bar'])
33 33
  end
34 34

  
35
  def test_read_body
36
    io = dummy_io(<<EOS)
37
HTTP/1.1 200 OK
38
Connection: close
39
Content-Length: 5
40

  
41
hello
42
EOS
43

  
44
    res = Net::HTTPResponse.read_new(io)
45

  
46
    body = nil
47

  
48
    res.reading_body io, true do
49
      body = res.read_body
50
    end
51

  
52
    assert_equal 'hello', body
53
  end
54

  
55
  def test_read_body_block
56
    io = dummy_io(<<EOS)
57
HTTP/1.1 200 OK
58
Connection: close
59
Content-Length: 5
60

  
61
hello
62
EOS
63

  
64
    res = Net::HTTPResponse.read_new(io)
65

  
66
    body = ''
67

  
68
    res.reading_body io, true do
69
      res.read_body do |chunk|
70
        body << chunk
71
      end
72
    end
73

  
74
    assert_equal 'hello', body
75
  end
76

  
77
  def test_read_body_content_encoding_deflate
78
    io = dummy_io(<<EOS)
79
HTTP/1.1 200 OK
80
Connection: close
81
Content-Encoding: deflate
82
Content-Length: 13
83

  
84
x\x9C\xCBH\xCD\xC9\xC9\a\x00\x06,\x02\x15
85
EOS
86

  
87
    res = Net::HTTPResponse.read_new(io)
88

  
89
    body = nil
90

  
91
    res.reading_body io, true do
92
      body = res.read_body
93
    end
94

  
95
    assert_equal 'hello', body
96
  end
97

  
98
  def test_read_body_content_encoding_deflate_chunked
99
    io = dummy_io(<<EOS)
100
HTTP/1.1 200 OK
101
Connection: close
102
Content-Encoding: deflate
103
Transfer-Encoding: chunked
104

  
105
6
106
x\x9C\xCBH\xCD\xC9
107
7
108
\xC9\a\x00\x06,\x02\x15
109
0
110

  
111
EOS
112

  
113
    res = Net::HTTPResponse.read_new(io)
114

  
115
    body = nil
116

  
117
    res.reading_body io, true do
118
      body = res.read_body
119
    end
120

  
121
    assert_equal 'hello', body
122
  end
123

  
124
  def test_read_body_content_encoding_deflate_no_length
125
    io = dummy_io(<<EOS)
126
HTTP/1.1 200 OK
127
Connection: close
128
Content-Encoding: deflate
129

  
130
x\x9C\xCBH\xCD\xC9\xC9\a\x00\x06,\x02\x15
131
EOS
132

  
133
    res = Net::HTTPResponse.read_new(io)
134

  
135
    body = nil
136

  
137
    res.reading_body io, true do
138
      body = res.read_body
139
    end
140

  
141
    assert_equal 'hello', body
142
  end
143

  
144
  def test_read_body_content_encoding_deflate_content_range
145
    io = dummy_io(<<EOS)
146
HTTP/1.1 200 OK
147
Accept-Ranges: bytes
148
Connection: close
149
Content-Encoding: gzip
150
Content-Length: 10
151
Content-Range: bytes 0-9/55
152

  
153
\x1F\x8B\b\x00\x00\x00\x00\x00\x00\x03
154
EOS
155

  
156
    res = Net::HTTPResponse.read_new(io)
157

  
158
    body = nil
159

  
160
    res.reading_body io, true do
161
      body = res.read_body
162
    end
163

  
164
    assert_equal "\x1F\x8B\b\x00\x00\x00\x00\x00\x00\x03", body
165
  end
166

  
167
  def test_read_body_string
168
    io = dummy_io(<<EOS)
169
HTTP/1.1 200 OK
170
Connection: close
171
Content-Length: 5
172

  
173
hello
174
EOS
175

  
176
    res = Net::HTTPResponse.read_new(io)
177

  
178
    body = ''
179

  
180
    res.reading_body io, true do
181
      res.read_body body
182
    end
183

  
184
    assert_equal 'hello', body
185
  end
186

  
35 187
private
36 188

  
37 189
  def dummy_io(str)
190
    str = str.gsub(/\n/, "\r\n")
191

  
38 192
    Net::BufferedIO.new(StringIO.new(str))
39 193
  end
40 194
end