Project

General

Profile

Download (8.76 KB)

Backport #8323 ยป 0001-io.c-conversion-from-bom-encoding.patch

nobu (Nobuyoshi Nakada), 04/25/2013 01:42 PM

View differences:

io.c
* Qnil => no encoding specified (internal only)
*/
static void
rb_io_ext_int_to_encs(rb_encoding *ext, rb_encoding *intern, rb_encoding **enc, rb_encoding **enc2)
rb_io_ext_int_to_encs(rb_encoding *ext, rb_encoding *intern, rb_encoding **enc, rb_encoding **enc2, int fmode)
{
int default_ext = 0;
......
if (intern == NULL && ext != rb_ascii8bit_encoding())
/* If external is ASCII-8BIT, no default transcoding */
intern = rb_default_internal_encoding();
if (intern == NULL || intern == (rb_encoding *)Qnil || intern == ext) {
if (intern == NULL || intern == (rb_encoding *)Qnil ||
(!(fmode & FMODE_SETENC_BY_BOM) && (intern == ext))) {
/* No internal encoding => use external + no transcoding */
*enc = (default_ext && intern != ext) ? NULL : ext;
*enc2 = NULL;
......
const char *p;
char encname[ENCODING_MAXNAMELEN+1];
int idx, idx2;
int fmode = fmode_p ? *fmode_p : 0;
rb_encoding *ext_enc, *int_enc;
/* parse estr as "enc" or "enc2:enc" or "enc:-" */
......
idx = -1;
else {
if (io_encname_bom_p(estr, len)) {
if (fmode_p) *fmode_p |= FMODE_SETENC_BY_BOM;
fmode |= FMODE_SETENC_BY_BOM;
estr += 4;
len -= 4;
}
......
else {
long len = strlen(estr);
if (io_encname_bom_p(estr, len)) {
if (fmode_p) *fmode_p |= FMODE_SETENC_BY_BOM;
fmode |= FMODE_SETENC_BY_BOM;
estr += 4;
len -= 4;
memcpy(encname, estr, len);
......
}
idx = rb_enc_find_index(estr);
}
if (fmode_p) *fmode_p = fmode;
if (idx >= 0)
ext_enc = rb_enc_from_index(idx);
......
idx2 = rb_enc_find_index(p);
if (idx2 < 0)
unsupported_encoding(p);
else if (idx2 == idx) {
else if (!(fmode & FMODE_SETENC_BY_BOM) && (idx2 == idx)) {
int_enc = (rb_encoding *)Qnil;
}
else
......
}
}
rb_io_ext_int_to_encs(ext_enc, int_enc, enc_p, enc2_p);
rb_io_ext_int_to_encs(ext_enc, int_enc, enc_p, enc2_p, fmode);
}
int
......
parse_mode_enc(StringValueCStr(tmp), enc_p, enc2_p, fmode_p);
}
else {
rb_io_ext_int_to_encs(rb_to_encoding(encoding), NULL, enc_p, enc2_p);
rb_io_ext_int_to_encs(rb_to_encoding(encoding), NULL, enc_p, enc2_p, 0);
}
}
else if (extenc != Qundef || intenc != Qundef) {
extracted = 1;
rb_io_ext_int_to_encs(extencoding, intencoding, enc_p, enc2_p);
rb_io_ext_int_to_encs(extencoding, intencoding, enc_p, enc2_p, 0);
}
return extracted;
}
......
vmode = *vmode_p;
/* Set to defaults */
rb_io_ext_int_to_encs(NULL, NULL, &enc, &enc2);
rb_io_ext_int_to_encs(NULL, NULL, &enc, &enc2, 0);
vmode_handle:
if (NIL_P(vmode)) {
......
rb_encoding *e;
e = (fmode & FMODE_BINMODE) ? rb_ascii8bit_encoding() : NULL;
rb_io_ext_int_to_encs(e, NULL, &enc, &enc2);
rb_io_ext_int_to_encs(e, NULL, &enc, &enc2, fmode);
}
}
......
oflags |= O_BINARY;
#endif
if (!has_enc)
rb_io_ext_int_to_encs(rb_ascii8bit_encoding(), NULL, &enc, &enc2);
rb_io_ext_int_to_encs(rb_ascii8bit_encoding(), NULL, &enc, &enc2, fmode);
}
#if DEFAULT_TEXTMODE
else if (NIL_P(vmode)) {
......
io_set_encoding_by_bom(VALUE io)
{
int idx = io_strip_bom(io);
rb_io_t *fptr;
GetOpenFile(io, fptr);
if (idx) {
rb_io_t *fptr;
GetOpenFile(io, fptr);
io_encoding_set(fptr, rb_enc_from_encoding(rb_enc_from_index(idx)),
rb_io_internal_encoding(io), Qnil);
}
else {
fptr->encs.enc2 = NULL;
}
}
static VALUE
......
convconfig_t cc;
if (!convconfig) {
/* Set to default encodings */
rb_io_ext_int_to_encs(NULL, NULL, &cc.enc, &cc.enc2);
rb_io_ext_int_to_encs(NULL, NULL, &cc.enc, &cc.enc2, fmode);
cc.ecflags = 0;
cc.ecopts = Qnil;
convconfig = &cc;
......
/* Set to default encodings */
e = (fmode & FMODE_BINMODE) ? rb_ascii8bit_encoding() : NULL;
rb_io_ext_int_to_encs(e, NULL, &convconfig.enc, &convconfig.enc2);
rb_io_ext_int_to_encs(e, NULL, &convconfig.enc, &convconfig.enc2, fmode);
convconfig.ecflags = 0;
convconfig.ecopts = Qnil;
}
......
else {
if (NIL_P(v1)) {
/* Set to default encodings */
rb_io_ext_int_to_encs(NULL, NULL, &enc, &enc2);
rb_io_ext_int_to_encs(NULL, NULL, &enc, &enc2, 0);
SET_UNIVERSAL_NEWLINE_DECORATOR_IF_ENC2(enc2, ecflags);
ecopts = Qnil;
}
......
ecflags = rb_econv_prepare_options(opt, &ecopts, ecflags);
}
else {
rb_io_ext_int_to_encs(find_encoding(v1), NULL, &enc, &enc2);
rb_io_ext_int_to_encs(find_encoding(v1), NULL, &enc, &enc2, 0);
SET_UNIVERSAL_NEWLINE_DECORATOR_IF_ENC2(enc2, ecflags);
ecopts = Qnil;
}
test/ruby/test_io_m17n.rb
def test_strip_bom
with_tmpdir {
text = "\uFEFFa"
stripped = "a"
%w/UTF-8 UTF-16BE UTF-16LE UTF-32BE UTF-32LE/.each do |name|
path = '%s-bom.txt' % name
content = text.encode(name)
......
result = File.read(path, mode: 'rb:BOM|UTF-8')
assert_equal(content[1].force_encoding("ascii-8bit"),
result.force_encoding("ascii-8bit"))
result = File.read(path, mode: 'rb:BOM|UTF-8:UTF-8')
assert_equal(Encoding::UTF_8, result.encoding)
assert_equal(stripped, result)
end
bug3407 = '[ruby-core:30641]'
result = File.read('UTF-8-bom.txt', encoding: 'BOM|UTF-8')
path = 'UTF-8-bom.txt'
result = File.read(path, encoding: 'BOM|UTF-8')
assert_equal("a", result.force_encoding("ascii-8bit"), bug3407)
bug8323 = '[ruby-core:54563] [Bug #8323]'
expected = "a\xff".force_encoding("utf-8")
open(path, 'ab') {|f| f.write("\xff")}
result = File.read(path, encoding: 'BOM|UTF-8')
assert_not_predicate(result, :valid_encoding?, bug8323)
assert_equal(expected, result, bug8323)
result = File.read(path, encoding: 'BOM|UTF-8:UTF-8')
assert_not_predicate(result, :valid_encoding?, bug8323)
assert_equal(expected, result, bug8323)
path = 'ascii.txt'
generate_file(path, stripped)
result = File.read(path, encoding: 'BOM|UTF-8')
assert_equal(stripped, result, bug8323)
result = File.read(path, encoding: 'BOM|UTF-8:UTF-8')
assert_equal(stripped, result, bug8323)
}
end
    (1-1/1)