Bug #7758

Ruby on Windows crashes when active codepage is codepage 65001 and outputting unicode character

Added by Josh C over 2 years ago. Updated almost 2 years ago.

[ruby-core:51763]
Status:Assigned
Priority:Normal
Assignee:Luis Lavena
ruby -v:ruby 1.9.3p374 Backport:

Description

=begin
I have a script that contains the unicode BOX_DRAWINGS_LIGHT_VERTICAL_AND_RIGHT character (http://www.fileformat.info/info/unicode/char/251c/index.htm). When the console code page is set to 65001 (utf-8), both ruby 1.9.3 p125 and p374 crash, but ruby 1.8.7 p357 does not crash (nor does it output the character correctly). When the code page is set to 437, ruby 1.9.3 (both versions) correctly output the character, while ruby 1.8.7 outputs the wrong character again.

I don't care about ruby 1.8.7 so much, but I would expect ruby 1.9.3 to output the correct character, especially since the console codepage is set to 65001 (utf-8).

Here is my script:

# -- coding: utf-8 --
puts "a"
puts "├"

I verified that the file is correctly encoded:

C:\test>od.exe -t x1 test65001.rb
0000000 23 20 2d 2a 2d 20 63 6f 64 69 6e 67 3a 20 75 74
0000020 66 2d 38 20 2d 2a 2d 0a 70 75 74 73 20 22 61 22
0000040 0a 70 75 74 73 20 22 e2 94 9c 22 0a 0a

Note the UTF-8 sequence e2 94 9c

Here are the codepage settings in the registry:
C:\test>reg query hklm\SYSTEM\CurrentControlSet\Control\Nls\CodePage /v acp
HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Nls\CodePage
acp REG_SZ 1252
C:\test>reg query hklm\SYSTEM\CurrentControlSet\Control\Nls\CodePage /v oemcp
HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Nls\CodePage
oemcp REG_SZ 437

On ruby 1.8.7:

C:\test>ruby --version
ruby 1.8.7 (2011-12-28 patchlevel 357) [i386-mingw32]

C:\test>chcp 437
Active code page: 437

C:\test>ruby test65001.rb
a
├

C:\test>chcp 65001
Active code page: 65001

C:\test>ruby test65001.rb
a
├

On ruby 1.9.3p125:

C:\test>ruby --version
ruby 1.9.3p125 (2012-02-16) [i386-mingw32]

C:\test>chcp 437
Active code page: 437

C:\test>ruby test65001.rb
a
Ã

C:\test>chcp 65001
Active code page: 65001

C:\test>ruby test65001.rb
a
├test65001.rb:3: [BUG] rb_sys_fail() - errno == 0
ruby 1.9.3p125 (2012-02-16) [i386-mingw32]

-- Control frame information -----------------------------------------------
c:0006 p:---- s:0014 b:0014 l:000013 d:000013 CFUNC :write
c:0005 p:---- s:0012 b:0012 l:000011 d:000011 CFUNC :puts
c:0004 p:---- s:0010 b:0010 l:000009 d:000009 CFUNC :puts
c:0003 p:0023 s:0006 b:0006 l:00215c d:0015b4 EVAL test65001.rb:3
c:0002 p:---- s:0004 b:0004 l:000003 d:000003 FINISH
c:0001 p:0000 s:0002 b:0002 l:00215c d:00215c TOP

-- Ruby level backtrace information ----------------------------------------
test65001.rb:3:in <main>'
test65001.rb:3:in
puts'
test65001.rb:3:in puts'
test65001.rb:3:in
write'

-- C level backtrace information -------------------------------------------
C:\Windows\SysWOW64\ntdll.dll(ZwWaitForSingleObject+0x15) [0x77bbf8b1]
C:\Windows\syswow64\kernel32.dll(WaitForSingleObjectEx+0x43) [0x772d1194]
C:\Windows\syswow64\kernel32.dll(WaitForSingleObject+0x12) [0x772d1148]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_vm_bugreport+0xf9) [0x62e5c61d]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_name_err_mesg_new+0x17a) [0x62d3a6fa]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_bug+0x2f) [0x62d3b45b]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_sys_fail+0x163) [0x62d3cfb7]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_io_close+0x171f) [0x62d651cb]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_vm_invoke_proc+0x2fe) [0x62e54586]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_funcall+0x59) [0x62e55035]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_io_puts+0x7e) [0x62d5fb7e]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_thread_mark+0x741) [0x62e49ca1]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_vm_invoke_proc+0x2fe) [0x62e54586]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_io_puts+0x200) [0x62d5fd00]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_thread_mark+0x741) [0x62e49ca1]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_vm_call+0x2b2) [0x62e5855a]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_vm_localjump_error+0x1f66) [0x62e4ed8a]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_vm_localjump_error+0x650e) [0x62e53332]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_iseq_eval_main+0x98) [0x62e596f0]
c:\ruby193\bin\msvcrt-ruby191.dll(rb_check_frozen+0x2a89) [0x62d3fed9]
c:\ruby193\bin\msvcrt-ruby191.dll(ruby_run_node+0x48) [0x62d41ff4]
[0x0040136f]
[0x004010b9]
[0x00401284]
C:\Windows\syswow64\kernel32.dll(BaseThreadInitThunk+0x12) [0x772d339a]
C:\Windows\SysWOW64\ntdll.dll(RtlInitializeExceptionChain+0x63) [0x77bd9ef2]

-- Other runtime information -----------------------------------------------

  • Loaded script: test65001.rb

  • Loaded features:

    0 enumerator.so
    1 c:/ruby193/lib/ruby/1.9.1/i386-mingw32/enc/encdb.so
    2 c:/ruby193/lib/ruby/1.9.1/i386-mingw32/enc/iso_8859_1.so
    3 c:/ruby193/lib/ruby/1.9.1/i386-mingw32/enc/trans/transdb.so
    4 c:/ruby193/lib/ruby/site_ruby/1.9.1/rubygems/defaults.rb
    5 c:/ruby193/lib/ruby/1.9.1/i386-mingw32/rbconfig.rb
    6 c:/ruby193/lib/ruby/site_ruby/1.9.1/rubygems/deprecate.rb
    7 c:/ruby193/lib/ruby/site_ruby/1.9.1/rubygems/exceptions.rb
    8 c:/ruby193/lib/ruby/site_ruby/1.9.1/rubygems/defaults/operating_system.rb
    9 c:/ruby193/lib/ruby/site_ruby/1.9.1/rubygems/custom_require.rb
    10 c:/ruby193/lib/ruby/site_ruby/1.9.1/rubygems.rb
    11 c:/ruby193/lib/ruby/1.9.1/i386-mingw32/enc/utf_16le.so
    12 c:/ruby193/lib/ruby/1.9.1/i386-mingw32/enc/trans/utf_16_32.so
    13 c:/ruby193/lib/ruby/1.9.1/i386-mingw32/enc/trans/single_byte.so

[NOTE]
You may have encountered a bug in the Ruby interpreter or extension libraries.
Bug reports are welcome.
For details: http://www.ruby-lang.org/bugreport.html

This application has requested the Runtime to terminate it in an unusual way.
Please contact the application's support team for more information.

On ruby 1.9.3p374:

c:\test>ruby --version
ruby 1.9.3p374 (2013-01-15) [i386-mingw32]

c:\test>chcp 437
Active code page: 437

c:\test>ruby test65001.rb
a
Ã

c:\test>chcp 65001
Active code page: 65001

c:\test>ruby test65001.rb
a
├test65001.rb:3: [BUG] rb_sys_fail_str() - errno == 0
ruby 1.9.3p374 (2013-01-15) [i386-mingw32]

-- Control frame information -----------------------------------------------
c:0006 p:---- s:0014 b:0014 l:000013 d:000013 CFUNC :write
c:0005 p:---- s:0012 b:0012 l:000011 d:000011 CFUNC :puts
c:0004 p:---- s:0010 b:0010 l:000009 d:000009 CFUNC :puts
c:0003 p:0023 s:0006 b:0006 l:001afc d:00059c EVAL test65001.rb:3
c:0002 p:---- s:0004 b:0004 l:000003 d:000003 FINISH
c:0001 p:0000 s:0002 b:0002 l:001afc d:001afc TOP

-- Ruby level backtrace information ----------------------------------------
test65001.rb:3:in <main>'
test65001.rb:3:in
puts'
test65001.rb:3:in puts'
test65001.rb:3:in
write'

-- C level backtrace information -------------------------------------------
C:\Windows\SysWOW64\ntdll.dll(ZwWaitForSingleObject+0x15) [0x77bbf8b1]
C:\Windows\syswow64\kernel32.dll(WaitForSingleObjectEx+0x43) [0x772d1194]
C:\Windows\syswow64\kernel32.dll(WaitForSingleObject+0x12) [0x772d1148]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_vm_bugreport+0xf9) [0x62e5c661]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_name_err_mesg_new+0x17a) [0x62d3a81a]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_bug+0x2f) [0x62d3b4f7]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_sys_fail_str+0x15f) [0x62d3d3a3]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_io_flush+0x1336) [0x62d64e22]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_vm_invoke_proc+0x2fe) [0x62e5498a]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_funcall+0x59) [0x62e553e9]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_io_puts+0x78) [0x62d60484]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_vm_mark+0x54d) [0x62e49b85]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_vm_invoke_proc+0x2fe) [0x62e5498a]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_io_puts+0x248) [0x62d60654]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_vm_mark+0x54d) [0x62e49b85]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_vm_call+0x2b2) [0x62e5882e]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_vm_localjump_error+0x1f84) [0x62e4f1b0]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_vm_localjump_error+0x6496) [0x62e536c2]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_iseq_eval_main+0xd7) [0x62e597ef]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(rb_check_frozen+0x2a85) [0x62d4055d]
c:\Ruby193p374\bin\msvcrt-ruby191.dll(ruby_run_node+0x48) [0x62d42688]
[0x0040136f]
[0x004010b9]
[0x00401284]
C:\Windows\syswow64\kernel32.dll(BaseThreadInitThunk+0x12) [0x772d339a]
C:\Windows\SysWOW64\ntdll.dll(RtlInitializeExceptionChain+0x63) [0x77bd9ef2]

-- Other runtime information -----------------------------------------------

  • Loaded script: test65001.rb

  • Loaded features:

    0 enumerator.so
    1 c:/Ruby193p374/lib/ruby/1.9.1/i386-mingw32/enc/encdb.so
    2 c:/Ruby193p374/lib/ruby/1.9.1/i386-mingw32/enc/iso_8859_1.so
    3 c:/Ruby193p374/lib/ruby/1.9.1/i386-mingw32/enc/trans/transdb.so
    4 c:/Ruby193p374/lib/ruby/site_ruby/1.9.1/rubygems/defaults.rb
    5 c:/Ruby193p374/lib/ruby/1.9.1/i386-mingw32/rbconfig.rb
    6 c:/Ruby193p374/lib/ruby/site_ruby/1.9.1/rubygems/deprecate.rb
    7 c:/Ruby193p374/lib/ruby/site_ruby/1.9.1/rubygems/exceptions.rb
    8 c:/Ruby193p374/lib/ruby/site_ruby/1.9.1/rubygems/defaults/operating_system.rb
    9 c:/Ruby193p374/lib/ruby/site_ruby/1.9.1/rubygems/custom_require.rb
    10 c:/Ruby193p374/lib/ruby/site_ruby/1.9.1/rubygems.rb
    11 c:/Ruby193p374/lib/ruby/1.9.1/i386-mingw32/enc/utf_16le.so
    12 c:/Ruby193p374/lib/ruby/1.9.1/i386-mingw32/enc/trans/utf_16_32.so
    13 c:/Ruby193p374/lib/ruby/1.9.1/i386-mingw32/enc/trans/single_byte.so

[NOTE]
You may have encountered a bug in the Ruby interpreter or extension libraries.
Bug reports are welcome.
For details: http://www.ruby-lang.org/bugreport.html

This application has requested the Runtime to terminate it in an unusual way.
Please contact the application's support team for more information.

Based on the line c:/Ruby193p374/lib/ruby/1.9.1/i386-mingw32/enc/iso_8859_1.so it makes me think ruby is looking at the registry for the active code page (cp1252) instead of calling GetConsoleOutputCP to see which codepage it should be using, but that is just a guess.

=end

win32-bug-7758.diff Magnifier (2.54 KB) Nobuyoshi Nakada, 02/15/2013 01:38 PM

History

#1 Updated by Luis Lavena over 2 years ago

  • Status changed from Open to Feedback
  • Assignee set to Luis Lavena

Can you tell us more about the specific version of Windows that you're using?

Nowhere in your output says which version of Windows is.

Have you tried this against Ruby 2.0 too?

#2 Updated by Heesob Park over 2 years ago

I confirmed this bug occurs on Windows 7 with ruby 1.9.3p374 and 2.0 trunk.

I tested with MinGW32 gcc compiled version(mingw32) and MSVC++ compiled version(mswin32).

When the windows console font is a console with raster font, this bug occurs only on mingw32.
But when the windows console font is a true type font, both mingw32 and mswin32 works fine.
So this is a mingw32 specific bug.

I found the main cause of this bug is _write function behavior.
The _write call in rb_w32_write function works differently on mingw32 version from mswin32 version.

Here is a workaround patch:

diff --git a/win32.c b/win32.c.new
index 984e03b..b99e6ba 100644
--- a/win32.c
+++ b/win32.c.new
@@ -6052,6 +6052,9 @@ rb_w32_write(int fd, const void *buf, size_t size)

     if ((_osfile(fd) & FTEXT) &&
         (!(_osfile(fd) & FPIPE) || fd == fileno(stdout) || fd == fileno(stderr))) {
+#ifdef __MINGW32__
+        _setmode(fd, _O_BINARY);
+#endif
    return _write(fd, buf, size);
     }

#3 Updated by Luis Lavena about 2 years ago

  • Assignee changed from Luis Lavena to Usaku NAKAMURA
  • Status changed from Feedback to Assigned

#4 Updated by Luis Lavena about 2 years ago

Usa, do you agree in resolution?

I haven't tested this against trunk or ruby_2_0_0 branch to confirm it happen too, but most likely.

#5 Updated by Usaku NAKAMURA about 2 years ago

  • Assignee changed from Usaku NAKAMURA to Yusuke Endoh

Sorry for late reply, luis.
Please commit it to trunk.

mame-san, this is reproducible SEGV bug.
So I think we need to backport it into ruby_2_0_0 (and ruby_1_9_3).

#6 Updated by Yusuke Endoh about 2 years ago

  • Assignee changed from Yusuke Endoh to Luis Lavena

Usa-san, thank you for the information!

Luis, could you please commit it to both trunk and ruby_2_0_0 branch?

Yusuke Endoh mame@tsg.ne.jp

#7 Updated by Hiroshi Shirosaki about 2 years ago

phasis68 (Heesob Park) wrote:

diff --git a/win32.c b/win32.c.new
index 984e03b..b99e6ba 100644
--- a/win32.c
+++ b/win32.c.new
@@ -6052,6 +6052,9 @@ rb_w32_write(int fd, const void *buf, size_t size)

     if ((_osfile(fd) & FTEXT) &&
         (!(_osfile(fd) & FPIPE) || fd == fileno(stdout) || fd == fileno(stderr))) {
+#ifdef __MINGW32__
+        _setmode(fd, _O_BINARY);
+#endif
  return _write(fd, buf, size);
     }

Above patch will change newline from CRLF to LF. Is it good to change?

ruby 2.0.0dev (2013-02-14 trunk 39231) [i386-mingw32] with the patch.

$ miniruby.exe test65001.rb |od -t x1
0000000 61 0a e2 94 9c 0a
0000006

I was not reproduce SEGV on my windows xp machine for some reason.

#8 Updated by Heesob Park about 2 years ago

I think the above patch is problematic.

Here is another possible patch:

diff --git a/io.c b/io.c.new
index a50d362..e393691 100644
--- a/io.c
+++ b/io.c.new
@@ -1326,6 +1326,7 @@ do_writeconv(VALUE str, rb_io_t *fptr)
 static long
 io_fwrite(VALUE str, rb_io_t *fptr, int nosync)
 {
+long n;
 #ifdef _WIN32
     if (fptr->mode & FMODE_TTY) {
    long len = rb_w32_write_console(str, fptr->fd);
@@ -1333,8 +1334,19 @@ io_fwrite(VALUE str, rb_io_t *fptr, int nosync)
     }
 #endif
     str = do_writeconv(str, fptr);
-    return io_binwrite(str, RSTRING_PTR(str), RSTRING_LEN(str),
+#ifdef __MINGW32__
+    if (fptr->mode & FMODE_TTY) {
+   setmode(fptr->fd, O_BINARY);
+    }
+#endif
+    n = io_binwrite(str, RSTRING_PTR(str), RSTRING_LEN(str),
               fptr, nosync);
+#ifdef __MINGW32__
+    if (fptr->mode & FMODE_TTY) {
+   setmode(fptr->fd, O_TEXT);
+    }
+#endif
+    return n;
 }

 ssize_t

#9 Updated by Nobuyoshi Nakada about 2 years ago

Although I can't reproduce it at all, I think it would be in
win32/win32.c but not in io.c.

#10 Updated by Heesob Park about 2 years ago

Here is an equivalent patch for win32.c:

diff --git a/win32.c b/win32.c.new
index 984e03b..70af7f8 100644
--- a/win32.c
+++ b/win32.c.new
@@ -6052,6 +6052,14 @@ rb_w32_write(int fd, const void *buf, size_t size)

     if ((_osfile(fd) & FTEXT) &&
         (!(_osfile(fd) & FPIPE) || fd == fileno(stdout) || fd == fileno(stderr))) {
+#ifdef __MINGW32__ 
+   if (rb_w32_isatty(fd)) {
+       setmode(fd, O_BINARY);
+       ret = _write(fd, buf, size);
+       setmode(fd, O_TEXT);
+       return ret;
+   }
+#endif 
    return _write(fd, buf, size);
     }

#11 Updated by Nobuyoshi Nakada about 2 years ago

Seems nice.

#12 Updated by Luis Lavena almost 2 years ago

Hello,

I missed this entire thread a few months ago, sorry about that.

Picking up this, is not clear to me if both io_fwrite and win32 rb_w32_write needs to be patched together.

Heesob Park, can you confirm?

Thank you.

#13 Updated by Heesob Park almost 2 years ago

The patch for io_fwrite is equivalent to the patch for rb_w32_write.

Only rb_w32_write needs to be patched.

Also available in: Atom PDF