diff --git a/io.c b/io.c index 0f6afb7..0fc01cd 100644 --- a/io.c +++ b/io.c @@ -10355,6 +10355,7 @@ struct copy_stream_struct { const char *notimp; rb_fdset_t fds; VALUE th; + int offload; }; static void * @@ -10484,6 +10485,113 @@ nogvl_copy_stream_wait_write(struct copy_stream_struct *stp) return 0; } +#if defined __linux__ && defined __NR_copy_file_range +# define USE_COPY_FILE_RANGE +#endif + +#ifdef USE_COPY_FILE_RANGE + +static ssize_t +simple_copy_file_range(int in_fd, off_t *in_offset, int out_fd, off_t *out_offset, size_t count, unsigned int flags) +{ + return syscall(__NR_copy_file_range, in_fd, in_offset, out_fd, out_offset, count, flags); +} + +static int +nogvl_copy_file_range(struct copy_stream_struct *stp) +{ + struct stat src_stat, dst_stat; + ssize_t ss; + int ret; + + off_t copy_length, src_offset, *src_offset_ptr; + + ret = fstat(stp->src_fd, &src_stat); + if (ret == -1) { + stp->syserr = "fstat"; + stp->error_no = errno; + return -1; + } + if (!S_ISREG(src_stat.st_mode)) + return 0; + + ret = fstat(stp->dst_fd, &dst_stat); + if (ret == -1) { + stp->syserr = "fstat"; + stp->error_no = errno; + return -1; + } + + src_offset = stp->src_offset; + if (src_offset != (off_t)-1) { + src_offset_ptr = &src_offset; + } + else { + src_offset_ptr = NULL; /* if src_offset_ptr is NULL, then bytes are read from in_fd starting from the file offset */ + } + + copy_length = stp->copy_length; + if (copy_length == (off_t)-1) { + if (src_offset == (off_t)-1) { + off_t current_offset; + errno = 0; + current_offset = lseek(stp->src_fd, 0, SEEK_CUR); + if (current_offset == (off_t)-1 && errno) { + stp->syserr = "lseek"; + stp->error_no = errno; + return -1; + } + copy_length = src_stat.st_size - current_offset; + } + else { + copy_length = src_stat.st_size - src_offset; + } + } + + retry_copy_file_range: +# if SIZEOF_OFF_T > SIZEOF_SIZE_T + /* we are limited by the 32-bit ssize_t return value on 32-bit */ + ss = (copy_length > (off_t)SSIZE_MAX) ? SSIZE_MAX : (ssize_t)copy_length; +# else + ss = (ssize_t)copy_length; +# endif + ss = simple_copy_file_range(stp->src_fd, src_offset_ptr, stp->dst_fd, NULL, ss, 0); + if (0 < ss) { + stp->total += ss; + copy_length -= ss; + if (0 < copy_length) { + goto retry_copy_file_range; + } + } + if (ss == -1) { + if (maygvl_copy_stream_continue_p(0, stp)) { + goto retry_copy_file_range; + } + switch (errno) { + case EINVAL: +#ifdef ENOSYS + case ENOSYS: +#endif +#ifdef EXDEV + case EXDEV: /* in_fd and out_fd are not on the same filesystem */ +#endif + return 0; + case EAGAIN: +#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN + case EWOULDBLOCK: +#endif + if (nogvl_copy_stream_wait_write(stp) == -1) + return -1; + goto retry_copy_file_range; + } + stp->syserr = "copy_file_range"; + stp->error_no = errno; + return -1; + } + return 1; +} +#endif + #ifdef HAVE_SENDFILE # ifdef __linux__ @@ -10782,6 +10890,14 @@ nogvl_copy_stream_func(void *arg) int ret; #endif +#ifdef USE_COPY_FILE_RANGE + if (stp->offload) { + ret = nogvl_copy_file_range(stp); + if (ret != 0) + goto finish; /* error or success */ + } +#endif + #ifdef USE_SENDFILE ret = nogvl_copy_stream_sendfile(stp); if (ret != 0) @@ -11036,12 +11152,12 @@ copy_stream_finalize(VALUE arg) static VALUE rb_io_s_copy_stream(int argc, VALUE *argv, VALUE io) { - VALUE src, dst, length, src_offset; + VALUE src, dst, length, src_offset, opt; struct copy_stream_struct st; MEMZERO(&st, struct copy_stream_struct, 1); - rb_scan_args(argc, argv, "22", &src, &dst, &length, &src_offset); + rb_scan_args(argc, argv, "22:", &src, &dst, &length, &src_offset, &opt); st.src = src; st.dst = dst; @@ -11056,6 +11172,17 @@ rb_io_s_copy_stream(int argc, VALUE *argv, VALUE io) else st.src_offset = NUM2OFFT(src_offset); + st.offload = 0; + if (!NIL_P(opt)) { + static ID offload_id; + VALUE offload; + CONST_ID(offload_id, "offload"); + rb_get_kwargs(opt, &offload_id, 0, 1, &offload); + if (offload != Qundef && RTEST(offload)) { + st.offload = 1; + } + } + rb_fd_init(&st.fds); rb_ensure(copy_stream_body, (VALUE)&st, copy_stream_finalize, (VALUE)&st); diff --git a/test/ruby/test_io.rb b/test/ruby/test_io.rb index 402d497..4e556ac 100644 --- a/test/ruby/test_io.rb +++ b/test/ruby/test_io.rb @@ -366,6 +366,16 @@ def test_copy_stream_small } end + def test_copy_stream_small_offload + mkcdtmpdir { + content = "foobar" + File.write("src", content) + ret = IO.copy_stream("src", "dst", offload: true) + assert_equal(content.bytesize, ret) + assert_equal(content, File.read("dst")) + } + end + def test_copy_stream_smaller with_srccontent {|src, content| @@ -415,6 +425,19 @@ def test_copy_stream_pipe } end + def test_copy_stream_pipe_offload + with_srccontent {|src, content| + pipe(proc do |w| + # it should fallback to read/write + ret = IO.copy_stream(src, w, offload: true) + assert_equal(content.bytesize, ret) + w.close + end, proc do |r| + assert_equal(content, r.read) + end) + } + end + def test_copy_stream_write_pipe with_srccontent {|src, content| with_pipe {|r, w| @@ -424,6 +447,15 @@ def test_copy_stream_write_pipe } end + def test_copy_stream_write_pipe_offload + with_srccontent {|src, content| + with_pipe {|r, w| + w.close + assert_raise(IOError) { IO.copy_stream(src, w, offload: true) } + } + } + end + def with_pipecontent mkcdtmpdir { yield "abc" @@ -635,6 +667,42 @@ def test_copy_stream_closed_pipe } end + def test_copy_stream_bigcontent_offload + with_bigsrc {|bigsrc, bigcontent| + ret = IO.copy_stream(bigsrc, "bigdst", offload: true) + assert_equal(bigcontent.bytesize, ret) + assert_equal(bigcontent, File.read("bigdst")) + } + end + + def test_copy_stream_bigcontent_mid_offload + with_bigsrc {|bigsrc, bigcontent| + ret = IO.copy_stream(bigsrc, "bigdst", 30000, 100, offload: true) + assert_equal(30000, ret) + assert_equal(bigcontent[100, 30000], File.read("bigdst")) + } + end + + def test_copy_stream_bigcontent_fpos_offload + with_bigsrc {|bigsrc, bigcontent| + File.open(bigsrc) {|f| + begin + assert_equal(0, f.pos) + ret = IO.copy_stream(f, "bigdst", nil, 10, offload: true) + assert_equal(bigcontent.bytesize-10, ret) + assert_equal(bigcontent[10..-1], File.read("bigdst")) + assert_equal(0, f.pos) + ret = IO.copy_stream(f, "bigdst", 40, 30, offload: true) + assert_equal(40, ret) + assert_equal(bigcontent[30, 40], File.read("bigdst")) + assert_equal(0, f.pos) + rescue NotImplementedError + #skip "pread(2) is not implemtented." + end + } + } + end + def with_megacontent yield "abc" * 1234567 end