From bcab8c3cd877506de75f50e0f9ed98827ed554b0 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 23 Feb 2021 16:28:56 -0500 Subject: [PATCH] Use mmap for allocating heap pages --- configure.ac | 16 ++++ gc.c | 149 ++++++++++++++++++++++++++--------- test/ruby/test_gc_compact.rb | 41 ++++++---- 3 files changed, 155 insertions(+), 51 deletions(-) diff --git a/configure.ac b/configure.ac index 2dcebdde9f..b1b190004d 100644 --- a/configure.ac +++ b/configure.ac @@ -1944,6 +1944,7 @@ AC_CHECK_FUNCS(memmem) AC_CHECK_FUNCS(mkfifo) AC_CHECK_FUNCS(mknod) AC_CHECK_FUNCS(mktime) +AC_CHECK_FUNCS(mmap) AC_CHECK_FUNCS(openat) AC_CHECK_FUNCS(pipe2) AC_CHECK_FUNCS(poll) @@ -2666,6 +2667,21 @@ main(int argc, char *argv[]) rb_cv_fork_with_pthread=yes)]) test x$rb_cv_fork_with_pthread = xyes || AC_DEFINE(CANNOT_FORK_WITH_PTHREAD) ]) + +AC_CHECK_HEADERS([sys/user.h]) +AS_IF([test "x$ac_cv_func_mmap" = xyes], [ + AC_CACHE_CHECK([whether PAGE_SIZE is compile-time const], rb_cv_const_page_size, + [malloc_headers=`sed -n '/MALLOC_HEADERS_BEGIN/,/MALLOC_HEADERS_END/p' ${srcdir}/gc.c` + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$malloc_headers + typedef char conftest_page[PAGE_SIZE]; + ]], [[]])], + [rb_cv_const_page_size=yes], + [rb_cv_const_page_size=no])]) +]) +AS_IF([test "x$rb_cv_const_page_size" = xyes], + [AC_DEFINE(HAVE_CONST_PAGE_SIZE, 1)], + [AC_DEFINE(HAVE_CONST_PAGE_SIZE, 0)] +) } : "runtime section" && { diff --git a/gc.c b/gc.c index f6acf3e117..6f8e5f242d 100644 --- a/gc.c +++ b/gc.c @@ -32,6 +32,7 @@ #include #include +/* MALLOC_HEADERS_BEGIN */ #ifndef HAVE_MALLOC_USABLE_SIZE # ifdef _WIN32 # define HAVE_MALLOC_USABLE_SIZE @@ -54,6 +55,12 @@ # endif #endif +#if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H) +/* LIST_HEAD conflicts with sys/queue.h on macOS */ +# include +#endif +/* MALLOC_HEADERS_END */ + #ifdef HAVE_SYS_TIME_H # include #endif @@ -821,6 +828,25 @@ enum { HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT), HEAP_PAGE_BITMAP_PLANES = 4 /* RGENGC: mark, unprotected, uncollectible, marking */ }; +#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG) +#define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN + +#ifdef HAVE_MMAP +# if HAVE_CONST_PAGE_SIZE +/* If we have the HEAP_PAGE and it is a constant, then we can directly use it. */ +static const bool USE_MMAP_ALIGNED_ALLOC = (PAGE_SIZE <= HEAP_PAGE_SIZE); +# elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE) +/* PAGE_SIZE <= HEAP_PAGE_SIZE */ +static const bool USE_MMAP_ALIGNED_ALLOC = true; +# else +/* Otherwise, fall back to determining if we can use mmap during runtime. */ +# define USE_MMAP_ALIGNED_ALLOC (use_mmap_aligned_alloc != false) + +static bool use_mmap_aligned_alloc; +# endif +#elif !defined(__MINGW32__) && !defined(_WIN32) +static const bool USE_MMAP_ALIGNED_ALLOC = false; +#endif struct heap_page { short total_slots; @@ -1752,14 +1778,14 @@ heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *pag heap->total_slots -= page->total_slots; } -static void rb_aligned_free(void *ptr); +static void rb_aligned_free(void *ptr, size_t size); static void heap_page_free(rb_objspace_t *objspace, struct heap_page *page) { heap_allocated_pages--; objspace->profile.total_freed_pages++; - rb_aligned_free(GET_PAGE_BODY(page->start)); + rb_aligned_free(GET_PAGE_BODY(page->start), HEAP_PAGE_SIZE); free(page); } @@ -1811,7 +1837,7 @@ heap_page_allocate(rb_objspace_t *objspace) /* assign heap_page entry */ page = calloc1(sizeof(struct heap_page)); if (page == 0) { - rb_aligned_free(page_body); + rb_aligned_free(page_body, HEAP_PAGE_SIZE); rb_memerror(); } @@ -3151,15 +3177,18 @@ Init_heap(void) { rb_objspace_t *objspace = &rb_objspace; -#if defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE) - /* If Ruby's heap pages are not a multiple of the system page size, we - * cannot use mprotect for the read barrier, so we must disable automatic - * compaction. */ - int pagesize; - pagesize = (int)sysconf(_SC_PAGE_SIZE); - if ((HEAP_PAGE_SIZE % pagesize) != 0) { - ruby_enable_autocompact = 0; - } +#if defined(HAVE_MMAP) && !HAVE_CONST_PAGE_SIZE && !defined(PAGE_MAX_SIZE) + /* Need to determine if we can use mmap at runtime. */ +# ifdef PAGE_SIZE + /* If the PAGE_SIZE macro can be used. */ + use_mmap_aligned_alloc = PAGE_SIZE <= HEAP_PAGE_SIZE; +# elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE) + /* If we can use sysconf to determine the page size. */ + use_mmap_aligned_alloc = sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE; +# else + /* Otherwise we can't determine the system page size, so don't use mmap. */ + use_mmap_aligned_alloc = FALSE; +# endif #endif objspace->next_object_id = INT2FIX(OBJ_ID_INITIAL); @@ -8535,6 +8564,14 @@ gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE /* For now, compact implies full mark / sweep, so ignore other flags */ if (RTEST(compact)) { + /* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for + * the read barrier, so we must disable compaction. */ +#if !defined(__MINGW32__) && !defined(_WIN32) + if (!USE_MMAP_ALIGNED_ALLOC) { + rb_raise(rb_eNotImpError, "Compaction isn't available on this platform"); + } +#endif + reason |= GPR_FLAG_COMPACT; } else { if (!RTEST(full_mark)) reason &= ~GPR_FLAG_FULL_MARK; @@ -9946,16 +9983,14 @@ gc_disable(rb_execution_context_t *ec, VALUE _) static VALUE gc_set_auto_compact(rb_execution_context_t *ec, VALUE _, VALUE v) { -#if defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE) - /* If Ruby's heap pages are not a multiple of the system page size, we - * cannot use mprotect for the read barrier, so we must disable automatic - * compaction. */ - int pagesize; - pagesize = (int)sysconf(_SC_PAGE_SIZE); - if ((HEAP_PAGE_SIZE % pagesize) != 0) { + /* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for + * the read barrier, so we must disable automatic compaction. */ +#if !defined(__MINGW32__) && !defined(_WIN32) + if (!USE_MMAP_ALIGNED_ALLOC) { rb_raise(rb_eNotImpError, "Automatic compaction isn't available on this platform"); } #endif + ruby_enable_autocompact = RTEST(v); return v; } @@ -10352,22 +10387,54 @@ rb_aligned_malloc(size_t alignment, size_t size) #elif defined _WIN32 void *_aligned_malloc(size_t, size_t); res = _aligned_malloc(size, alignment); -#elif defined(HAVE_POSIX_MEMALIGN) - if (posix_memalign(&res, alignment, size) == 0) { - return res; +#else + if (USE_MMAP_ALIGNED_ALLOC) { + GC_ASSERT(alignment % sysconf(_SC_PAGE_SIZE) == 0); + + char *ptr = mmap(NULL, alignment + size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (ptr == MAP_FAILED) { + return NULL; + } + + char *aligned = ptr + alignment; + aligned -= ((VALUE)aligned & (alignment - 1)); + GC_ASSERT(aligned > ptr); + GC_ASSERT(aligned <= ptr + alignment); + + size_t start_out_of_range_size = aligned - ptr; + GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0); + if (start_out_of_range_size > 0) { + if (munmap(ptr, start_out_of_range_size)) { + rb_bug("rb_aligned_malloc: munmap failed for start"); + } + } + + size_t end_out_of_range_size = alignment - start_out_of_range_size; + GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0); + if (end_out_of_range_size > 0) { + if (munmap(aligned + size, end_out_of_range_size)) { + rb_bug("rb_aligned_malloc: munmap failed for end"); + } + } + + res = (void *)aligned; } else { - return NULL; +# if defined(HAVE_POSIX_MEMALIGN) + if (posix_memalign(&res, alignment, size) != 0) { + return NULL; + } +# elif defined(HAVE_MEMALIGN) + res = memalign(alignment, size); +# else + char* aligned; + res = malloc(alignment + size + sizeof(void*)); + aligned = (char*)res + alignment + sizeof(void*); + aligned -= ((VALUE)aligned & (alignment - 1)); + ((void**)aligned)[-1] = res; + res = (void*)aligned; +# endif } -#elif defined(HAVE_MEMALIGN) - res = memalign(alignment, size); -#else - char* aligned; - res = malloc(alignment + size + sizeof(void*)); - aligned = (char*)res + alignment + sizeof(void*); - aligned -= ((VALUE)aligned & (alignment - 1)); - ((void**)aligned)[-1] = res; - res = (void*)aligned; #endif /* alignment must be a power of 2 */ @@ -10377,16 +10444,26 @@ rb_aligned_malloc(size_t alignment, size_t size) } static void -rb_aligned_free(void *ptr) +rb_aligned_free(void *ptr, size_t size) { #if defined __MINGW32__ __mingw_aligned_free(ptr); #elif defined _WIN32 _aligned_free(ptr); -#elif defined(HAVE_MEMALIGN) || defined(HAVE_POSIX_MEMALIGN) - free(ptr); #else - free(((void**)ptr)[-1]); + if (USE_MMAP_ALIGNED_ALLOC) { + GC_ASSERT(size % sysconf(_SC_PAGE_SIZE) == 0); + if (munmap(ptr, size)) { + rb_bug("rb_aligned_free: munmap failed"); + } + } + else { +# if defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN) + free(ptr); +# else + free(((void**)ptr)[-1]); +# endif + } #endif } diff --git a/test/ruby/test_gc_compact.rb b/test/ruby/test_gc_compact.rb index 4a8cff33f4..f5cab55ba7 100644 --- a/test/ruby/test_gc_compact.rb +++ b/test/ruby/test_gc_compact.rb @@ -4,12 +4,32 @@ require 'etc' class TestGCCompact < Test::Unit::TestCase - class AutoCompact < Test::Unit::TestCase + module SupportsCompact def setup skip "autocompact not supported on this platform" unless supports_auto_compact? super end + private + + def supports_auto_compact? + return true unless defined?(Etc::SC_PAGE_SIZE) + + begin + return GC::INTERNAL_CONSTANTS[:HEAP_PAGE_SIZE] % Etc.sysconf(Etc::SC_PAGE_SIZE) == 0 + rescue NotImplementedError + rescue ArgumentError + end + + true + end + end + + include SupportsCompact + + class AutoCompact < Test::Unit::TestCase + include SupportsCompact + def test_enable_autocompact before = GC.auto_compact GC.auto_compact = true @@ -59,26 +79,17 @@ def test_implicit_compaction_does_something ensure GC.auto_compact = before end - - private - - def supports_auto_compact? - return true unless defined?(Etc::SC_PAGE_SIZE) - - begin - return GC::INTERNAL_CONSTANTS[:HEAP_PAGE_SIZE] % Etc.sysconf(Etc::SC_PAGE_SIZE) == 0 - rescue NotImplementedError - rescue ArgumentError - end - - true - end end def os_page_size return true unless defined?(Etc::SC_PAGE_SIZE) end + def setup + skip "autocompact not supported on this platform" unless supports_auto_compact? + super + end + def test_gc_compact_stats list = [] -- 2.30.1 (Apple Git-130)