Project

General

Profile

Bug #18133 » mmap.patch

peterzhu2118 (Peter Zhu), 12/14/2021 08:39 PM

View differences:

configure.ac
AC_CHECK_FUNCS(mkfifo)
AC_CHECK_FUNCS(mknod)
AC_CHECK_FUNCS(mktime)
AC_CHECK_FUNCS(mmap)
AC_CHECK_FUNCS(openat)
AC_CHECK_FUNCS(pipe2)
AC_CHECK_FUNCS(poll)
......
rb_cv_fork_with_pthread=yes)])
test x$rb_cv_fork_with_pthread = xyes || AC_DEFINE(CANNOT_FORK_WITH_PTHREAD)
])
AC_CHECK_HEADERS([sys/user.h])
AS_IF([test "x$ac_cv_func_mmap" = xyes], [
AC_CACHE_CHECK([whether PAGE_SIZE is compile-time const], rb_cv_const_page_size,
[malloc_headers=`sed -n '/MALLOC_HEADERS_BEGIN/,/MALLOC_HEADERS_END/p' ${srcdir}/gc.c`
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$malloc_headers
typedef char conftest_page[PAGE_SIZE];
]], [[]])],
[rb_cv_const_page_size=yes],
[rb_cv_const_page_size=no])])
])
AS_IF([test "x$rb_cv_const_page_size" = xyes],
[AC_DEFINE(HAVE_CONST_PAGE_SIZE, 1)],
[AC_DEFINE(HAVE_CONST_PAGE_SIZE, 0)]
)
}
: "runtime section" && {
gc.c
#include <stdarg.h>
#include <stdio.h>
/* MALLOC_HEADERS_BEGIN */
#ifndef HAVE_MALLOC_USABLE_SIZE
# ifdef _WIN32
# define HAVE_MALLOC_USABLE_SIZE
......
# endif
#endif
#if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H)
/* LIST_HEAD conflicts with sys/queue.h on macOS */
# include <sys/user.h>
#endif
/* MALLOC_HEADERS_END */
#ifdef HAVE_SYS_TIME_H
# include <sys/time.h>
#endif
......
HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),
HEAP_PAGE_BITMAP_PLANES = 4 /* RGENGC: mark, unprotected, uncollectible, marking */
};
#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG)
#define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN
#ifdef HAVE_MMAP
# if HAVE_CONST_PAGE_SIZE
/* If we have the HEAP_PAGE and it is a constant, then we can directly use it. */
static const bool USE_MMAP_ALIGNED_ALLOC = (PAGE_SIZE <= HEAP_PAGE_SIZE);
# elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE)
/* PAGE_SIZE <= HEAP_PAGE_SIZE */
static const bool USE_MMAP_ALIGNED_ALLOC = true;
# else
/* Otherwise, fall back to determining if we can use mmap during runtime. */
# define USE_MMAP_ALIGNED_ALLOC (use_mmap_aligned_alloc != false)
static bool use_mmap_aligned_alloc;
# endif
#elif !defined(__MINGW32__) && !defined(_WIN32)
static const bool USE_MMAP_ALIGNED_ALLOC = false;
#endif
struct heap_page {
short total_slots;
......
heap->total_slots -= page->total_slots;
}
static void rb_aligned_free(void *ptr);
static void rb_aligned_free(void *ptr, size_t size);
static void
heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
{
heap_allocated_pages--;
objspace->profile.total_freed_pages++;
rb_aligned_free(GET_PAGE_BODY(page->start));
rb_aligned_free(GET_PAGE_BODY(page->start), HEAP_PAGE_SIZE);
free(page);
}
......
/* assign heap_page entry */
page = calloc1(sizeof(struct heap_page));
if (page == 0) {
rb_aligned_free(page_body);
rb_aligned_free(page_body, HEAP_PAGE_SIZE);
rb_memerror();
}
......
{
rb_objspace_t *objspace = &rb_objspace;
#if defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
/* If Ruby's heap pages are not a multiple of the system page size, we
* cannot use mprotect for the read barrier, so we must disable automatic
* compaction. */
int pagesize;
pagesize = (int)sysconf(_SC_PAGE_SIZE);
if ((HEAP_PAGE_SIZE % pagesize) != 0) {
ruby_enable_autocompact = 0;
}
#if defined(HAVE_MMAP) && !HAVE_CONST_PAGE_SIZE && !defined(PAGE_MAX_SIZE)
/* Need to determine if we can use mmap at runtime. */
# ifdef PAGE_SIZE
/* If the PAGE_SIZE macro can be used. */
use_mmap_aligned_alloc = PAGE_SIZE <= HEAP_PAGE_SIZE;
# elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
/* If we can use sysconf to determine the page size. */
use_mmap_aligned_alloc = sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE;
# else
/* Otherwise we can't determine the system page size, so don't use mmap. */
use_mmap_aligned_alloc = FALSE;
# endif
#endif
objspace->next_object_id = INT2FIX(OBJ_ID_INITIAL);
......
/* For now, compact implies full mark / sweep, so ignore other flags */
if (RTEST(compact)) {
/* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for
* the read barrier, so we must disable compaction. */
#if !defined(__MINGW32__) && !defined(_WIN32)
if (!USE_MMAP_ALIGNED_ALLOC) {
rb_raise(rb_eNotImpError, "Compaction isn't available on this platform");
}
#endif
reason |= GPR_FLAG_COMPACT;
} else {
if (!RTEST(full_mark)) reason &= ~GPR_FLAG_FULL_MARK;
......
static VALUE
gc_set_auto_compact(rb_execution_context_t *ec, VALUE _, VALUE v)
{
#if defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
/* If Ruby's heap pages are not a multiple of the system page size, we
* cannot use mprotect for the read barrier, so we must disable automatic
* compaction. */
int pagesize;
pagesize = (int)sysconf(_SC_PAGE_SIZE);
if ((HEAP_PAGE_SIZE % pagesize) != 0) {
/* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for
* the read barrier, so we must disable automatic compaction. */
#if !defined(__MINGW32__) && !defined(_WIN32)
if (!USE_MMAP_ALIGNED_ALLOC) {
rb_raise(rb_eNotImpError, "Automatic compaction isn't available on this platform");
}
#endif
ruby_enable_autocompact = RTEST(v);
return v;
}
......
#elif defined _WIN32
void *_aligned_malloc(size_t, size_t);
res = _aligned_malloc(size, alignment);
#elif defined(HAVE_POSIX_MEMALIGN)
if (posix_memalign(&res, alignment, size) == 0) {
return res;
#else
if (USE_MMAP_ALIGNED_ALLOC) {
GC_ASSERT(alignment % sysconf(_SC_PAGE_SIZE) == 0);
char *ptr = mmap(NULL, alignment + size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (ptr == MAP_FAILED) {
return NULL;
}
char *aligned = ptr + alignment;
aligned -= ((VALUE)aligned & (alignment - 1));
GC_ASSERT(aligned > ptr);
GC_ASSERT(aligned <= ptr + alignment);
size_t start_out_of_range_size = aligned - ptr;
GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
if (start_out_of_range_size > 0) {
if (munmap(ptr, start_out_of_range_size)) {
rb_bug("rb_aligned_malloc: munmap failed for start");
}
}
size_t end_out_of_range_size = alignment - start_out_of_range_size;
GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
if (end_out_of_range_size > 0) {
if (munmap(aligned + size, end_out_of_range_size)) {
rb_bug("rb_aligned_malloc: munmap failed for end");
}
}
res = (void *)aligned;
}
else {
return NULL;
# if defined(HAVE_POSIX_MEMALIGN)
if (posix_memalign(&res, alignment, size) != 0) {
return NULL;
}
# elif defined(HAVE_MEMALIGN)
res = memalign(alignment, size);
# else
char* aligned;
res = malloc(alignment + size + sizeof(void*));
aligned = (char*)res + alignment + sizeof(void*);
aligned -= ((VALUE)aligned & (alignment - 1));
((void**)aligned)[-1] = res;
res = (void*)aligned;
# endif
}
#elif defined(HAVE_MEMALIGN)
res = memalign(alignment, size);
#else
char* aligned;
res = malloc(alignment + size + sizeof(void*));
aligned = (char*)res + alignment + sizeof(void*);
aligned -= ((VALUE)aligned & (alignment - 1));
((void**)aligned)[-1] = res;
res = (void*)aligned;
#endif
/* alignment must be a power of 2 */
......
}
static void
rb_aligned_free(void *ptr)
rb_aligned_free(void *ptr, size_t size)
{
#if defined __MINGW32__
__mingw_aligned_free(ptr);
#elif defined _WIN32
_aligned_free(ptr);
#elif defined(HAVE_MEMALIGN) || defined(HAVE_POSIX_MEMALIGN)
free(ptr);
#else
free(((void**)ptr)[-1]);
if (USE_MMAP_ALIGNED_ALLOC) {
GC_ASSERT(size % sysconf(_SC_PAGE_SIZE) == 0);
if (munmap(ptr, size)) {
rb_bug("rb_aligned_free: munmap failed");
}
}
else {
# if defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN)
free(ptr);
# else
free(((void**)ptr)[-1]);
# endif
}
#endif
}
(1-1/2)