Project

General

Profile

Feature #14955 ยป 0001-gc.c-use-MADV_FREE-to-release-most-the-heap-page-bod.patch

normalperson (Eric Wong), 08/01/2018 12:44 AM

View differences:

configure.ac
988 988
AC_CHECK_HEADERS(sys/id.h)
989 989
AC_CHECK_HEADERS(sys/ioctl.h)
990 990
AC_CHECK_HEADERS(sys/mkdev.h)
991
AC_CHECK_HEADERS(sys/mman.h)
991 992
AC_CHECK_HEADERS(sys/param.h)
992 993
AC_CHECK_HEADERS(sys/prctl.h)
993 994
AC_CHECK_HEADERS(sys/resource.h)
......
1799 1800
AC_CHECK_FUNCS(lutimes)
1800 1801
AC_CHECK_FUNCS(malloc_usable_size)
1801 1802
AC_CHECK_FUNCS(malloc_size)
1803
AC_CHECK_FUNCS(madvise)
1802 1804
AC_CHECK_FUNCS(mblen)
1803 1805
AC_CHECK_FUNCS(memalign)
1804 1806
AC_CHECK_FUNCS(memset_s)
gc.c
85 85
#include <malloc.h>
86 86
#endif
87 87

  
88
#if defined(HAVE_SYS_MMAN_H)
89
#  include <sys/mman.h>
90
#endif
91

  
88 92
#define rb_setjmp(env) RUBY_SETJMP(env)
89 93
#define rb_jmp_buf rb_jmpbuf_t
90 94

  
......
674 678
    HEAP_PAGE_BITMAP_PLANES = USE_RGENGC ? 4 : 1 /* RGENGC: mark, unprotected, uncollectible, marking */
675 679
};
676 680

  
681
/*
682
 * TODO: add platform-specific SC_PAGESIZE_ASSUMED definitions, here.
683
 * 4K is the most common, so we assume it by default.
684
 * Probably not worth the effort to support MADV_FREE on (rare) platforms
685
 * where HEAP_PAGE_ALIGN <= _SC_PAGESIZE, so define to 0 in that case.
686
 */
687
#if defined(__linux__) && defined(__alpha__)
688
#  define SC_PAGESIZE_ASSUMED (1 << 13) /* XXX untested */
689
#endif
690

  
691
/* this is true for x86_64 and x86 on Linux and FreeBSD */
692
#ifndef SC_PAGESIZE_ASSUMED
693
#  define SC_PAGESIZE_ASSUMED (1 << 12)
694
#endif
695

  
696
#if defined(MADV_FREE) && defined(HAVE_MADVISE) && SC_PAGESIZE_ASSUMED
697
#  define USE_MADV_FREE (1)
698
#else
699
#  define USE_MADV_FREE (0)
700
#endif
701

  
702
struct heap_page_madv_freed {
703
    char junk[HEAP_PAGE_ALIGN - SC_PAGESIZE_ASSUMED];
704
    struct list_node fnode;
705
};
706

  
707
STATIC_ASSERT(heap_page_madv_freed_size,
708
                sizeof(struct heap_page_madv_freed) <= HEAP_PAGE_SIZE);
709

  
710
#if USE_MADV_FREE
711
static long sc_pagesize;
712
static LIST_HEAD(madv_free_head);
713
#endif
714

  
677 715
struct heap_page {
678 716
    short total_slots;
679 717
    short free_slots;
......
1480 1518
    heap->total_slots -= page->total_slots;
1481 1519
}
1482 1520

  
1521
static int
1522
heap_page_madv_free_put(struct heap_page_body *body)
1523
{
1524
#if USE_MADV_FREE
1525
    static int err;
1526
    struct heap_page_madv_freed *hpmf = (struct heap_page_madv_freed *)body;
1527

  
1528
    if (err) return FALSE;
1529
    if (sc_pagesize != SC_PAGESIZE_ASSUMED) {
1530
        /* can't use rb_warning during VM shutdown */
1531
        if (RTEST(ruby_verbose)) {
1532
            fprintf(stderr,
1533
                    "unable to use MADV_FREE, page size unexpected: %ld != %u\n"
1534
                    "RUBY_PLATFORM: "RUBY_PLATFORM"\n"
1535
                    "Please report to https://bugs.ruby-lang.org/\n",
1536
                    sc_pagesize, SC_PAGESIZE_ASSUMED);
1537
        }
1538
        err = -1;
1539
        return FALSE;
1540
    }
1541
    if (madvise(hpmf->junk, sizeof(hpmf->junk), MADV_FREE) == 0) {
1542
        list_add(&madv_free_head, &hpmf->fnode);
1543
        return TRUE;
1544
    }
1545

  
1546
    err = errno;
1547
    /* can happen if libc headers are newer than kernel version */
1548
    if (RTEST(ruby_verbose)) {
1549
        fprintf(stderr, "MADV_FREE failed: %s", strerror(err));
1550
    }
1551
#endif
1552
    return FALSE;
1553
}
1554

  
1555
static void *
1556
heap_page_madv_free_get(void)
1557
{
1558
#if USE_MADV_FREE
1559
    return list_pop(&madv_free_head, struct heap_page_madv_freed, fnode);
1560
#endif
1561
    return 0;
1562
}
1563

  
1483 1564
static void
1484 1565
heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
1485 1566
{
1567
    struct heap_page_body *body = GET_PAGE_BODY(page->start);
1486 1568
    heap_allocated_pages--;
1487 1569
    objspace->profile.total_freed_pages++;
1488
    aligned_free(GET_PAGE_BODY(page->start));
1570

  
1571
    if (!heap_page_madv_free_put(body)) {
1572
	aligned_free(body);
1573
    }
1574

  
1489 1575
    free(page);
1490 1576
}
1491 1577

  
......
1523 1609
    int limit = HEAP_PAGE_OBJ_LIMIT;
1524 1610

  
1525 1611
    /* assign heap_page body (contains heap_page_header and RVALUEs) */
1526
    page_body = (struct heap_page_body *)aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
1612
    page_body = heap_page_madv_free_get();
1613
    if (!page_body) {
1614
        page_body = aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
1615
    }
1527 1616
    if (page_body == 0) {
1528 1617
	rb_memerror();
1529 1618
    }
......
1531 1620
    /* assign heap_page entry */
1532 1621
    page = (struct heap_page *)calloc(1, sizeof(struct heap_page));
1533 1622
    if (page == 0) {
1534
	aligned_free(page_body);
1535
	rb_memerror();
1623
        if (!heap_page_madv_free_put(page_body)) {
1624
            aligned_free(page_body);
1625
        }
1626
        rb_memerror();
1536 1627
    }
1537 1628

  
1538 1629
    /* adjust obj_limit (object number available in this page) */
......
2396 2487
{
2397 2488
    rb_objspace_t *objspace = &rb_objspace;
2398 2489

  
2490
#if USE_MADV_FREE
2491
    sc_pagesize = sysconf(_SC_PAGESIZE);
2492
#endif
2493

  
2399 2494
    gc_stress_set(objspace, ruby_initial_gc_stress);
2400 2495

  
2401 2496
#if RGENGC_ESTIMATE_OLDMALLOC
2402
-