Ruby » Ruby master

#include "gc.h"

// #define ARRAY_DEBUG

FL_UNSET_RAW((a), RARRAY_TRANSIENT_FLAG); \

ary_verify(a); \

assert(!ARY_TRANSIENT_P(ary)); \

#ifdef ARRAY_DEBUG

#define ary_verify(ary) ary_verify_(ary, __FILE__, __LINE__)

static void

ary_verify_(VALUE ary, const char *file, int line)

{

if (FL_TEST(ary, ELTS_SHARED)) {

VALUE root = RARRAY(ary)->as.heap.aux.shared;

const VALUE *ptr = RARRAY_CONST_PTR(ary);

const VALUE *root_ptr = RARRAY_CONST_PTR(root);

long len = RARRAY_LEN(ary), root_len = RARRAY_LEN(root);

assert(FL_TEST(root, RARRAY_SHARED_ROOT_FLAG));

assert(root_ptr <= ptr && ptr + len <= root_ptr + root_len);

ary_verify(root);

}

else if (ARY_EMBED_P(ary)) {

assert(!ARY_TRANSIENT_P(ary));

assert(!ARY_SHARED_P(ary));

assert(RARRAY_LEN(ary) <= RARRAY_EMBED_LEN_MAX);

}

else {

#if 0

const VALUE *ptr = RARRAY_CONST_PTR(ary);

long i, len = RARRAY_LEN(ary);

volatile VALUE v;

for (i=0; i<len; i++) {

v = ptr[i]; // access check

}

#endif

}

}

#else

#define ary_verify(ary) ((void)0)

#endif

void *rb_transient_heap_alloc(VALUE obj, size_t size);

static VALUE *

ary_heap_alloc(VALUE ary, size_t capa)

{

VALUE *ptr = rb_transient_heap_alloc(ary, sizeof(VALUE) * capa);

if (ptr != NULL) {

FL_SET_RAW(ary, RARRAY_TRANSIENT_FLAG);

//fprintf(stderr, "ary:%p (%p) is TRANSIENT.\n", (void *)ary, ptr);

}

else {

FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG);

ptr = ALLOC_N(VALUE, capa);

//fprintf(stderr, "ary:%p (%p) is not TRANSIENT.\n", (void *)ary, ptr);

}

return ptr;

}

static void

ary_heap_free_ptr(VALUE ary, const VALUE *ptr, long size)

{

if (ARY_TRANSIENT_P(ary)) {

/* ignore it */

// fprintf(stderr, "ary_heap_free: %p is transient.\n", (void *)ary);

}

else {

// fprintf(stderr, "ary_heap_free: %p is freed.\n", (void *)ary);

ruby_sized_xfree((void *)ptr, size);

}

}

static void

ary_heap_free(VALUE ary)

{

// fprintf(stderr, "ary_heap_free: %p\n", (void *)ary);

if (ARY_TRANSIENT_P(ary)) {

/* ignore */

}

else {

ary_heap_free_ptr(ary, ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary));

}

}

static void

ary_heap_realloc(VALUE ary, size_t new_capa)

{

size_t old_capa = RARRAY(ary)->as.heap.aux.capa;

if (ARY_TRANSIENT_P(ary)) {

if (new_capa <= old_capa) {

/* do nothing */

}

else {

VALUE *new_ptr = rb_transient_heap_alloc(ary, sizeof(VALUE) * new_capa);

if (new_ptr == NULL) {

new_ptr = ALLOC_N(VALUE, new_capa);

FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG);

}

MEMCPY(new_ptr, ARY_HEAP_PTR(ary), VALUE, old_capa);

ARY_SET_PTR(ary, new_ptr);

}

}

else {

SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, new_capa, old_capa);

}

}

void

rb_ary_detransient(VALUE ary)

{

// fprintf(stderr, "rb_ary_detransient:\n");

// fprintf(stderr, "(1) %s\n", rb_obj_info(ary));

if (ARY_TRANSIENT_P(ary)) {

VALUE *new_ptr;

long capa = RARRAY(ary)->as.heap.aux.capa;

long len = RARRAY(ary)->as.heap.len;

assert(ARY_OWNS_HEAP_P(ary));

assert(ARY_TRANSIENT_P(ary));

if (ARY_SHARED_ROOT_P(ary)) {

capa = len;

}

new_ptr = ALLOC_N(VALUE, capa);

MEMCPY(new_ptr, ARY_HEAP_PTR(ary), VALUE, capa);

RARRAY(ary)->as.heap.ptr = new_ptr;

/* do not use ARY_SET_PTR() because they assert !frozen */

FL_UNSET_RAW(ary, RARRAY_TRANSIENT_FLAG);

// fprintf(stderr, "(2) %s\n", rb_obj_info(ary));

}

}

// fprintf(stderr, "ary_resize_capa (%ld): %s\n", capacity, rb_obj_info(ary));

VALUE *ptr = ary_heap_alloc(ary, capacity);

// fprintf(stderr, "ary_resize_capa %s\n", rb_obj_info(ary));

ary_heap_realloc(ary, capacity);

ARY_SET_CAPA(ary, capacity);

// fprintf(stderr, "-> ary_resize_capa: %s\n", rb_obj_info(ary));

// fprintf(stderr, "ary_resize_capa %p len:%ld capa:%ld - %s\n", (void *)ary, RARRAY_LEN(ary), capacity, rb_obj_info(ary));

long old_capa = RARRAY(ary)->as.heap.aux.capa;

ary_heap_free_ptr(ary, ptr, old_capa);

// fprintf(stderr, "ary_resize_capa: heap->embed %p len:%ld\n", (void *)ary, len);

ary_verify(ary);

if (old_capa > capacity) ary_heap_realloc(ary, capacity);

ary_verify(ary);

// fprintf(stderr, "ary_double_capa: %p %d\n", (void *)ary, FL_TEST(ary, RARRAY_TRANSIENT_FLAG) ? 1 : 0);

ary_verify(ary);

ary_verify(ary);

ary_verify(ary);

VALUE *ptr = ary_heap_alloc(ary, len);

ary_verify(ary);

ary_verify(shared);

ary_verify(ary);

ary_verify(ary);

void rb_transient_heap_dump(void);

ptr = ary_heap_alloc(ary, capa);

VALUE ary = ary_new(0, capa);

rb_ary_detransient(ary);

return ary;

ary_heap_free(ary);

RBASIC(ary)->flags &= ~(RARRAY_EMBED_LEN_MASK | RARRAY_TRANSIENT_FLAG);

ary_verify(ary);

rb_ary_detransient(ary);

const VALUE *ptr;

rb_ary_detransient(ary);

ptr = ARY_HEAP_PTR(ary);

FL_UNSET_EMBED(shared);

ARY_SET_PTR((VALUE)shared, ptr);

ary_verify(ary);

ary_heap_free(ary);

static VALUE

ary_initialize_copy(VALUE self, VALUE orig)

{

FL_UNSET(self,

FL_USER1 | FL_USER2 | FL_USER3 | /* embed */

FL_USER5 | /* shard */

FL_USER13);

return rb_ary_replace(self, orig);

}

ary_verify(result);

ary_verify(ary);

ary_verify(ary);

ary_verify(ary);

ary_verify(ary);

ary_verify(ary);

ary_verify(ary);

ary_verify(ary);

ary_verify(ary);

ary_heap_realloc(ary, len);

ary_verify(ary);

ary_heap_free(ary);

ary_verify(ary);

ary_heap_free(copy);

ary_heap_free(copy);

ARY_SET_EMBED_LEN(ary, 0);

else {

ARY_SET_LEN(ary, 0);

if (ARY_DEFAULT_SIZE * 2 < ARY_CAPA(ary)) {

}

rb_define_method(rb_cArray, "initialize_copy", ary_initialize_copy, 1);

transient_heap.$(OBJEXT) \

transient_heap.$(OBJEXT): {$(VPATH)}transient_heap.c

void rb_transient_heap_start_marking(int full_marking);

void rb_transient_heap_finish_marking(void);

void rb_transient_heap_mark(VALUE obj, const void *ptr);

void rb_transient_heap_promote(VALUE obj);

rb_transient_heap_promote(obj);

VALUE root = any->as.array.as.heap.aux.shared;

gc_mark(objspace, root);

gc_mark(objspace, ptr[i]);

}

if (objspace->mark_func_data == NULL) {

if (!FL_TEST_RAW(obj, RARRAY_EMBED_FLAG) &&

ARY_TRANSIENT_P(obj)) {

rb_transient_heap_mark(obj, ptr);

}

rb_transient_heap_finish_marking();

rb_transient_heap_start_marking(do_full_mark);

if (is_pointer_to_heap(&rb_objspace, (void *)obj)) {

}

else {

/* fake */

snprintf(buff, buff_size, "%p [%dXXXX] %s",

(void *)obj, age,

obj_type_name(obj));

}

if (FL_TEST(obj, ELTS_SHARED)) {

snprintf(buff, buff_size, "%s shared -> %s", buff,

rb_obj_info(RARRAY(obj)->as.heap.aux.shared));

}

else if (FL_TEST(obj, RARRAY_EMBED_FLAG)) {

snprintf(buff, buff_size, "%s [%s%s] len: %d (embed)", buff,

}

else {

snprintf(buff, buff_size, "%s [%s%s%s] len: %d, capa:%d ptr:%p", buff,

C(ARY_EMBED_P(obj), "E"),

C(ARY_SHARED_P(obj), "S"),

C(ARY_TRANSIENT_P(obj), "T"),

(int)RARRAY_LEN(obj),

ARY_EMBED_P(obj) ? -1 : (int)RARRAY(obj)->as.heap.aux.capa,

RARRAY_CONST_PTR(obj));

}

CALL(TransientHeap);

#define RARRAY_TRANSIENT_FLAG FL_USER13

#define ARY_TRANSIENT_P(ary) FL_TEST_RAW((ary), RARRAY_TRANSIENT_FLAG)

#include "ruby/ruby.h"

#include "ruby/debug.h"

#include "vm_debug.h"

#include "gc.h"

#include "internal.h"

#include <sys/mman.h>

#include <errno.h>

#include "ruby_assert.h"

/*

* 1: enable assertions

* 2: enable verify

*/

#ifndef TRANSIENT_HEAP_CHECK_MODE

#define TRANSIENT_HEAP_CHECK_MODE 0

#endif

#define TH_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(TRANSIENT_HEAP_CHECK_MODE > 0, expr, #expr)

/*

* 1: show events

* 2: show dump at events

* 3: show all operations

*/

#define TRANSIENT_HEAP_DEBUG 0

/* Provide blocks infinitely for debug.

* This mode generates blocks unlimitedly

* and prohibit access free'ed blocks to check invalid access.

*/

#define TRANSIENT_HEAP_INFINITE_BLOCK_MODE 0

enum transient_heap_status {

transient_heap_none,

transient_heap_marking,

transient_heap_escaping

};

struct transient_heap_block {

struct transient_heap_block_header {

int16_t size; /* sizeof(block) = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header) */

int16_t index;

int16_t last_marked_index;

int16_t objects;

struct transient_heap_block *next_block;

} info;

char buff[];

};

struct transient_heap {

struct transient_heap_block *using_blocks;

struct transient_heap_block *marked_blocks;

struct transient_heap_block *free_blocks;

int total_objects;

int total_marked_objects;

int total_blocks;

enum transient_heap_status status;

VALUE *promoted_objects;

int promoted_objects_size;

int promoted_objects_index;

};

struct transient_alloc_header {

uint16_t magic;

uint16_t size;

int16_t next_marked_index;

int16_t dummy;

VALUE obj;

};

static struct transient_heap global_transient_heap;

#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE 1024

/* K M */

#define TRANSIENT_HEAP_TOTAL_SIZE (1024 * 1024 * 16)

#define TRANSIENT_HEAP_BLOCK_SIZE (1024 * 32 ) /* int16_t */

#define TRANSIENT_HEAP_ALLOC_MAX (1024 * 2 )

#define TRANSIENT_HEAP_BLOCK_NUM (TRANSIENT_HEAP_TOTAL_SIZE / TRANSIENT_HEAP_BLOCK_SIZE)

#define TRANSIENT_HEAP_ALLOC_MAGIC 0xfeab

#define TRANSIENT_HEAP_ALLOC_ALIGN RUBY_ALIGNOF(void *)

#define TRANSIENT_HEAP_ALLOC_MARKING_LAST -1

#define TRANSIENT_HEAP_ALLOC_MARKING_FREE -2

#define ROUND_UP(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1))

static void

transient_heap_block_dump(struct transient_heap* theap, struct transient_heap_block *block)

{

int i=0, n=0;

struct transient_alloc_header *header = NULL;

while (i<block->info.index) {

header = (void *)&block->buff[i];

fprintf(stderr, "%4d %8d %p size:%4d next:%4d %s\n", n, i, header, header->size, header->next_marked_index, rb_obj_info(header->obj));

i += header->size;

n++;

}

}

static void

transient_heap_blocks_dump(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str)

{

while (block) {

fprintf(stderr, "- transient_heap_dump: %s:%p index:%d objects:%d last_marked_index:%d next:%p\n",

type_str, block, block->info.index, block->info.objects, block->info.last_marked_index, block->info.next_block);

transient_heap_block_dump(theap, block);

block = block->info.next_block;

}

}

static void

transient_heap_dump(struct transient_heap* theap)

{

fprintf(stderr, "transient_heap_dump objects:%d marked_objects:%d blocks:%d\n", theap->total_objects, theap->total_marked_objects, theap->total_blocks);

transient_heap_blocks_dump(theap, theap->using_blocks, "using_blocks");

transient_heap_blocks_dump(theap, theap->marked_blocks, "marked_blocks");

transient_heap_blocks_dump(theap, theap->free_blocks, "free_blocks");

}

void

rb_transient_heap_dump(void)

{

transient_heap_dump(&global_transient_heap);

}

#if TRANSIENT_HEAP_CHECK_MODE >= 2

static int

transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block)

{

int i=0, n=0;

struct transient_alloc_header *header;

while (i<block->info.index) {

header = (void *)&block->buff[i];

TH_ASSERT(header->magic == TRANSIENT_HEAP_ALLOC_MAGIC);

n ++;

i += header->size;

}

TH_ASSERT(block->info.objects == n);

return n;

}

#endif

static void

transient_heap_verify(struct transient_heap *theap)

{

#if TRANSIENT_HEAP_CHECK_MODE >= 2

struct transient_heap_block *block;

int n=0;

// using_blocks

block = theap->using_blocks;

while (block) {

n += transient_heap_block_verify(theap, block);

block = block->info.next_block;

}

// marked_blocks

block = theap->marked_blocks;

while (block) {

n += transient_heap_block_verify(theap, block);

TH_ASSERT(block->info.index > 0);

block = block->info.next_block;

}

TH_ASSERT(n == theap->total_objects);

TH_ASSERT(n >= theap->total_marked_objects);

#endif

}

static struct transient_heap*

transient_heap_get(void)

{

struct transient_heap* theap = &global_transient_heap;

transient_heap_verify(theap);

return theap;

}

static void

reset_block(struct transient_heap_block *block)

{

block->info.size = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header);

block->info.index = 0;

block->info.objects = 0;

block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;

block->info.next_block = NULL;

}

static void

connect_to_free_blocks(struct transient_heap *theap, struct transient_heap_block *block)

{

block->info.next_block = theap->free_blocks;

theap->free_blocks = block;

}

static void

connect_to_using_blocks(struct transient_heap *theap, struct transient_heap_block *block)

{

block->info.next_block = theap->using_blocks;

theap->using_blocks = block;

}

#if 0

static void

connect_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *block)

{

block->info.next_block = theap->marked_blocks;

theap->marked_blocks = block;

}

#endif

static void

append_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *append_blocks)

{

if (theap->marked_blocks) {

struct transient_heap_block *block = theap->marked_blocks, *last_block = NULL;

while (block) {

last_block = block;

block = block->info.next_block;

}

TH_ASSERT(last_block->info.next_block == NULL);

last_block->info.next_block = append_blocks;

}

else {

theap->marked_blocks = append_blocks;

}

}

static struct transient_heap_block *

transient_heap_block_alloc(struct transient_heap* theap)

{

struct transient_heap_block *block;

block = mmap(NULL, TRANSIENT_HEAP_BLOCK_SIZE, PROT_READ | PROT_WRITE,

MAP_PRIVATE | MAP_ANONYMOUS,

-1, 0);

if (block == MAP_FAILED) rb_bug("transient_heap_block_alloc: err:%d\n", errno);

reset_block(block);

TH_ASSERT(((intptr_t)block->buff & (TRANSIENT_HEAP_ALLOC_ALIGN-1)) == 0);

theap->total_blocks++;

// fprintf(stderr, "transient_heap_block_alloc: %d\n", theap->total_blocks);

return block;

}

static struct transient_heap_block *

transient_heap_allocatable_block(struct transient_heap* theap)

{

struct transient_heap_block *block;

#if TRANSIENT_HEAP_INFINITE_BLOCK_MODE

block = transient_heap_block_alloc(theap);

#else

// get one block from free_blocks

block = theap->free_blocks;

if (block) {

theap->free_blocks = block->info.next_block;

block->info.next_block = NULL;

}

#endif

return block;

}

static struct transient_alloc_header *

transient_heap_allocatable_header(struct transient_heap* theap, size_t size)

{

struct transient_heap_block *block = theap->using_blocks;

while (block) {

TH_ASSERT(block->info.size >= block->info.index);

if (block->info.size - block->info.index >= (int32_t)size) {

struct transient_alloc_header *header = (void *)&block->buff[block->info.index];

block->info.index += size;

block->info.objects++;

return header;

}

else {

block = transient_heap_allocatable_block(theap);

if (block) connect_to_using_blocks(theap, block);

}

}

return NULL;

}

void *

rb_transient_heap_alloc(VALUE obj, size_t req_size)

{

struct transient_heap* theap = transient_heap_get();

size_t size = ROUND_UP(req_size + sizeof(struct transient_alloc_header), TRANSIENT_HEAP_ALLOC_ALIGN);

if (size > TRANSIENT_HEAP_ALLOC_MAX) {

// fprintf(stderr, "rb_transient_heap_alloc: NULL (too big: %ld)\n", (long)size);

return NULL;

}

else if (RB_OBJ_PROMOTED_RAW(obj)) {

// fprintf(stderr, "rb_transient_heap_alloc: NULL (not for promoted objects)\n");

return NULL;

}

else {

struct transient_alloc_header *header = transient_heap_allocatable_header(theap, size);

if (header) {

void *ptr;

header->size = size;

header->magic = TRANSIENT_HEAP_ALLOC_MAGIC;

header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;

header->obj = obj; // TODO: for verify

// stat info

theap->total_objects++;

ptr = header + 1;

// fprintf(stderr, "rb_transient_heap_alloc: header:%p ptr:%p size:%d obj:%s\n", header, ptr, (int)size, rb_obj_info(obj));

return ptr;

}

else {

// fprintf(stderr, "rb_transient_heap_alloc: NULL (no enough space: %ld)\n", (long)size);

return NULL;

}

}

}

void

Init_TransientHeap(void)

{

int i, block_num;

struct transient_heap* theap = transient_heap_get();

#if TRANSIENT_HEAP_INFINITE_BLOCK_MODE

block_num = 1;

#else

TH_ASSERT(TRANSIENT_HEAP_BLOCK_SIZE * TRANSIENT_HEAP_BLOCK_NUM == TRANSIENT_HEAP_TOTAL_SIZE);

block_num = TRANSIENT_HEAP_BLOCK_NUM;

#endif

for (i=0; i<block_num-1; i++) {

connect_to_free_blocks(theap, transient_heap_block_alloc(theap));

}

theap->using_blocks = transient_heap_block_alloc(theap);

theap->promoted_objects_size = TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE;

theap->promoted_objects_index = 0;

/* should not use ALLOC_N to be free from GC */

theap->promoted_objects = malloc(sizeof(VALUE) * theap->promoted_objects_size);

if (theap->promoted_objects == NULL) rb_bug("Init_TransientHeap: malloc failed.");

}

static struct transient_heap_block *

blocks_alloc_header_to_block(struct transient_heap *theap, struct transient_heap_block *blocks, struct transient_alloc_header *header)

{

struct transient_heap_block *block = blocks;

TH_ASSERT(theap->status == transient_heap_marking);

while (block) {

if (block->buff <= (char *)header && (char *)header < block->buff + block->info.size) {

return block;

}

block = block->info.next_block;

}

return NULL;

}

static struct transient_heap_block *

alloc_header_to_block(struct transient_heap *theap, struct transient_alloc_header *header)

{

struct transient_heap_block *block;

if ((block = blocks_alloc_header_to_block(theap, theap->marked_blocks, header)) != NULL) {

if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in marked_blocks\n");

return block;

}

else if ((block = blocks_alloc_header_to_block(theap, theap->using_blocks, header)) != NULL) {

if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in using_blocks\n");

return block;

}

else {

transient_heap_dump(theap);

rb_bug("alloc_header_to_block: not found in mark_blocks (%p)\n", header);

}

}

void

rb_transient_heap_mark(VALUE obj, const void *ptr)

{

struct transient_alloc_header *header = (void *)ptr;

header = header - 1;

if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_mark: %s (%p)\n", rb_obj_info(obj), ptr);

if (header->next_marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE) {

// already marked

return;

}

if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) {

struct transient_heap* theap = transient_heap_get();

transient_heap_dump(theap);

rb_bug("rb_transient_heap_mark: magic is broken");

}

else if (header->obj != obj) {

struct transient_heap* theap = transient_heap_get();

transient_heap_dump(theap);

rb_bug("rb_transient_heap_mark: unmatch\n");

}

else {

struct transient_heap* theap = transient_heap_get();

struct transient_heap_block *block = alloc_header_to_block(theap, header);

header->next_marked_index = block->info.last_marked_index;

block->info.last_marked_index = (int)((char *)header - block->buff);

theap->total_marked_objects++;

}

}

static void *

transient_heap_ptr(VALUE obj, int error)

{

void *ptr;

switch (BUILTIN_TYPE(obj)) {

case T_ARRAY:

if (ARY_TRANSIENT_P(obj)) {

ptr = (VALUE *)RARRAY_CONST_PTR(obj);

}

else {

ptr = NULL;

}

break;

default:

if (error) {

rb_bug("transient_heap_ptr: unknown obj %s\n", rb_obj_info(obj));

}

else {

ptr = NULL;

}

}

return ptr;

}

void

rb_transient_heap_promote(VALUE obj)

{

if (transient_heap_ptr(obj, FALSE)) {

struct transient_heap* theap = transient_heap_get();

if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_promote: %s\n", rb_obj_info(obj));

if (theap->promoted_objects_size <= theap->promoted_objects_index) {

theap->promoted_objects_size *= 2;

if (TRANSIENT_HEAP_DEBUG >= 0) fprintf(stderr, "rb_transient_heap_promote: expand table to %d\n", theap->promoted_objects_size);

theap->promoted_objects = realloc(theap->promoted_objects, theap->promoted_objects_size * sizeof(VALUE));

if (theap->promoted_objects == NULL) rb_bug("rb_transient_heap_promote: realloc failed");

}

theap->promoted_objects[theap->promoted_objects_index++] = obj;

}

else {

/* ignore */

}

}

void

rb_transient_heap_promoted(VALUE obj, const void *ptr)

{

struct transient_alloc_header *header = (void *)ptr;

header = header - 1;

}

static struct transient_alloc_header *

alloc_header(struct transient_heap_block* block, int index)

{

return (void *)&block->buff[index];

}

void rb_ary_detransient(VALUE ary);

static void

transient_heap_reset(void)

{

struct transient_heap* theap = transient_heap_get();

struct transient_heap_block* block;

if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset\n");

block = theap->marked_blocks;

while (block) {

struct transient_heap_block *next_block = block->info.next_block;

theap->total_objects -= block->info.objects;

#if TRANSIENT_HEAP_INFINITE_BLOCK_MODE

// debug mode

if (madvise(block, TRANSIENT_HEAP_BLOCK_SIZE, MADV_DONTNEED) != 0) {

rb_bug("madvise err:%d", errno);

}

if (mprotect(block, TRANSIENT_HEAP_BLOCK_SIZE, PROT_NONE) != 0) {

rb_bug("mprotect err:%d", errno);

}

theap->total_blocks--;

#else

reset_block(block);

connect_to_free_blocks(theap, block);

#endif

block = next_block;

}

theap->marked_blocks = NULL;

theap->total_marked_objects = 0;

}

static void

transient_heap_block_escape(struct transient_heap* theap, struct transient_heap_block* block)

{

int marked_index = block->info.last_marked_index;

block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;

while (marked_index >= 0) {

struct transient_alloc_header *header = alloc_header(block, marked_index);

VALUE obj = header->obj;

if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, " * transient_heap_block_escape %p %s\n", header, rb_obj_info(obj));

if (obj != Qnil) {

switch (BUILTIN_TYPE(obj)) {

case T_ARRAY:

rb_ary_detransient(obj);

break;

default:

rb_bug("unsupporeted");

}

header->obj = Qundef; // to verify

}

marked_index = header->next_marked_index;

}

}

static void

transient_heap_update_status(struct transient_heap* theap, enum transient_heap_status status)

{

TH_ASSERT(theap->status != status);

theap->status = status;

}

static void

transient_heap_escape(void *dmy)

{

struct transient_heap* theap = transient_heap_get();

if (theap->status == transient_heap_marking) {

if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_escape: skip while transient_heap_marking\n");

}

else {

VALUE gc_disabled = rb_gc_disable();

struct transient_heap_block* block;

if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_escape start total_blocks:%d\n", theap->total_blocks);

if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);

TH_ASSERT(theap->status == transient_heap_none);

transient_heap_update_status(theap, transient_heap_escaping);

// escape marked blocks

block = theap->marked_blocks;

while (block) {

transient_heap_block_escape(theap, block);

block = block->info.next_block;

}

// escape using blocks

// only affect incremental marking

block = theap->using_blocks;

while (block) {

transient_heap_block_escape(theap, block);

block = block->info.next_block;

}

// all objects in marked_objects are escaped.

transient_heap_reset();

if (TRANSIENT_HEAP_DEBUG > 0) {

fprintf(stderr, "!! transient_heap_escape end total_blocks:%d\n", theap->total_blocks);

// transient_heap_dump(theap);

}

transient_heap_verify(theap);

transient_heap_update_status(theap, transient_heap_none);

if (gc_disabled != Qtrue) rb_gc_enable();

}

}

static void

clear_marked_index(struct transient_heap_block* block)

{

int marked_index = block->info.last_marked_index;

while (marked_index != TRANSIENT_HEAP_ALLOC_MARKING_LAST) {

struct transient_alloc_header *header = alloc_header(block, marked_index);

TH_ASSERT(marked_index != TRANSIENT_HEAP_ALLOC_MARKING_FREE);

// fprintf(stderr, "clear_marked_index - block:%p mark_index:%d\n", block, marked_index);

marked_index = header->next_marked_index;

header->next_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_FREE;

}

block->info.last_marked_index = TRANSIENT_HEAP_ALLOC_MARKING_LAST;

}

void

rb_transient_heap_start_marking(int full_marking)

{

struct transient_heap* theap = transient_heap_get();

struct transient_heap_block* block;

if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_start_marking objects:%d blocks:%d full_marking:%d\n",

theap->total_objects, theap->total_blocks, full_marking);

if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);

// clear marking info

block = theap->marked_blocks;

while (block) {

clear_marked_index(block);

block = block->info.next_block;

}

block = theap->using_blocks;

while (block) {

clear_marked_index(block);

block = block->info.next_block;

}

if (theap->using_blocks) {

if (theap->using_blocks->info.objects > 0) {

append_to_marked_blocks(theap, theap->using_blocks);

theap->using_blocks = NULL;

}

else {

append_to_marked_blocks(theap, theap->using_blocks->info.next_block);

theap->using_blocks->info.next_block = NULL;

}

}

if (theap->using_blocks == NULL) {

theap->using_blocks = transient_heap_allocatable_block(theap);

}

TH_ASSERT(theap->status == transient_heap_none);

transient_heap_update_status(theap, transient_heap_marking);

theap->total_marked_objects = 0;

if (full_marking) {

theap->promoted_objects_index = 0;

}

else { /* mark promoted objects */

int i;

for (i=0; i<theap->promoted_objects_index; i++) {

VALUE obj = theap->promoted_objects[i];

void *ptr = transient_heap_ptr(obj, TRUE);

if (ptr) {

rb_transient_heap_mark(obj, ptr);

}

}

}

transient_heap_verify(theap);

}

void

rb_transient_heap_finish_marking(void)

{

struct transient_heap* theap = transient_heap_get();

if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_finish_marking objects:%d, marked:%d\n",

theap->total_objects,

theap->total_marked_objects);

if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);

TH_ASSERT(theap->total_objects >= theap->total_marked_objects);

TH_ASSERT(theap->status == transient_heap_marking);

transient_heap_update_status(theap, transient_heap_none);

if (theap->total_marked_objects > 0) {

if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "-> rb_transient_heap_finish_marking register escape func.\n");

rb_postponed_job_register_one(0, transient_heap_escape, NULL);

}

else {

transient_heap_reset();

}

transient_heap_verify(theap);

}