Project

General

Profile

direct_marking.diff

direct marking, removing mark array - tenderlovemaking (Aaron Patterson), 02/06/2018 12:32 AM

View differences:

compile.c
562 562
#define APPEND_ELEM(anchor, before, elem) APPEND_ELEM(iseq, (anchor), (before), (elem))
563 563
#endif
564 564

  
565
static int
566
iseq_add_mark_object(const rb_iseq_t *iseq, VALUE v)
567
{
568
    if (!SPECIAL_CONST_P(v)) {
569
	rb_iseq_add_mark_object(iseq, v);
570
    }
571
    return COMPILE_OK;
572
}
573

  
574 565
static int
575 566
iseq_add_mark_object_compile_time(const rb_iseq_t *iseq, VALUE v)
576 567
{
......
749 740
	encoded[i] = (VALUE)table[insn];
750 741
	i += len;
751 742
    }
743
    FL_SET(iseq, ISEQ_TRANSLATED);
752 744
#endif
753 745
    return COMPILE_OK;
754 746
}
......
1235 1227
				    rb_iseq_path(iseq), rb_iseq_realpath(iseq),
1236 1228
				    INT2FIX(line_no), parent, type, ISEQ_COMPILE_DATA(iseq)->option);
1237 1229
    debugs("[new_child_iseq]< ---------------------------------------\n");
1238
    iseq_add_mark_object(iseq, (VALUE)ret_iseq);
1230
    iseq_add_mark_object_compile_time(iseq, (VALUE)ret_iseq);
1239 1231
    return ret_iseq;
1240 1232
}
1241 1233

  
......
1250 1242
				 rb_iseq_path(iseq), rb_iseq_realpath(iseq),
1251 1243
				 INT2FIX(line_no), parent, type, ISEQ_COMPILE_DATA(iseq)->option);
1252 1244
    debugs("[new_child_iseq_ifunc]< ---------------------------------------\n");
1253
    iseq_add_mark_object(iseq, (VALUE)ret_iseq);
1245
    iseq_add_mark_object_compile_time(iseq, (VALUE)ret_iseq);
1254 1246
    return ret_iseq;
1255 1247
}
1256 1248

  
......
1501 1493
	    switch (nd_type(val_node)) {
1502 1494
	      case NODE_LIT:
1503 1495
		dv = val_node->nd_lit;
1504
		iseq_add_mark_object(iseq, dv);
1505 1496
		break;
1506 1497
	      case NODE_NIL:
1507 1498
		dv = Qnil;
......
2027 2018
			    rb_hash_rehash(map);
2028 2019
			    freeze_hide_obj(map);
2029 2020
			    generated_iseq[code_index + 1 + j] = map;
2021
			    FL_SET(iseq, ISEQ_MARKABLE_ISEQ);
2030 2022
			    break;
2031 2023
			}
2032 2024
		      case TS_LINDEX:
......
2037 2029
			{
2038 2030
			    VALUE v = operands[j];
2039 2031
			    generated_iseq[code_index + 1 + j] = v;
2032
			    if (!SPECIAL_CONST_P(v)) {
2033
				FL_SET(iseq, ISEQ_MARKABLE_ISEQ);
2034
			    }
2040 2035
			    break;
2041 2036
			}
2042 2037
		      case TS_VALUE:	/* VALUE */
......
2044 2039
			    VALUE v = operands[j];
2045 2040
			    generated_iseq[code_index + 1 + j] = v;
2046 2041
			    /* to mark ruby object */
2047
			    iseq_add_mark_object(iseq, v);
2042
			    if (!SPECIAL_CONST_P(v)) {
2043
				FL_SET(iseq, ISEQ_MARKABLE_ISEQ);
2044
			    }
2048 2045
			    break;
2049 2046
			}
2050 2047
		      case TS_IC: /* inline cache */
......
2057 2054
			    generated_iseq[code_index + 1 + j] = (VALUE)ic;
2058 2055
			    break;
2059 2056
			}
2057
		      case TS_ISE: /* inline storage entry */
2058
			{
2059
			    unsigned int ic_index = FIX2UINT(operands[j]);
2060
			    IC ic = (IC)&iseq->body->is_entries[ic_index];
2061
			    if (UNLIKELY(ic_index >= iseq->body->is_size)) {
2062
				rb_bug("iseq_set_sequence: ic_index overflow: index: %d, size: %d", ic_index, iseq->body->is_size);
2063
			    }
2064
			    generated_iseq[code_index + 1 + j] = (VALUE)ic;
2065
			    FL_SET(iseq, ISEQ_MARKABLE_ISEQ);
2066
			    break;
2067
			}
2060 2068
		      case TS_CALLINFO: /* call info */
2061 2069
			{
2062 2070
			    struct rb_call_info *base_ci = (struct rb_call_info *)operands[j];
......
2209 2217
	    entry->end = label_get_position((LABEL *)(ptr[2] & ~1));
2210 2218
	    entry->iseq = (rb_iseq_t *)ptr[3];
2211 2219

  
2212
	    /* register iseq as mark object */
2213
	    if (entry->iseq != 0) {
2214
		iseq_add_mark_object(iseq, (VALUE)entry->iseq);
2215
	    }
2216

  
2217 2220
	    /* stack depth */
2218 2221
	    if (ptr[4]) {
2219 2222
		LABEL *lobj = (LABEL *)(ptr[4] & ~1);
......
4808 4811
    }
4809 4812

  
4810 4813
    if (only_special_literals) {
4811
	iseq_add_mark_object(iseq, literals);
4814
	iseq_add_mark_object_compile_time(iseq, literals);
4812 4815

  
4813 4816
	ADD_INSN(ret, nd_line(orig_node), dup);
4814 4817
	ADD_INSN2(ret, nd_line(orig_node), opt_case_dispatch, literals, elselabel);
......
7285 7288
		    break;
7286 7289
		}
7287 7290
	      case TS_IC:	/* inline cache */
7291
	      case TS_ISE:	/* inline storage entry */
7288 7292
		rb_str_catf(str, "<ic:%d>", FIX2INT(OPERAND_AT(iobj, j)));
7289 7293
		break;
7290 7294
	      case TS_CALLINFO: /* call info */
......
7524 7528
    }
7525 7529

  
7526 7530
    loaded_iseq = rb_iseqw_to_iseq(iseqw);
7527
    iseq_add_mark_object(iseq, (VALUE)loaded_iseq);
7528 7531
    return loaded_iseq;
7529 7532
}
7530 7533

  
......
7655 7658
			break;
7656 7659
		      case TS_VALUE:
7657 7660
			argv[j] = op;
7658
			iseq_add_mark_object(iseq, op);
7659 7661
			break;
7660 7662
		      case TS_ISEQ:
7661 7663
			{
......
7672 7674
			argv[j] = (VALUE)rb_global_entry(SYM2ID(op));
7673 7675
			break;
7674 7676
		      case TS_IC:
7677
		      case TS_ISE:
7675 7678
			argv[j] = op;
7676 7679
			if (NUM2UINT(op) >= iseq->body->is_size) {
7677 7680
			    iseq->body->is_size = NUM2INT(op) + 1;
......
7702 7705
			    }
7703 7706
			    RB_GC_GUARD(op);
7704 7707
			    argv[j] = map;
7705
			    rb_iseq_add_mark_object(iseq, map);
7706 7708
			}
7707 7709
			break;
7708 7710
		      case TS_FUNCPTR:
......
8278 8280
		code[code_index] = (VALUE)ibf_dump_iseq(dump, (const rb_iseq_t *)op);
8279 8281
		break;
8280 8282
	      case TS_IC:
8283
	      case TS_ISE:
8281 8284
		{
8282 8285
		    unsigned int i;
8283 8286
		    for (i=0; i<iseq->body->is_size; i++) {
......
8343 8346
		code[code_index] = (VALUE)ibf_load_iseq(load, (const rb_iseq_t *)op);
8344 8347
		break;
8345 8348
	      case TS_IC:
8349
	      case TS_ISE:
8346 8350
		code[code_index] = (VALUE)&is_entries[(int)op];
8347 8351
		break;
8348 8352
	      case TS_CALLINFO:
......
8643 8647
    dump_body.is_entries =           NULL;
8644 8648
    dump_body.ci_entries =           ibf_dump_ci_entries(dump, iseq);
8645 8649
    dump_body.cc_entries =           NULL;
8646
    dump_body.mark_ary =             ISEQ_FLIP_CNT(iseq);
8650
    dump_body.variable.coverage      = Qnil;
8651
    dump_body.variable.original_iseq = Qnil;
8647 8652

  
8648 8653
    return ibf_dump_write(dump, &dump_body, sizeof(dump_body));
8649 8654
}
......
8675 8680
    load_body->ci_kw_size = body->ci_kw_size;
8676 8681
    load_body->insns_info.size = body->insns_info.size;
8677 8682

  
8678
    RB_OBJ_WRITE(iseq, &load_body->mark_ary, iseq_mark_ary_create((int)body->mark_ary));
8683
    iseq_mark_ary_create(iseq, (int)body->variable.flip_count);
8679 8684

  
8680 8685
    {
8681 8686
	VALUE realpath = Qnil, path = ibf_load_object(load, body->location.pathobj);
......
9306 9311

  
9307 9312
	    rb_ary_store(load->obj_list, (long)object_index, obj);
9308 9313
	}
9309
	iseq_add_mark_object(load->iseq, obj);
9310 9314
	return obj;
9311 9315
    }
9312 9316
}
......
9493 9497
	    ibf_load_iseq_complete(iseq);
9494 9498
#endif /* !USE_LAZY_LOAD */
9495 9499

  
9496
	    if (load->iseq) {
9497
		iseq_add_mark_object(load->iseq, (VALUE)iseq);
9498
	    }
9499 9500
	    return iseq;
9500 9501
	}
9501 9502
    }
insns.def
972 972
/* run iseq only once */
973 973
DEFINE_INSN
974 974
once
975
(ISEQ iseq, IC ic)
975
(ISEQ iseq, ISE ise)
976 976
()
977 977
(VALUE val)
978 978
{
979
    val = vm_once_dispatch(ec, iseq, ic);
979
    val = vm_once_dispatch(ec, iseq, ise);
980 980
}
981 981

  
982 982
/* case dispatcher, jump by table if possible */
iseq.c
113 113
    RUBY_FREE_LEAVE("iseq");
114 114
}
115 115

  
116
#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
117
static int
118
rb_vm_insn_addr2insn2(const void *addr)
119
{
120
    int insn;
121
    const void * const *table = rb_vm_get_insns_address_table();
122

  
123
    for (insn = 0; insn < VM_INSTRUCTION_SIZE; insn++) {
124
	if (table[insn] == addr) {
125
	    return insn;
126
	}
127
    }
128
    rb_bug("rb_vm_insn_addr2insn: invalid insn address: %p", addr);
129
}
130
#endif
131

  
132
static int
133
rb_vm_insn_null_translator(const void *addr)
134
{
135
    return (int)addr;
136
}
137

  
138
typedef void iseq_value_itr_t(void *ctx, VALUE obj);
139
typedef int rb_vm_insns_translator_t(const void *addr);
140

  
141
static int
142
iseq_extract_values(const VALUE *code, size_t pos, iseq_value_itr_t * func, void *data, rb_vm_insns_translator_t * translator)
143
{
144
    VALUE insn = translator((void *)code[pos]);
145
    int len = insn_len(insn);
146
    int op_no;
147
    const char *types = insn_op_types(insn);
148

  
149
    for (op_no = 0; types[op_no]; op_no++) {
150
	char type = types[op_no];
151
	switch (type) {
152
	    case TS_CDHASH:
153
	    case TS_ISEQ:
154
	    case TS_VALUE:
155
		{
156
		    VALUE op = code[pos + op_no + 1];
157
		    if (!SPECIAL_CONST_P(op)) {
158
			func(data, op);
159
		    }
160
		    break;
161
		}
162
	    case TS_ISE:
163
		{
164
		    union iseq_inline_storage_entry *const is = (union iseq_inline_storage_entry *)code[pos + op_no + 1];
165
		    if (is->once.value) {
166
			func(data, is->once.value);
167
		    }
168
		    break;
169
		}
170
	    default:
171
		break;
172
	}
173
    }
174

  
175
    return len;
176
}
177

  
178
static void
179
rb_iseq_each_value(const rb_iseq_t *iseq, iseq_value_itr_t * func, void *data)
180
{
181
    unsigned int size;
182
    const VALUE *code;
183
    size_t n;
184
    rb_vm_insns_translator_t * translator;
185

  
186
    size = iseq->body->iseq_size;
187
    code = iseq->body->iseq_encoded;
188

  
189
#if OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE
190
    if (FL_TEST(iseq, ISEQ_TRANSLATED)) {
191
	translator = rb_vm_insn_addr2insn2;
192
    } else {
193
	translator = rb_vm_insn_null_translator;
194
    }
195
#else
196
    translator = rb_vm_insn_null_translator;
197
#endif
198

  
199
    for (n = 0; n < size;) {
200
	n += iseq_extract_values(code, n, func, data, translator);
201
    }
202
}
203

  
204
static void
205
each_insn_value(void *ctx, VALUE obj)
206
{
207
    rb_gc_mark(obj);
208
}
209

  
116 210
void
117 211
rb_iseq_mark(const rb_iseq_t *iseq)
118 212
{
......
121 215
    if (iseq->body) {
122 216
	const struct rb_iseq_constant_body *body = iseq->body;
123 217

  
124
	RUBY_MARK_UNLESS_NULL(body->mark_ary);
218
	if(FL_TEST(iseq, ISEQ_MARKABLE_ISEQ)) {
219
	    rb_iseq_each_value(iseq, each_insn_value, NULL);
220
	}
221

  
222
	rb_gc_mark(body->variable.coverage);
223
	rb_gc_mark(body->variable.original_iseq);
125 224
	rb_gc_mark(body->location.label);
126 225
	rb_gc_mark(body->location.base_label);
127 226
	rb_gc_mark(body->location.pathobj);
128 227
	RUBY_MARK_UNLESS_NULL((VALUE)body->parent_iseq);
228

  
229
	if (body->catch_table) {
230
	    const struct iseq_catch_table *table = body->catch_table;
231
	    unsigned int i;
232
	    for(i = 0; i < table->size; i++) {
233
		const struct iseq_catch_table_entry *entry;
234
		entry = &table->entries[i];
235
		if (entry->iseq) {
236
		    rb_gc_mark((VALUE)entry->iseq);
237
		}
238
	    }
239
	}
129 240
    }
130 241

  
242

  
131 243
    if (FL_TEST(iseq, ISEQ_NOT_LOADED_YET)) {
132 244
	rb_gc_mark(iseq->aux.loader.obj);
133 245
    }
......
297 409
    }
298 410
}
299 411

  
300
void
301
rb_iseq_add_mark_object(const rb_iseq_t *iseq, VALUE obj)
302
{
303
    /* TODO: check dedup */
304
    rb_ary_push(ISEQ_MARK_ARY(iseq), obj);
305
}
306

  
307 412
static VALUE
308 413
prepare_iseq_build(rb_iseq_t *iseq,
309 414
		   VALUE name, VALUE path, VALUE realpath, VALUE first_lineno, const rb_code_location_t *code_location,
......
324 429
    if (iseq != iseq->body->local_iseq) {
325 430
	RB_OBJ_WRITE(iseq, &iseq->body->location.base_label, iseq->body->local_iseq->body->location.label);
326 431
    }
327
    RB_OBJ_WRITE(iseq, &iseq->body->mark_ary, iseq_mark_ary_create(0));
432
    iseq_mark_ary_create(iseq, 0);
328 433

  
329 434
    ISEQ_COMPILE_DATA_ALLOC(iseq);
330 435
    RB_OBJ_WRITE(iseq, &ISEQ_COMPILE_DATA(iseq)->err_info, err_info);
......
1610 1715
	break;
1611 1716

  
1612 1717
      case TS_IC:
1718
      case TS_ISE:
1613 1719
	ret = rb_sprintf("<is:%"PRIdPTRDIFF">", (union iseq_inline_storage_entry *)op - iseq->body->is_entries);
1614 1720
	break;
1615 1721

  
......
2353 2459
		}
2354 2460
		break;
2355 2461
	      case TS_IC:
2462
	      case TS_ISE:
2356 2463
		{
2357 2464
		    union iseq_inline_storage_entry *is = (union iseq_inline_storage_entry *)*seq;
2358 2465
		    rb_ary_push(ary, INT2FIX(is - iseq->body->is_entries));
iseq.h
28 28
    return sizeof(struct rb_call_info_kw_arg) + sizeof(VALUE) * (keyword_len - 1);
29 29
}
30 30

  
31
enum iseq_mark_ary_index {
32
    ISEQ_MARK_ARY_COVERAGE,
33
    ISEQ_MARK_ARY_FLIP_CNT,
34
    ISEQ_MARK_ARY_ORIGINAL_ISEQ,
35
    ISEQ_MARK_ARY_INITIAL_SIZE
36
};
37

  
38
static inline VALUE
39
iseq_mark_ary_create(int flip_cnt)
31
static inline void
32
iseq_mark_ary_create(rb_iseq_t *iseq, int flip_cnt)
40 33
{
41
    VALUE ary = rb_ary_tmp_new(ISEQ_MARK_ARY_INITIAL_SIZE);
42
    rb_ary_push(ary, Qnil);              /* ISEQ_MARK_ARY_COVERAGE */
43
    rb_ary_push(ary, INT2FIX(flip_cnt)); /* ISEQ_MARK_ARY_FLIP_CNT */
44
    rb_ary_push(ary, Qnil);              /* ISEQ_MARK_ARY_ORIGINAL_ISEQ */
45
    return ary;
34
    RB_OBJ_WRITE(iseq, &iseq->body->variable.coverage, Qnil);
35
    RB_OBJ_WRITE(iseq, &iseq->body->variable.original_iseq, Qnil);
36
    iseq->body->variable.flip_count = flip_cnt;
46 37
}
47 38

  
48
#define ISEQ_MARK_ARY(iseq)           (iseq)->body->mark_ary
49

  
50
#define ISEQ_COVERAGE(iseq)           RARRAY_AREF(ISEQ_MARK_ARY(iseq), ISEQ_MARK_ARY_COVERAGE)
51
#define ISEQ_COVERAGE_SET(iseq, cov)  RARRAY_ASET(ISEQ_MARK_ARY(iseq), ISEQ_MARK_ARY_COVERAGE, cov)
39
#define ISEQ_COVERAGE(iseq)           iseq->body->variable.coverage
40
#define ISEQ_COVERAGE_SET(iseq, cov)  RB_OBJ_WRITE(iseq, &iseq->body->variable.coverage, cov)
52 41
#define ISEQ_LINE_COVERAGE(iseq)      RARRAY_AREF(ISEQ_COVERAGE(iseq), COVERAGE_INDEX_LINES)
53 42
#define ISEQ_BRANCH_COVERAGE(iseq)    RARRAY_AREF(ISEQ_COVERAGE(iseq), COVERAGE_INDEX_BRANCHES)
54 43

  
55
#define ISEQ_FLIP_CNT(iseq) FIX2INT(RARRAY_AREF(ISEQ_MARK_ARY(iseq), ISEQ_MARK_ARY_FLIP_CNT))
44
#define ISEQ_FLIP_CNT(iseq) (iseq)->body->variable.flip_count
56 45

  
57 46
static inline int
58 47
ISEQ_FLIP_CNT_INCREMENT(const rb_iseq_t *iseq)
59 48
{
60
    int cnt = ISEQ_FLIP_CNT(iseq);
61
    RARRAY_ASET(ISEQ_MARK_ARY(iseq), ISEQ_MARK_ARY_FLIP_CNT, INT2FIX(cnt+1));
49
    int cnt = iseq->body->variable.flip_count;
50
    iseq->body->variable.flip_count += 1;
62 51
    return cnt;
63 52
}
64 53

  
65 54
static inline VALUE *
66 55
ISEQ_ORIGINAL_ISEQ(const rb_iseq_t *iseq)
67 56
{
68
    VALUE str = RARRAY_AREF(ISEQ_MARK_ARY(iseq), ISEQ_MARK_ARY_ORIGINAL_ISEQ);
57
    VALUE str = iseq->body->variable.original_iseq;
69 58
    if (RTEST(str)) return (VALUE *)RSTRING_PTR(str);
70 59
    return NULL;
71 60
}
......
73 62
static inline void
74 63
ISEQ_ORIGINAL_ISEQ_CLEAR(const rb_iseq_t *iseq)
75 64
{
76
    RARRAY_ASET(ISEQ_MARK_ARY(iseq), ISEQ_MARK_ARY_ORIGINAL_ISEQ, Qnil);
65
    RB_OBJ_WRITE(iseq, &iseq->body->variable.original_iseq, Qnil);
77 66
}
78 67

  
79 68
static inline VALUE *
80 69
ISEQ_ORIGINAL_ISEQ_ALLOC(const rb_iseq_t *iseq, long size)
81 70
{
82 71
    VALUE str = rb_str_tmp_new(size * sizeof(VALUE));
83
    RARRAY_ASET(ISEQ_MARK_ARY(iseq), ISEQ_MARK_ARY_ORIGINAL_ISEQ, str);
72
    RB_OBJ_WRITE(iseq, &iseq->body->variable.original_iseq, str);
84 73
    return (VALUE *)RSTRING_PTR(str);
85 74
}
86 75

  
......
94 83

  
95 84
#define ISEQ_NOT_LOADED_YET   IMEMO_FL_USER1
96 85
#define ISEQ_USE_COMPILE_DATA IMEMO_FL_USER2
86
#define ISEQ_TRANSLATED       IMEMO_FL_USER3
87
#define ISEQ_MARKABLE_ISEQ    IMEMO_FL_USER4
97 88

  
98 89
struct iseq_compile_data {
99 90
    /* GC is needed */
......
173 164
			    VALUE exception, VALUE body);
174 165

  
175 166
/* iseq.c */
176
void rb_iseq_add_mark_object(const rb_iseq_t *iseq, VALUE obj);
177 167
VALUE rb_iseq_load(VALUE data, VALUE parent, VALUE opt);
178 168
VALUE rb_iseq_parameters(const rb_iseq_t *iseq, int is_proc);
179 169
struct st_table *ruby_insn_make_insn_table(void);
tool/ruby_vm/models/typemap.rb
18 18
  "GENTRY"         => %w[G TS_GENTRY],
19 19
  "IC"             => %w[K TS_IC],
20 20
  "ID"             => %w[I TS_ID],
21
  "ISE"            => %w[T TS_ISE],
21 22
  "ISEQ"           => %w[S TS_ISEQ],
22 23
  "OFFSET"         => %w[O TS_OFFSET],
23 24
  "VALUE"          => %w[V TS_VALUE],
vm_core.h
407 407
				      */
408 408
    struct rb_call_cache *cc_entries; /* size is ci_size = ci_kw_size */
409 409

  
410
    VALUE mark_ary;     /* Array: includes operands which should be GC marked */
410
    struct {
411
      rb_num_t flip_count;
412
      VALUE coverage;
413
      VALUE original_iseq;
414
    } variable;
411 415

  
412 416
    unsigned int local_table_size;
413 417
    unsigned int is_size;
......
998 1002

  
999 1003
/* inline cache */
1000 1004
typedef struct iseq_inline_cache_entry *IC;
1005
typedef union iseq_inline_storage_entry *ISE;
1001 1006
typedef struct rb_call_info *CALL_INFO;
1002 1007
typedef struct rb_call_cache *CALL_CACHE;
1003 1008

  
vm_insnhelper.c
3281 3281
}
3282 3282

  
3283 3283
static VALUE
3284
vm_once_dispatch(rb_execution_context_t *ec, ISEQ iseq, IC ic)
3284
vm_once_dispatch(rb_execution_context_t *ec, ISEQ iseq, ISE is)
3285 3285
{
3286 3286
    rb_thread_t *th = rb_ec_thread_ptr(ec);
3287 3287
    rb_thread_t *const RUNNING_THREAD_ONCE_DONE = (rb_thread_t *)(0x1);
3288
    union iseq_inline_storage_entry *const is = (union iseq_inline_storage_entry *)ic;
3289 3288

  
3290 3289
  again:
3291 3290
    if (is->once.running_thread == RUNNING_THREAD_ONCE_DONE) {
......
3294 3293
    else if (is->once.running_thread == NULL) {
3295 3294
	VALUE val;
3296 3295
	is->once.running_thread = th;
3297
	val = is->once.value = rb_ensure(vm_once_exec, (VALUE)iseq, vm_once_clear, (VALUE)is);
3296
	val = rb_ensure(vm_once_exec, (VALUE)iseq, vm_once_clear, (VALUE)is);
3297
	RB_OBJ_WRITE(ec->cfp->iseq, &is->once.value, val);
3298 3298
	/* is->once.running_thread is cleared by vm_once_clear() */
3299 3299
	is->once.running_thread = RUNNING_THREAD_ONCE_DONE; /* success */
3300
	rb_iseq_add_mark_object(ec->cfp->iseq, val);
3301 3300
	return val;
3302 3301
    }
3303 3302
    else if (is->once.running_thread == th) {