Project

General

Profile

Feature #10341 ยป 0005-Optimize-fiber_switch-callees.patch

nome (Knut Franke), 10/11/2014 08:57 PM

View differences:

cont.c
132 132

  
133 133
typedef struct rb_fiber_struct {
134 134
    rb_context_t cont;
135
    VALUE prev;
135
    struct rb_fiber_struct *prev;
136 136
    enum fiber_status status;
137 137
    /* If a fiber invokes "transfer",
138 138
     * then this fiber can't "resume" any more after that.
......
205 205
		rb_thread_t *th;
206 206
                rb_fiber_t *fib = (rb_fiber_t*)cont;
207 207
		GetThreadPtr(cont->saved_thread.self, th);
208
		if ((th->fiber != cont->self) && fib->status == RUNNING) {
208
		if ((th->fiber != fib) && fib->status == RUNNING) {
209 209
		    rb_gc_mark_locations(cont->machine.stack,
210 210
					 cont->machine.stack + cont->machine.stack_size);
211 211
		}
......
236 236
	}
237 237
	else {
238 238
	    /* fiber */
239
	    rb_fiber_t *fib = (rb_fiber_t*)cont;
239 240
#ifdef _WIN32
240
	    if (GET_THREAD()->fiber != cont->self && cont->type != ROOT_FIBER_CONTEXT) {
241
	    if (GET_THREAD()->fiber != fib && cont->type != ROOT_FIBER_CONTEXT) {
241 242
		/* don't delete root fiber handle */
242 243
		rb_fiber_t *fib = (rb_fiber_t*)cont;
243 244
		if (fib->fib_handle) {
......
245 246
		}
246 247
	    }
247 248
#else /* not WIN32 */
248
	    if (GET_THREAD()->fiber != cont->self) {
249
	    if (GET_THREAD()->fiber != fib) {
249 250
                rb_fiber_t *fib = (rb_fiber_t*)cont;
250 251
                if (fib->ss_sp) {
251 252
                    if (cont->type == ROOT_FIBER_CONTEXT) {
......
304 305
    return size;
305 306
}
306 307

  
308
void
309
rb_fiber_mark_self(rb_fiber_t *fib)
310
{
311
    if (fib)
312
	rb_gc_mark(fib->cont.self);
313
}
314

  
307 315
static void
308 316
fiber_mark(void *ptr)
309 317
{
310 318
    RUBY_MARK_ENTER("cont");
311 319
    if (ptr) {
312 320
	rb_fiber_t *fib = ptr;
313
	rb_gc_mark(fib->prev);
321
	rb_fiber_mark_self(fib->prev);
314 322
	cont_mark(&fib->cont);
315 323
    }
316 324
    RUBY_MARK_LEAVE("cont");
317 325
}
318 326

  
327

  
319 328
static void
320 329
fiber_free(void *ptr)
321 330
{
......
409 418
    NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY
410 419
};
411 420

  
412
static void
421
static inline void
413 422
cont_save_thread(rb_context_t *cont, rb_thread_t *th)
414 423
{
415 424
    rb_thread_t *sth = &cont->saved_thread;
......
526 535
    }
527 536
}
528 537

  
529
static void
538
static inline void
530 539
cont_restore_thread(rb_context_t *cont)
531 540
{
532 541
    rb_thread_t *th = GET_THREAD(), *sth = &cont->saved_thread;
......
534 543
    /* restore thread context */
535 544
    if (cont->type == CONTINUATION_CONTEXT) {
536 545
	/* continuation */
537
	VALUE fib;
546
	rb_fiber_t *fib;
538 547

  
539 548
	th->fiber = sth->fiber;
540 549
	fib = th->fiber ? th->fiber : th->root_fiber;
541 550

  
542 551
	if (fib) {
543
	    rb_fiber_t *fcont;
544
	    GetFiberPtr(fib, fcont);
545
	    th->stack_size = fcont->cont.saved_thread.stack_size;
546
	    th->stack = fcont->cont.saved_thread.stack;
552
	    th->stack_size = fib->cont.saved_thread.stack_size;
553
	    th->stack = fib->cont.saved_thread.stack;
547 554
	}
548 555
#ifdef CAPTURE_JUST_VALID_VM_STACK
549 556
	MEMCPY(th->stack, cont->vm_stack, VALUE, cont->vm_stack_slen);
......
558 565
	th->stack = sth->stack;
559 566
	th->stack_size = sth->stack_size;
560 567
	th->local_storage = sth->local_storage;
561
	th->fiber = cont->self;
568
	th->fiber = (rb_fiber_t*)cont;
562 569
    }
563 570

  
564 571
    th->cfp = sth->cfp;
......
719 726
    /* oldfib->machine.stack_end should be NULL */
720 727
    oldfib->cont.saved_thread.machine.stack_end = 0;
721 728
#ifndef _WIN32
722
    if (!newfib->context.uc_stack.ss_sp && th->root_fiber != newfib->cont.self) {
729
    if (!newfib->context.uc_stack.ss_sp && th->root_fiber != newfib) {
723 730
	rb_bug("non_root_fiber->context.uc_stac.ss_sp should not be NULL");
724 731
    }
725 732
#endif
......
1051 1058
	rb_raise(rb_eRuntimeError, "continuation called across stack rewinding barrier");
1052 1059
    }
1053 1060
    if (cont->saved_thread.fiber) {
1054
	rb_fiber_t *fcont;
1055
	GetFiberPtr(cont->saved_thread.fiber, fcont);
1056

  
1057 1061
	if (th->fiber != cont->saved_thread.fiber) {
1058 1062
	    rb_raise(rb_eRuntimeError, "continuation called across fiber");
1059 1063
	}
......
1163 1167
    fib->cont.self = fibval;
1164 1168
    fib->cont.type = FIBER_CONTEXT;
1165 1169
    cont_init(&fib->cont, th);
1166
    fib->prev = Qnil;
1170
    fib->prev = NULL;
1167 1171
    fib->status = CREATED;
1168 1172

  
1169 1173
    DATA_PTR(fibval) = fib;
......
1229 1233
    return fiber_init(fiber_alloc(rb_cFiber), rb_proc_new(func, obj));
1230 1234
}
1231 1235

  
1232
static VALUE
1233
return_fiber(void)
1234
{
1235
    rb_fiber_t *fib;
1236
    VALUE curr = rb_fiber_current();
1237
    VALUE prev;
1238
    GetFiberPtr(curr, fib);
1239

  
1240
    prev = fib->prev;
1241
    if (NIL_P(prev)) {
1242
	const VALUE root_fiber = GET_THREAD()->root_fiber;
1243

  
1244
	if (root_fiber == curr) {
1245
	    rb_raise(rb_eFiberError, "can't yield from root fiber");
1246
	}
1247
	return root_fiber;
1248
    }
1249
    else {
1250
	fib->prev = Qnil;
1251
	return prev;
1252
    }
1253
}
1254

  
1255
VALUE rb_fiber_transfer(VALUE fib, int argc, const VALUE *argv);
1256

  
1257
static void
1258
rb_fiber_terminate(rb_fiber_t *fib)
1259
{
1260
    VALUE value = fib->cont.value;
1261
    fib->status = TERMINATED;
1262
#if FIBER_USE_NATIVE && !defined(_WIN32)
1263
    /* Ruby must not switch to other thread until storing terminated_machine_stack */
1264
    terminated_machine_stack.ptr = fib->ss_sp;
1265
    terminated_machine_stack.size = fib->ss_size / sizeof(VALUE);
1266
    fib->ss_sp = NULL;
1267
    fib->context.uc_stack.ss_sp = NULL;
1268
    fib->cont.machine.stack = NULL;
1269
    fib->cont.machine.stack_size = 0;
1270
#endif
1271
    rb_fiber_transfer(return_fiber(), 1, &value);
1272
}
1236
static void rb_fiber_terminate(rb_fiber_t *fib);
1273 1237

  
1274 1238
void
1275 1239
rb_fiber_start(void)
1276 1240
{
1277 1241
    rb_thread_t *th = GET_THREAD();
1278
    rb_fiber_t *fib;
1242
    rb_fiber_t *fib = th->fiber;
1279 1243
    rb_proc_t *proc;
1280 1244
    int state;
1281 1245

  
1282
    GetFiberPtr(th->fiber, fib);
1283

  
1284 1246
    TH_PUSH_TAG(th);
1285 1247
    if ((state = EXEC_TAG()) == 0) {
1286 1248
	rb_context_t *cont = &VAR_FROM_MEMORY(fib)->cont;
......
1331 1293
    return fib;
1332 1294
}
1333 1295

  
1334
VALUE
1335
rb_fiber_current(void)
1296
static inline rb_fiber_t*
1297
fiber_current(void)
1336 1298
{
1337 1299
    rb_thread_t *th = GET_THREAD();
1338 1300
    if (th->fiber == 0) {
1339 1301
	/* save root */
1340 1302
	rb_fiber_t *fib = root_fiber_alloc(th);
1341
	th->root_fiber = th->fiber = fib->cont.self;
1303
	th->root_fiber = th->fiber = fib;
1342 1304
    }
1343 1305
    return th->fiber;
1344 1306
}
1345 1307

  
1346
static VALUE
1347
fiber_store(rb_fiber_t *next_fib)
1308
static inline rb_fiber_t*
1309
return_fiber(void)
1310
{
1311
    rb_fiber_t *fib = fiber_current();
1312
    rb_fiber_t *prev = fib->prev;
1313

  
1314
    if (!prev) {
1315
	rb_fiber_t *root_fiber = GET_THREAD()->root_fiber;
1316

  
1317
	if (root_fiber == fib) {
1318
	    rb_raise(rb_eFiberError, "can't yield from root fiber");
1319
	}
1320
	return root_fiber;
1321
    }
1322
    else {
1323
	fib->prev = NULL;
1324
	return prev;
1325
    }
1326
}
1327

  
1328
VALUE
1329
rb_fiber_current(void)
1330
{
1331
    return fiber_current()->cont.self;
1332
}
1333

  
1334
static inline VALUE
1335
fiber_store(rb_fiber_t *next_fib, rb_thread_t *th)
1348 1336
{
1349
    rb_thread_t *th = GET_THREAD();
1350 1337
    rb_fiber_t *fib;
1351 1338

  
1352 1339
    if (th->fiber) {
1353
	GetFiberPtr(th->fiber, fib);
1340
	fib = th->fiber;
1354 1341
	cont_save_thread(&fib->cont, th);
1355 1342
    }
1356 1343
    else {
1357 1344
	/* create current fiber */
1358 1345
	fib = root_fiber_alloc(th);
1359
	th->root_fiber = th->fiber = fib->cont.self;
1346
	th->root_fiber = th->fiber = fib;
1360 1347
    }
1361 1348

  
1362 1349
#if FIBER_USE_NATIVE
......
1380 1367
	terminated_machine_stack.ptr = NULL;
1381 1368
	terminated_machine_stack.size = 0;
1382 1369
    }
1383
    GetFiberPtr(th->fiber, fib);
1370
    fib = th->fiber;
1384 1371
    if (fib->cont.argc == -1) rb_exc_raise(fib->cont.value);
1385 1372
    return fib->cont.value;
1386 1373
#endif /* not _WIN32 */
......
1389 1376
    cont_save_machine_stack(th, &fib->cont);
1390 1377
    if (ruby_setjmp(fib->cont.jmpbuf)) {
1391 1378
	/* restored */
1392
	GetFiberPtr(th->fiber, fib);
1379
	fib = th->fiber;
1393 1380
	if (fib->cont.argc == -1) rb_exc_raise(fib->cont.value);
1394 1381
	if (nextfib->cont.value == Qundef) {
1395 1382
	    cont_restore_0(nextfib->cont, &nextfib->cont.value);
......
1406 1393
}
1407 1394

  
1408 1395
static inline VALUE
1409
fiber_switch(VALUE fibval, int argc, const VALUE *argv, int is_resume)
1396
fiber_switch(rb_fiber_t *fib, int argc, const VALUE *argv, int is_resume)
1410 1397
{
1411 1398
    VALUE value;
1412
    rb_fiber_t *fib;
1413
    rb_context_t *cont;
1399
    rb_context_t *cont = &fib->cont;
1414 1400
    rb_thread_t *th = GET_THREAD();
1415 1401

  
1416
    GetFiberPtr(fibval, fib);
1417
    cont = &fib->cont;
1418

  
1419
    if (th->fiber == fibval) {
1402
    if (th->fiber == fib) {
1420 1403
	/* ignore fiber context switch
1421 1404
         * because destination fiber is same as current fiber
1422 1405
	 */
......
1432 1415
    else if (fib->status == TERMINATED) {
1433 1416
	value = rb_exc_new2(rb_eFiberError, "dead fiber called");
1434 1417

  
1435
	GetFiberPtr(th->fiber, fib);
1436
	if (fib->status != TERMINATED) rb_exc_raise(value);
1418
	if (th->fiber->status != TERMINATED) rb_exc_raise(value);
1437 1419

  
1438 1420
	/* th->fiber is also dead => switch to root fiber */
1439 1421
	/* (this means we're being called from rb_fiber_terminate, */
1440 1422
	/* and the terminated fiber's return_fiber() is already dead) */
1441
	GetFiberPtr(th->root_fiber, fib);
1442
	cont = &fib->cont;
1423
	cont = &th->root_fiber->cont;
1443 1424
	cont->argc = -1;
1444 1425
	cont->value = value;
1445 1426
#if FIBER_USE_NATIVE
1446
	{
1447
	    VALUE oldfibval = th->fiber;
1448
	    rb_fiber_t *oldfib;
1449
	    GetFiberPtr(oldfibval, oldfib);
1450
	    fiber_setcontext(fib, oldfib);
1451
	}
1427
	fiber_setcontext(th->root_fiber, th->fiber);
1452 1428
#else
1453 1429
	cont_restore_0(cont, &value);
1454 1430
#endif
......
1456 1432
    }
1457 1433

  
1458 1434
    if (is_resume) {
1459
	fib->prev = rb_fiber_current();
1435
	fib->prev = fiber_current();
1460 1436
    }
1461 1437
    else {
1462 1438
	/* restore `tracing' context. see [Feature #4347] */
......
1466 1442
    cont->argc = argc;
1467 1443
    cont->value = make_passing_arg(argc, argv);
1468 1444

  
1469
    value = fiber_store(fib);
1445
    value = fiber_store(fib, th);
1470 1446
    RUBY_VM_CHECK_INTS(th);
1471 1447

  
1472 1448
    return value;
1473 1449
}
1474 1450

  
1475 1451
VALUE
1476
rb_fiber_transfer(VALUE fib, int argc, const VALUE *argv)
1452
rb_fiber_transfer(VALUE fibval, int argc, const VALUE *argv)
1477 1453
{
1454
    rb_fiber_t *fib;
1455
    GetFiberPtr(fibval, fib);
1478 1456
    return fiber_switch(fib, argc, argv, 0);
1479 1457
}
1480 1458

  
1459
static void
1460
rb_fiber_terminate(rb_fiber_t *fib)
1461
{
1462
    VALUE value = fib->cont.value;
1463
    fib->status = TERMINATED;
1464
#if FIBER_USE_NATIVE && !defined(_WIN32)
1465
    /* Ruby must not switch to other thread until storing terminated_machine_stack */
1466
    terminated_machine_stack.ptr = fib->ss_sp;
1467
    terminated_machine_stack.size = fib->ss_size / sizeof(VALUE);
1468
    fib->ss_sp = NULL;
1469
    fib->context.uc_stack.ss_sp = NULL;
1470
    fib->cont.machine.stack = NULL;
1471
    fib->cont.machine.stack_size = 0;
1472
#endif
1473
    fiber_switch(return_fiber(), 1, &value, 0);
1474
}
1475

  
1481 1476
VALUE
1482 1477
rb_fiber_resume(VALUE fibval, int argc, const VALUE *argv)
1483 1478
{
1484 1479
    rb_fiber_t *fib;
1485 1480
    GetFiberPtr(fibval, fib);
1486 1481

  
1487
    if (fib->prev != Qnil || fib->cont.type == ROOT_FIBER_CONTEXT) {
1482
    if (fib->prev != 0 || fib->cont.type == ROOT_FIBER_CONTEXT) {
1488 1483
	rb_raise(rb_eFiberError, "double resume");
1489 1484
    }
1490 1485
    if (fib->transfered != 0) {
1491 1486
	rb_raise(rb_eFiberError, "cannot resume transferred Fiber");
1492 1487
    }
1493 1488

  
1494
    return fiber_switch(fibval, argc, argv, 1);
1489
    return fiber_switch(fib, argc, argv, 1);
1495 1490
}
1496 1491

  
1497 1492
VALUE
1498 1493
rb_fiber_yield(int argc, const VALUE *argv)
1499 1494
{
1500
    return rb_fiber_transfer(return_fiber(), argc, argv);
1495
    return fiber_switch(return_fiber(), argc, argv, 0);
1501 1496
}
1502 1497

  
1503 1498
void
1504 1499
rb_fiber_reset_root_local_storage(VALUE thval)
1505 1500
{
1506 1501
    rb_thread_t *th;
1507
    rb_fiber_t	*fib;
1508 1502

  
1509 1503
    GetThreadPtr(thval, th);
1510 1504
    if (th->root_fiber && th->root_fiber != th->fiber) {
1511
	GetFiberPtr(th->root_fiber, fib);
1512
	th->local_storage = fib->cont.saved_thread.local_storage;
1505
	th->local_storage = th->root_fiber->cont.saved_thread.local_storage;
1513 1506
    }
1514 1507
}
1515 1508

  
......
1602 1595
    rb_fiber_t *fib;
1603 1596
    GetFiberPtr(fibval, fib);
1604 1597
    fib->transfered = 1;
1605
    return rb_fiber_transfer(fibval, argc, argv);
1598
    return fiber_switch(fib, argc, argv, 0);
1606 1599
}
1607 1600

  
1608 1601
/*
vm.c
1967 1967
    ruby_xfree(stack);
1968 1968
}
1969 1969

  
1970
void rb_fiber_mark_self(rb_fiber_t *fib);
1971

  
1970 1972
void
1971 1973
rb_thread_mark(void *ptr)
1972 1974
{
......
2013 2015
	RUBY_MARK_UNLESS_NULL(th->root_svar);
2014 2016
	RUBY_MARK_UNLESS_NULL(th->top_self);
2015 2017
	RUBY_MARK_UNLESS_NULL(th->top_wrapper);
2016
	RUBY_MARK_UNLESS_NULL(th->fiber);
2017
	RUBY_MARK_UNLESS_NULL(th->root_fiber);
2018
	rb_fiber_mark_self(th->fiber);
2019
	rb_fiber_mark_self(th->root_fiber);
2018 2020
	RUBY_MARK_UNLESS_NULL(th->stat_insn_usage);
2019 2021
	RUBY_MARK_UNLESS_NULL(th->last_status);
2020 2022

  
vm_core.h
560 560

  
561 561
typedef char rb_thread_id_string_t[sizeof(rb_nativethread_id_t) * 2 + 3];
562 562

  
563
typedef struct rb_fiber_struct rb_fiber_t;
564

  
563 565
typedef struct rb_thread_struct {
564 566
    struct list_node vmlt_node;
565 567
    VALUE self;
......
681 683
    struct rb_trace_arg_struct *trace_arg; /* trace information */
682 684

  
683 685
    /* fiber */
684
    VALUE fiber;
685
    VALUE root_fiber;
686
    rb_fiber_t *fiber;
687
    rb_fiber_t *root_fiber;
686 688
    rb_jmpbuf_t root_jmpbuf;
687 689

  
688 690
    /* ensure & callcc */
689
-