diff --git a/ext/date/date_core.c b/ext/date/date_core.c index 72d697c8ea233e..0d69633ab48fd1 100644 --- a/ext/date/date_core.c +++ b/ext/date/date_core.c @@ -992,12 +992,16 @@ c_valid_commercial_p(int y, int w, int d, double sg, if (w < 0) { int rjd2; + if (w < -53) return 0; c_commercial_to_jd(y + 1, 1, 1, sg, &rjd2, &ns2); c_jd_to_commercial(rjd2 + w * 7, sg, &ry2, &rw2, &rd2); if (ry2 != y) return 0; w = rw2; } + else { + if (w < 1 || 53 < w) return 0; + } c_commercial_to_jd(y, w, d, sg, rjd, ns); c_jd_to_commercial(*rjd, sg, &ry2, rw, rd); if (y != ry2 || w != *rw || d != *rd) @@ -2621,9 +2625,9 @@ offset_to_sec(VALUE vof, int *rof) if (!FIXNUM_P(vn)) return 0; n = FIX2LONG(vn); - if (n < -DAY_IN_SECONDS || n > DAY_IN_SECONDS) - return 0; } + if (n < -DAY_IN_SECONDS || n > DAY_IN_SECONDS) + return 0; *rof = (int)n; return 1; } @@ -3195,34 +3199,66 @@ date_s_gregorian_leap_p(VALUE klass, VALUE y) return f_boolcast(c_gregorian_leap_p(ry)); } +#ifndef HAVE_RB_GC_MARK_MOVABLE +#define rb_gc_mark_movable rb_gc_mark +#else static void -d_lite_gc_mark(void *ptr) +d_lite_gc_compact(void *ptr) { union DateData *dat = ptr; if (simple_dat_p(dat)) - rb_gc_mark(dat->s.nth); + dat->s.nth = rb_gc_location(dat->s.nth); else { - rb_gc_mark(dat->c.nth); - rb_gc_mark(dat->c.sf); + dat->c.nth = rb_gc_location(dat->c.nth); + dat->c.sf = rb_gc_location(dat->c.sf); } } +#endif -static size_t -d_lite_memsize(const void *ptr) +static void +d_lite_gc_mark(void *ptr) { - const union DateData *dat = ptr; - return complex_dat_p(dat) ? sizeof(struct ComplexDateData) : sizeof(struct SimpleDateData); + union DateData *dat = ptr; + if (simple_dat_p(dat)) + rb_gc_mark_movable(dat->s.nth); + else { + rb_gc_mark_movable(dat->c.nth); + rb_gc_mark_movable(dat->c.sf); + } } #ifndef HAVE_RB_EXT_RACTOR_SAFE # define RUBY_TYPED_FROZEN_SHAREABLE 0 #endif +#if defined(RUBY_TYPED_EMBEDDABLE) || defined(HAVE_CONST_RUBY_TYPED_EMBEDDABLE) +# define HAVE_RUBY_TYPED_EMBEDDABLE 1 +#else +# define RUBY_TYPED_EMBEDDABLE 0 +#endif + +static size_t +d_lite_memsize(const void *ptr) +{ +#ifdef HAVE_RUBY_TYPED_EMBEDDABLE + return 0; +#else + return sizeof(union DateData); +#endif +} + static const rb_data_type_t d_lite_type = { "Date", - {d_lite_gc_mark, RUBY_TYPED_DEFAULT_FREE, d_lite_memsize,}, + { + d_lite_gc_mark, + RUBY_TYPED_DEFAULT_FREE, + d_lite_memsize, +#ifdef HAVE_RB_GC_MARK_MOVABLE + d_lite_gc_compact, +#endif + }, 0, 0, - RUBY_TYPED_FREE_IMMEDIATELY|RUBY_TYPED_WB_PROTECTED|RUBY_TYPED_FROZEN_SHAREABLE, + RUBY_TYPED_FREE_IMMEDIATELY|RUBY_TYPED_WB_PROTECTED|RUBY_TYPED_EMBEDDABLE|RUBY_TYPED_FROZEN_SHAREABLE, }; inline static VALUE @@ -3233,10 +3269,11 @@ d_simple_new_internal(VALUE klass, unsigned flags) { struct SimpleDateData *dat; + union DateData *u_dat; VALUE obj; - obj = TypedData_Make_Struct(klass, struct SimpleDateData, - &d_lite_type, dat); + obj = TypedData_Make_Struct(klass, union DateData, &d_lite_type, u_dat); + dat = &u_dat->s; set_to_simple(obj, dat, nth, jd, sg, y, m, d, flags); assert(have_jd_p(dat) || have_civil_p(dat)); @@ -3254,10 +3291,11 @@ d_complex_new_internal(VALUE klass, unsigned flags) { struct ComplexDateData *dat; + union DateData *u_dat; VALUE obj; - obj = TypedData_Make_Struct(klass, struct ComplexDateData, - &d_lite_type, dat); + obj = TypedData_Make_Struct(klass, union DateData, &d_lite_type, u_dat); + dat = &u_dat->c; set_to_complex(obj, dat, nth, jd, df, sf, of, sg, y, m, d, h, min, s, flags); @@ -4515,7 +4553,8 @@ d_new_by_frags(VALUE klass, VALUE hash, VALUE sg) } VALUE date__strptime(const char *str, size_t slen, - const char *fmt, size_t flen, VALUE hash); + const char *fmt, size_t flen, + VALUE hash, rb_encoding *enc); static VALUE date_s__strptime_internal(int argc, VALUE *argv, VALUE klass, @@ -4524,6 +4563,7 @@ date_s__strptime_internal(int argc, VALUE *argv, VALUE klass, VALUE vstr, vfmt, hash; const char *str, *fmt; size_t slen, flen; + rb_encoding *enc; rb_scan_args(argc, argv, "11", &vstr, &vfmt); @@ -4537,33 +4577,18 @@ date_s__strptime_internal(int argc, VALUE *argv, VALUE klass, if (argc < 2) { fmt = default_fmt; flen = strlen(default_fmt); + enc = rb_enc_get(vstr); } else { if (!rb_enc_str_asciicompat_p(vfmt)) rb_raise(rb_eArgError, "format should have ASCII compatible encoding"); + enc = rb_enc_check(vstr, vfmt); fmt = RSTRING_PTR(vfmt); flen = RSTRING_LEN(vfmt); } hash = rb_hash_new(); - if (NIL_P(date__strptime(str, slen, fmt, flen, hash))) - return Qnil; - - { - VALUE zone = ref_hash("zone"); - VALUE left = ref_hash("leftover"); - - if (!NIL_P(zone)) { - rb_enc_copy(zone, vstr); - set_hash("zone", zone); - } - if (!NIL_P(left)) { - rb_enc_copy(left, vstr); - set_hash("leftover", left); - } - } - - return hash; + return date__strptime(str, slen, fmt, flen, hash, enc); } /* @@ -7804,8 +7829,6 @@ d_lite_marshal_load(VALUE self, VALUE a) if (simple_dat_p(dat)) { if (df || !f_zero_p(sf) || of) { /* loading a fractional date; promote to complex */ - dat = ruby_xrealloc(dat, sizeof(struct ComplexDateData)); - RTYPEDDATA(self)->data = dat; goto complex_data; } set_to_simple(self, &dat->s, nth, jd, sg, 0, 0, 0, HAVE_JD); diff --git a/ext/date/date_strptime.c b/ext/date/date_strptime.c index 1dde5fa3ec93b5..0ad03a0da9c693 100644 --- a/ext/date/date_strptime.c +++ b/ext/date/date_strptime.c @@ -41,7 +41,6 @@ static const int ABBREVIATED_MONTH_NAME_LENGTH = 3; #define f_match(r,s) rb_funcall(r, rb_intern("match"), 1, s) #define f_aref(o,i) rb_funcall(o, rb_intern("[]"), 1, i) -#define f_end(o,i) rb_funcall(o, rb_intern("end"), 1, i) #define issign(c) ((c) == '-' || (c) == '+') @@ -144,7 +143,7 @@ valid_range_p(VALUE v, int a, int b) do { \ size_t l; \ l = date__strptime_internal(&str[si], slen - si, \ - fmt, sizeof fmt - 1, hash); \ + fmt, sizeof fmt - 1, hash, enc); \ if (fail_p()) \ return 0; \ si += l; \ @@ -160,7 +159,8 @@ head_match_p(size_t len, const char *name, const char *str, size_t slen, size_t static size_t date__strptime_internal(const char *str, size_t slen, - const char *fmt, size_t flen, VALUE hash) + const char *fmt, size_t flen, + VALUE hash, rb_encoding *enc) { size_t si, fi; int c; @@ -597,15 +597,14 @@ date__strptime_internal(const char *str, size_t slen, b = rb_backref_get(); rb_match_busy(b); - m = f_match(pat, rb_usascii_str_new(&str[si], slen - si)); + m = f_match(pat, rb_enc_str_new(&str[si], slen - si, enc)); if (!NIL_P(m)) { - VALUE s, l, o; + VALUE s, o; s = rb_reg_nth_match(1, m); - l = f_end(m, INT2FIX(0)); o = date_zone_to_diff(s); - si += NUM2LONG(l); + si += RSTRING_LEN(s); set_hash("zone", s); set_hash("offset", o); rb_backref_set(b); @@ -654,12 +653,13 @@ date__strptime_internal(const char *str, size_t slen, VALUE date__strptime(const char *str, size_t slen, - const char *fmt, size_t flen, VALUE hash) + const char *fmt, size_t flen, + VALUE hash, rb_encoding *enc) { size_t si; VALUE cent, merid; - si = date__strptime_internal(str, slen, fmt, flen, hash); + si = date__strptime_internal(str, slen, fmt, flen, hash, enc); if (fail_p()) return Qnil; @@ -667,7 +667,7 @@ date__strptime(const char *str, size_t slen, if (slen > si) { VALUE s; - s = rb_usascii_str_new(&str[si], slen - si); + s = rb_enc_str_new(&str[si], slen - si, enc); set_hash("leftover", s); } diff --git a/ext/date/extconf.rb b/ext/date/extconf.rb index 3ab534f833edfd..963f96f0ac3201 100644 --- a/ext/date/extconf.rb +++ b/ext/date/extconf.rb @@ -13,4 +13,7 @@ have_var("altzone", "time.h", opt) end +have_func("rb_gc_mark_movable", "ruby.h") # RUBY_VERSION >= 2.7 +have_const("RUBY_TYPED_EMBEDDABLE", "ruby.h") # RUBY_VERSION >= 3.3 + create_makefile('date_core') diff --git a/iseq.c b/iseq.c index a2e71a1d451183..4cffe90f4d56a9 100644 --- a/iseq.c +++ b/iseq.c @@ -407,8 +407,6 @@ rb_iseq_mark_and_move(rb_iseq_t *iseq, bool reference_updating) { RUBY_MARK_ENTER("iseq"); - rb_gc_mark_and_move(&iseq->wrapper); - if (ISEQ_BODY(iseq)) { struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); @@ -1721,24 +1719,11 @@ static const rb_data_type_t iseqw_data_type = { static VALUE iseqw_new(const rb_iseq_t *iseq) { - if (iseq->wrapper) { - if (*(const rb_iseq_t **)rb_check_typeddata(iseq->wrapper, &iseqw_data_type) != iseq) { - rb_raise(rb_eTypeError, "wrong iseq wrapper: %" PRIsVALUE " for %p", - iseq->wrapper, (void *)iseq); - } - return iseq->wrapper; - } - else { - rb_iseq_t **ptr; - VALUE obj = TypedData_Make_Struct(rb_cISeq, rb_iseq_t *, &iseqw_data_type, ptr); - RB_OBJ_WRITE(obj, ptr, iseq); - - /* cache a wrapper object */ - RB_OBJ_SET_FROZEN_SHAREABLE((VALUE)obj); - RB_OBJ_WRITE((VALUE)iseq, &iseq->wrapper, obj); - - return obj; - } + rb_iseq_t **ptr; + VALUE obj = TypedData_Make_Struct(rb_cISeq, rb_iseq_t *, &iseqw_data_type, ptr); + RB_OBJ_WRITE(obj, ptr, iseq); + RB_OBJ_SET_FROZEN_SHAREABLE((VALUE)obj); + return obj; } VALUE diff --git a/ractor.c b/ractor.c index 201e363d2bb99f..5311c4b96ba861 100644 --- a/ractor.c +++ b/ractor.c @@ -418,9 +418,7 @@ free_targeted_hooks(st_table *hooks_tbl) st_foreach(hooks_tbl, free_targeted_hook_lists, 0); } -#ifdef RUBY_THREAD_PTHREAD_H void rb_thread_sched_destroy(struct rb_thread_sched *); -#endif static void ractor_free(void *ptr) @@ -429,13 +427,8 @@ ractor_free(void *ptr) RUBY_DEBUG_LOG("free r:%d", rb_ractor_id(r)); free_targeted_hooks(&r->pub.targeted_hooks); -#ifdef RUBY_THREAD_PTHREAD_H rb_thread_sched_destroy(&r->threads.sched); -#endif rb_native_mutex_destroy(&r->sync.lock); -#ifdef RUBY_THREAD_WIN32_H - rb_native_cond_destroy(&r->sync.wakeup_cond); -#endif ractor_local_storage_free(r); rb_hook_list_free(&r->pub.hooks); rb_st_free_embedded_table(&r->pub.targeted_hooks); @@ -1024,39 +1017,6 @@ rb_vm_ractor_blocking_cnt_dec(rb_vm_t *vm, rb_ractor_t *cr, const char *file, in ractor_status_set(cr, ractor_running); } -static void -ractor_check_blocking(rb_ractor_t *cr, unsigned int remained_thread_cnt, const char *file, int line) -{ - VM_ASSERT(cr == GET_RACTOR()); - -#ifdef RUBY_THREAD_PTHREAD_H - // vm->ractor.blocking_cnt is only consumed by the win32 scheduler; the - // pthread one must not pay a VM lock per blocking region for it. The - // running<->blocking status flips stop with it (all callers), matching - // rb_ractor_blocking_threads_dec skipping the reverse transition. - return; -#endif - - RUBY_DEBUG_LOG2(file, line, - "cr->threads.cnt:%u cr->threads.blocking_cnt:%u vm->ractor.cnt:%u vm->ractor.blocking_cnt:%u", - cr->threads.cnt, cr->threads.blocking_cnt, - GET_VM()->ractor.cnt, GET_VM()->ractor.blocking_cnt); - - VM_ASSERT(cr->threads.cnt >= cr->threads.blocking_cnt + 1); - - if (remained_thread_cnt > 0 && - // will be block - cr->threads.cnt == cr->threads.blocking_cnt + 1) { - // change ractor status: running -> blocking - rb_vm_t *vm = GET_VM(); - - RB_VM_LOCKING() { - rb_vm_ractor_blocking_cnt_inc(vm, cr, file, line); - } - } -} - - /* Remove a child that never started (send_parameters failed during creation). The * creator calls this (rb_ractor_living_threads_remove assumes the current Ractor); * leaving the set and disowning the objspace share one VM-lock section, no window. */ @@ -1100,8 +1060,6 @@ rb_ractor_living_threads_remove(rb_ractor_t *cr, rb_thread_t *th) { VM_ASSERT(cr == GET_RACTOR()); RUBY_DEBUG_LOG("r->threads.cnt:%d--", cr->threads.cnt); - ractor_check_blocking(cr, cr->threads.cnt - 1, __FILE__, __LINE__); - if (cr->threads.cnt == 1) { vm_remove_ractor(th->vm, cr); @@ -1124,7 +1082,6 @@ rb_ractor_blocking_threads_inc(rb_ractor_t *cr, const char *file, int line) VM_ASSERT(cr->threads.cnt > 0); VM_ASSERT(cr == GET_RACTOR()); - ractor_check_blocking(cr, cr->threads.cnt, __FILE__, __LINE__); cr->threads.blocking_cnt++; } @@ -1137,17 +1094,6 @@ rb_ractor_blocking_threads_dec(rb_ractor_t *cr, const char *file, int line) VM_ASSERT(cr == GET_RACTOR()); -#ifndef RUBY_THREAD_PTHREAD_H - // see rb_ractor_blocking_threads_inc - if (cr->threads.cnt == cr->threads.blocking_cnt) { - rb_vm_t *vm = GET_VM(); - - RB_VM_LOCKING() { - rb_vm_ractor_blocking_cnt_dec(vm, cr, __FILE__, __LINE__); - } - } -#endif - cr->threads.blocking_cnt--; } @@ -1236,7 +1182,6 @@ rb_ractor_terminate_all(void) rb_vm_ractor_blocking_cnt_inc(vm, cr, __FILE__, __LINE__); rb_del_running_thread(rb_ec_thread_ptr(cr->threads.running_ec)); rb_vm_cond_timedwait(vm, &vm->ractor.sync.terminate_cond, 1000 /* ms */); -#ifdef RUBY_THREAD_PTHREAD_H while (vm->ractor.sched.barrier_is_waiting) { // A barrier is waiting. Threads relinquish the VM lock before joining the barrier and // since we just acquired the VM lock back, we're blocking other threads from joining it. @@ -1246,7 +1191,6 @@ rb_ractor_terminate_all(void) unsigned int lev; RB_VM_LOCK_ENTER_LEV_NB(&lev); } -#endif rb_add_running_thread(rb_ec_thread_ptr(cr->threads.running_ec)); rb_vm_ractor_blocking_cnt_dec(vm, cr, __FILE__, __LINE__); diff --git a/ractor_core.h b/ractor_core.h index d918648c1f6276..6a545251473dc5 100644 --- a/ractor_core.h +++ b/ractor_core.h @@ -24,10 +24,6 @@ struct rb_ractor_sync { VALUE locked_by; #endif -#ifndef RUBY_THREAD_PTHREAD_H - rb_nativethread_cond_t wakeup_cond; -#endif - // incoming messages struct ractor_queue *recv_queue; diff --git a/ractor_sync.c b/ractor_sync.c index 9a63a5f84757c1..c6e4d8a6d31df1 100644 --- a/ractor_sync.c +++ b/ractor_sync.c @@ -945,9 +945,6 @@ ractor_sync_init(rb_ractor_t *r) // no receive is rebuilding a payload yet -#ifndef RUBY_THREAD_PTHREAD_H - rb_native_cond_initialize(&r->sync.wakeup_cond); -#endif } /* Create the default port. Call only after the Ractor joined vm->ractor.set, so the @@ -1304,73 +1301,6 @@ basket_type_name(enum ractor_basket_type type) #endif // USE_RUBY_DEBUG_LOG -#ifdef RUBY_THREAD_PTHREAD_H - -// - -#else // win32 - -static void -ractor_cond_wait(rb_ractor_t *r, const rb_hrtime_t *end) -{ -#if RACTOR_CHECK_MODE > 0 - VALUE locked_by = r->sync.locked_by; - r->sync.locked_by = Qnil; -#endif - if (end) { - rb_hrtime_t now = rb_hrtime_now(); - rb_hrtime_t rel = *end > now ? *end - now : 0; - // the condvar takes msec: never round a live timeout down to 0 - unsigned long msec = (unsigned long)(rel / RB_HRTIME_PER_MSEC); - rb_native_cond_timedwait(&r->sync.wakeup_cond, &r->sync.lock, msec > 0 ? msec : 1); - } - else { - rb_native_cond_wait(&r->sync.wakeup_cond, &r->sync.lock); - } - -#if RACTOR_CHECK_MODE > 0 - r->sync.locked_by = locked_by; -#endif -} - -static void * -ractor_wait_no_gvl(void *ptr) -{ - struct ractor_waiter *waiter = (struct ractor_waiter *)ptr; - rb_ractor_t *cr = waiter->th->ractor; - - RACTOR_LOCK_SELF(cr); - { - if (waiter->wakeup_status == wakeup_none) { - ractor_cond_wait(cr, waiter->end); - } - } - RACTOR_UNLOCK_SELF(cr); - return NULL; -} - -static void -rb_ractor_sched_wait(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf, void *ptr) -{ - struct ractor_waiter *waiter = (struct ractor_waiter *)ptr; - - RACTOR_UNLOCK(cr); - { - rb_nogvl(ractor_wait_no_gvl, waiter, - ubf, waiter, - RB_NOGVL_UBF_ASYNC_SAFE | RB_NOGVL_INTR_FAIL); - } - RACTOR_LOCK(cr); -} - -static void -rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *th) -{ - // ractor lock is acquired - rb_native_cond_broadcast(&r->sync.wakeup_cond); -} -#endif - static bool ractor_wakeup_all(rb_ractor_t *r, enum ractor_wakeup_status wakeup_status) { diff --git a/test/date/test_date_new.rb b/test/date/test_date_new.rb index eddeeff820f59e..b839c5283b4053 100644 --- a/test/date/test_date_new.rb +++ b/test/date/test_date_new.rb @@ -192,6 +192,27 @@ def test_civil__ex end end + def test_civil__offset + d = DateTime.civil(2001,2,3, 0,0,0, Rational(1, 1)) + assert_equal(1.to_r, d.offset) + d = DateTime.civil(2001,2,3, 0,0,0, Rational(-1, 1)) + assert_equal(-1.to_r, d.offset) + + # An out-of-range offset is ignored, as it is for the equivalent Integer. + assert_warning(/invalid offset/) do + d = DateTime.civil(2001,2,3, 0,0,0, 2) + end + assert_equal(0, d.offset) + assert_warning(/invalid offset/) do + d = DateTime.civil(2001,2,3, 0,0,0, Rational(2, 1)) + end + assert_equal(0, d.offset) + assert_warning(/invalid offset/) do + d = DateTime.civil(2001,2,3, 0,0,0, Rational(49710, 1)) + end + assert_equal(0, d.offset) + end + def test_civil__reform d = Date.jd(Date::ENGLAND, Date::ENGLAND) dt = DateTime.jd(Date::ENGLAND, 0,0,0,0, Date::ENGLAND) @@ -256,6 +277,11 @@ def test_commercial__ex end end + def test_commercial_p + assert_equal(false, Date.valid_commercial?(2024, -53, 1)) + assert_equal(false, Date.valid_commercial?(2024, -1227133565, 1)) + end + def test_weeknum d = Date.weeknum dt = DateTime.weeknum diff --git a/test/date/test_date_strptime.rb b/test/date/test_date_strptime.rb index 6aa7db292de424..f68f749e054269 100644 --- a/test/date/test_date_strptime.rb +++ b/test/date/test_date_strptime.rb @@ -533,4 +533,11 @@ def to_str assert_equal(6, d[:mon]) assert_equal(1, d[:mday]) end + + def test_nonascii_string + nonalpha = "\u{2600 fe0f}" + s = "2011-10-05T22:26:12#{nonalpha}" + + assert_nil(DateTime._strptime(s)) + end end diff --git a/test/ruby/test_iseq.rb b/test/ruby/test_iseq.rb index 9d946c8e8efd69..c01bb3dc90bfa1 100644 --- a/test/ruby/test_iseq.rb +++ b/test/ruby/test_iseq.rb @@ -837,21 +837,6 @@ def test_iseq_of } end - def test_iseq_of_twice_for_same_code - [ - proc{}, - method(:test_iseq_of_twice_for_same_code), - RubyVM::InstructionSequence.compile("p 1"), - begin; raise "error"; rescue => error; error.backtrace_locations[0]; end - ].each{|src| - iseq1 = RubyVM::InstructionSequence.of(src) - iseq2 = RubyVM::InstructionSequence.of(src) - - # ISeq objects should be same for same src - assert_equal iseq1.object_id, iseq2.object_id - } - end - def test_iseq_builtin_to_a invokebuiltin = eval(EnvUtil.invoke_ruby(['-e', <<~EOS], '', true).first) insns = RubyVM::InstructionSequence.of([].method(:pack)).to_a.last diff --git a/thread.c b/thread.c index ea404dd66dd97f..4736a8cd67ae75 100644 --- a/thread.c +++ b/thread.c @@ -278,12 +278,19 @@ MAYBE_UNUSED(NOINLINE(static int thread_start_func_2(rb_thread_t *th, VALUE *sta MAYBE_UNUSED(static bool th_has_dedicated_nt(const rb_thread_t *th)); MAYBE_UNUSED(static int waitfd_to_waiting_flag(int wfd_event)); -#include THREAD_IMPL_SRC +#ifdef RB_THREAD_SCHED_NONE +// The no-thread model is not a set of primitives under the common scheduler: +// it replaces the scheduler with stubs, so it stands alone. +# include THREAD_IMPL_SRC +#else +// The scheduler pulls in the platform implementation (THREAD_IMPL_SRC) itself: +// the platform primitives come first, the scheduler is built on top of them. +# include "thread_sched.c" +#endif /* * TODO: somebody with win32 knowledge should be able to get rid of - * timer-thread by busy-waiting on signals. And it should be possible - * to make the GVL in thread_pthread.c be platform-independent. + * timer-thread by busy-waiting on signals. */ #ifndef BUSY_WAIT_SIGNALS # define BUSY_WAIT_SIGNALS (0) @@ -841,7 +848,7 @@ thread_start_func_2(rb_thread_t *th, VALUE *stack_start) // Run the coroutine thread's epilogue here, while th is still valid; // co_start then only makes the final transfer (see - // coroutine_thread_terminated in thread_pthread_mn.c). + // coroutine_thread_terminated in thread_sched_mn.c). coroutine_thread_terminated(th); rb_ractor_postmortem_free(&pf); return 0; @@ -6112,9 +6119,7 @@ rb_check_deadlock(rb_ractor_t *r) { if (GET_THREAD()->vm->thread_ignore_deadlock) return; -#ifdef RUBY_THREAD_PTHREAD_H if (r->threads.sched.readyq_cnt > 0) return; -#endif int sleeper_num = rb_ractor_sleeper_thread_num(r); int ltnum = rb_ractor_living_thread_num(r); diff --git a/thread_none.c b/thread_none.c index 1a3ff8bcd2cd39..7de59b6f9af6f2 100644 --- a/thread_none.c +++ b/thread_none.c @@ -42,12 +42,10 @@ rb_thread_sched_init(struct rb_thread_sched *sched, bool atfork) { } -#if 0 -static void +void rb_thread_sched_destroy(struct rb_thread_sched *sched) { } -#endif // Do nothing for mutex guard void @@ -311,6 +309,25 @@ rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr) // do nothing } +void +rb_ractor_sched_barrier_end(rb_vm_t *vm, rb_ractor_t *cr) +{ + // do nothing +} + +void +rb_ractor_sched_wait(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf, void *ptr) +{ + // nothing can wake this: with no threads there is nobody to send + rb_bug("unreachable"); +} + +void +rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *th) +{ + // do nothing +} + bool diff --git a/thread_none.h b/thread_none.h index ac47e52bdaeba6..7cc045ea1dfbca 100644 --- a/thread_none.h +++ b/thread_none.h @@ -8,12 +8,16 @@ // based implementation in vm.c #define RB_THREAD_LOCAL_SPECIFIER -struct rb_native_thread { - void *thread_id; // NULL -}; +// This model brings its own scheduler stubs (thread_none.c) instead of the +// common one; thread.c keys off this. +#define RB_THREAD_SCHED_NONE 1 -struct rb_thread_sched_item {}; -struct rb_thread_sched {}; +// The scheduler's types are shared with the threaded platforms: the code that +// reads them (vm_sync.c, ractor.c, thread.c) is compiled for every thread +// model, so it needs the real fields even here, where nothing ever runs +// concurrently and they all stay zero. thread_none.c only ever passes the +// structs around, never looks inside them. +#include "thread_sched.h" RUBY_EXTERN struct rb_execution_context_struct *ruby_current_ec; NOINLINE(struct rb_execution_context_struct *rb_current_ec_noinline(void)); // for assertions diff --git a/thread_pthread.c b/thread_pthread.c index f31cad114a303f..275a4f9b97a4de 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -62,6 +62,11 @@ static pthread_condattr_t *condattr_monotonic = &condattr_mono; static const void *const condattr_monotonic = NULL; #endif +// Whether native_cond_timedwait() takes an rb_hrtime_t deadline as is. It +// does when the condvar counts in the same clock rb_hrtime_now() reads; +// otherwise the caller has to restate the deadline in the condvar's clock. +#define RB_NATIVE_COND_HRTIME_DEADLINE_P() (condattr_monotonic != NULL) + /* A retiring shared native thread frees its own context while the threads it * parked are still suspended with that context as their target. */ #define COROUTINE_TARGET_MAY_BE_FREED 1 @@ -324,21 +329,6 @@ static rb_serial_t current_fork_gen = 1; /* We can't use GET_VM()->fork_gen */ # define USE_UBF_LIST 1 #endif -static void threadptr_trap_interrupt(rb_thread_t *); - -static void native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt); -static void native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt); -static void native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th); - -static void ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r); -static void timer_thread_wakeup(void); -static void timer_thread_wakeup_locked(rb_vm_t *vm); -static void timer_thread_wakeup_force(void); -static void timer_thread_wake_fence(struct rb_thread_struct *th); -static bool ractor_sched_timeout_arm(rb_thread_t *th, const rb_hrtime_t *rel); -static bool ractor_sched_timeout_disarm(rb_thread_t *th); -static void thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th); -static void ractor_sched_cancel_enq(rb_vm_t *vm, struct rb_thread_sched *sched); #if USE_MN_THREADS static void nt_machine_stack_atfork(void); @@ -357,1788 +347,102 @@ struct rb_thread_context { static bool thread_sched_reclaim(struct coroutine_context *dead_co); #endif -static void coroutine_transfer0(struct coroutine_context *transfer_from, - struct coroutine_context *transfer_to, bool to_dead); - -#define thread_sched_dump(s) thread_sched_dump_(__FILE__, __LINE__, s) - -static bool -th_has_dedicated_nt(const rb_thread_t *th) -{ - // TODO: th->has_dedicated_nt - return th->nt->dedicated > 0; -} - -RBIMPL_ATTR_MAYBE_UNUSED() -static void -thread_sched_dump_(const char *file, int line, struct rb_thread_sched *sched) -{ - fprintf(stderr, "@%s:%d running:%d\n", file, line, sched->running ? (int)sched->running->serial : -1); - rb_thread_t *th; - int i = 0; - ccan_list_for_each(&sched->readyq, th, sched.node.readyq) { - i++; if (i>10) rb_bug("too many"); - fprintf(stderr, " ready:%d (%sNT:%d)\n", th->serial, - th->nt ? (th->nt->dedicated ? "D" : "S") : "x", - th->nt ? (int)th->nt->serial : -1); - } -} - -#define ractor_sched_dump(s) ractor_sched_dump_(__FILE__, __LINE__, s) - -RBIMPL_ATTR_MAYBE_UNUSED() -static void -ractor_sched_dump_(const char *file, int line, rb_vm_t *vm) -{ - rb_ractor_t *r; - fprintf(stderr, "ractor_sched_dump %s:%d\n", file, line); - int i = 0; - ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) { - i++; - if (i>10) rb_bug("!!"); - fprintf(stderr, " %d ready:%d\n", i, rb_ractor_id(r)); - } -} - -#define thread_sched_lock(a, b) thread_sched_lock_(a, b, __FILE__, __LINE__) -#define thread_sched_unlock(a, b) thread_sched_unlock_(a, b, __FILE__, __LINE__) - -static void -thread_sched_set_locked(struct rb_thread_sched *sched, rb_thread_t *th) +#ifdef RB_THREAD_T_HAS_NATIVE_ID +static int +get_native_thread_id(void) { -#if VM_CHECK_MODE > 0 - VM_ASSERT(sched->lock_owner == NULL); - - sched->lock_owner = th; +#ifdef __linux__ + return (int)syscall(SYS_gettid); +#elif defined(__FreeBSD__) + return pthread_getthreadid_np(); #endif } - -static void -thread_sched_set_unlocked(struct rb_thread_sched *sched, rb_thread_t *th) -{ -#if VM_CHECK_MODE > 0 - VM_ASSERT(sched->lock_owner == th); - - sched->lock_owner = NULL; #endif -} -static void -thread_sched_lock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line) -{ - rb_native_mutex_lock(&sched->lock_); -#if VM_CHECK_MODE - RUBY_DEBUG_LOG2(file, line, "r:%d th:%u", th ? (int)rb_ractor_id(th->ractor) : -1, rb_th_serial(th)); +#ifdef RB_THREAD_LOCAL_SPECIFIER +static RB_THREAD_LOCAL_SPECIFIER rb_thread_t *ruby_native_thread; #else - RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th)); +static pthread_key_t ruby_native_thread_key; #endif - thread_sched_set_locked(sched, th); -} - static void -thread_sched_unlock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line) +null_func(int i) { - RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th)); - - thread_sched_set_unlocked(sched, th); - - rb_native_mutex_unlock(&sched->lock_); + /* null */ + // This function can be called from signal handler + // RUBY_DEBUG_LOG("i:%d", i); } -#if USE_MN_THREADS -// Like thread_sched_unlock(), but never dereferences th (the debug log above -// reads th->serial). For the MN termination epilogue, which unlocks after th -// may already be collectable. Keep in sync with thread_sched_unlock_. -static void -thread_sched_unlock_no_log(struct rb_thread_sched *sched, rb_thread_t *th) +rb_thread_t * +ruby_thread_from_native(void) { - thread_sched_set_unlocked(sched, th); // pointer compare only - - rb_native_mutex_unlock(&sched->lock_); -} +#ifdef RB_THREAD_LOCAL_SPECIFIER + return ruby_native_thread; +#else + return pthread_getspecific(ruby_native_thread_key); #endif +} -static void -ASSERT_thread_sched_locked(struct rb_thread_sched *sched, rb_thread_t *th) +int +ruby_thread_set_native(rb_thread_t *th) { - VM_ASSERT(rb_native_mutex_trylock(&sched->lock_) == EBUSY); - -#if VM_CHECK_MODE if (th) { - VM_ASSERT(sched->lock_owner == th); - } - else { - VM_ASSERT(sched->lock_owner != NULL); - } +#ifdef USE_UBF_LIST + ccan_list_node_init(&th->sched.node.ubf); #endif -} - -#define ractor_sched_lock(a, b) ractor_sched_lock_(a, b, __FILE__, __LINE__) -#define ractor_sched_unlock(a, b) ractor_sched_unlock_(a, b, __FILE__, __LINE__) - -RBIMPL_ATTR_MAYBE_UNUSED() -static unsigned int -rb_ractor_serial(const rb_ractor_t *r) -{ - if (r) { - return rb_ractor_id(r); - } - else { - return 0; } -} - -static void -ractor_sched_set_locked(rb_vm_t *vm, rb_ractor_t *cr) -{ -#if VM_CHECK_MODE > 0 - VM_ASSERT(vm->ractor.sched.lock_owner == NULL); - VM_ASSERT(vm->ractor.sched.locked == false); - - vm->ractor.sched.lock_owner = cr; - vm->ractor.sched.locked = true; -#endif -} - -static void -ractor_sched_set_unlocked(rb_vm_t *vm, rb_ractor_t *cr) -{ -#if VM_CHECK_MODE > 0 - VM_ASSERT(vm->ractor.sched.locked); - VM_ASSERT(vm->ractor.sched.lock_owner == cr); - - vm->ractor.sched.locked = false; - vm->ractor.sched.lock_owner = NULL; -#endif -} - -static void -ractor_sched_lock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line) -{ - rb_native_mutex_lock(&vm->ractor.sched.lock); + // setup TLS -#if VM_CHECK_MODE - RUBY_DEBUG_LOG2(file, line, "cr:%u prev_owner:%u", rb_ractor_serial(cr), rb_ractor_serial(vm->ractor.sched.lock_owner)); + if (th && th->ec) { + rb_ractor_set_current_ec(th->ractor, th->ec); + } +#ifdef RB_THREAD_LOCAL_SPECIFIER + ruby_native_thread = th; + return 1; #else - RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr)); + return pthread_setspecific(ruby_native_thread_key, th) == 0; #endif - - ractor_sched_set_locked(vm, cr); -} - -static void -ractor_sched_unlock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line) -{ - RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr)); - - ractor_sched_set_unlocked(vm, cr); - rb_native_mutex_unlock(&vm->ractor.sched.lock); -} - -static void -ASSERT_ractor_sched_locked(rb_vm_t *vm, rb_ractor_t *cr) -{ - VM_ASSERT(rb_native_mutex_trylock(&vm->ractor.sched.lock) == EBUSY); - VM_ASSERT(vm->ractor.sched.locked); - VM_ASSERT(cr == NULL || vm->ractor.sched.lock_owner == cr); -} - -static void ractor_sched_barrier_join_signal_locked(rb_vm_t *vm); - -/* ntlist registration: a thread that executes Ruby code is always registered, - * in its snt's nt->running_th or on running_dnts via its dedicated nt. The - * only unregistered execution is scheduler glue (parking, resuming), which - * touches no Ruby heap, and the barrier wait below. */ -static void -ntlist_add_running(rb_vm_t *vm, rb_thread_t *th) -{ - struct rb_native_thread *nt = th->nt; - - // a dedicated nt is not on the snts list the scans walk: running_dnts instead - if (nt != NULL && nt->dedicated == 0) { - rb_native_mutex_lock(&nt->running_th_lock); - { - VM_ASSERT(nt->running_th == NULL); - nt->running_th = th; - } - rb_native_mutex_unlock(&nt->running_th_lock); - } - else { - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); - { - // an snt gone dedicated (rb_thread_lock_native_thread) has no - // creation-time running_thread: the registration supplies it - nt->running_thread = th; - ccan_list_add(&vm->ractor.sched.ntlist.running_dnts, &nt->running_dnts_node); - } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); - } -} - -// Returns whether the active barrier's walk had counted this registration: -// such a deregistration owes the snapshot count a decrement. Read and -// cleared under the registration's own lock, so it pairs with the walk. -static bool -ntlist_del_running(rb_vm_t *vm, rb_thread_t *th) -{ - struct rb_native_thread *nt = th->nt; - uint32_t serial; - bool counted; - bool in_running_th; - - // The registration itself says where it is: nt->running_th holds th, or - // th's nt hangs on running_dnts. barrier_serial is read inside the - // registration's lock, ordered with the walk that stamped there. - rb_native_mutex_lock(&nt->running_th_lock); - { - in_running_th = (nt->running_th == th); - if (in_running_th) { - nt->running_th = NULL; - serial = vm->ractor.sched.barrier_serial; - counted = (nt->barrier_counted_serial == serial); - nt->barrier_counted_serial = serial - 1; // only once per barrier - } - } - rb_native_mutex_unlock(&nt->running_th_lock); - - if (!in_running_th) { - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); - { - ccan_list_del_init(&nt->running_dnts_node); - serial = vm->ractor.sched.barrier_serial; - counted = (nt->barrier_counted_serial == serial); - nt->barrier_counted_serial = serial - 1; - } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); - } - return counted; -} - -// Stamp a registration into the active barrier's snapshot unless the walk -// already counted it; returns whether it stamped. Called under sched.lock, -// so it is serialized with the walk: the stamp says exactly whether the -// registration came first. -static bool -ntlist_stamp_if_uncounted(rb_vm_t *vm, rb_thread_t *th) -{ - struct rb_native_thread *nt = th->nt; - uint32_t serial = vm->ractor.sched.barrier_serial; // sched.lock is held - bool stamped; - bool in_running_th; - - rb_native_mutex_lock(&nt->running_th_lock); - { - in_running_th = (nt->running_th == th); - if (in_running_th) { - stamped = (nt->barrier_counted_serial != serial); - nt->barrier_counted_serial = serial; - } - } - rb_native_mutex_unlock(&nt->running_th_lock); - - if (!in_running_th) { - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); - { - stamped = (nt->barrier_counted_serial != serial); - nt->barrier_counted_serial = serial; - } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); - } - return stamped; -} - -// Record a thread entering/leaving the running set, with no global lock and -// no count: the records themselves are what the barrier counts. Pairing: -// the barrier sets barrier_is_waiting and then walks the records under their -// locks; we move a record and then read the flag, so one side sees the other. -// List sched for the timer's timeslice ticks. The caller holds sched->lock_ -// with the readyq non-empty, so the timer cannot prune the entry meanwhile. -static void -timeslice_sched_link(rb_vm_t *vm, struct rb_thread_sched *sched) -{ - rb_native_mutex_lock(&vm->ractor.sched.timeslice.lock); - { - if (sched->timeslice_node.next == &sched->timeslice_node) { - ccan_list_add_tail(&vm->ractor.sched.timeslice.scheds, &sched->timeslice_node); - } - } - rb_native_mutex_unlock(&vm->ractor.sched.timeslice.lock); -} - -static void -thread_sched_setup_running_threads(struct rb_thread_sched *sched, rb_ractor_t *cr, rb_vm_t *vm, - rb_thread_t *add_th, rb_thread_t *del_th) -{ - RUBY_DEBUG_LOG("+:%u -:%u", rb_th_serial(add_th), rb_th_serial(del_th)); - - if (del_th) { - bool counted = ntlist_del_running(vm, del_th); - sched->is_running = false; - - // The first load is only a filter; the one under sched.lock decides. - // A missed flag means this deregistration preceded the barrier's walk. - if (UNLIKELY(RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting))) { - ractor_sched_lock(vm, cr); - { - if (RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting)) { - if (counted) { - VM_ASSERT(vm->ractor.sched.barrier_running_cnt > 0); - vm->ractor.sched.barrier_running_cnt--; - } - ractor_sched_barrier_join_signal_locked(vm); - } - } - ractor_sched_unlock(vm, cr); - } - } - - if (add_th) { - ntlist_add_running(vm, add_th); - - if (UNLIKELY(RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting))) { - // A stop-the-world section. In its waiting phase sched.lock is - // takable: join the snapshot count and take the interrupt (this - // thread joins at its next check, like any walked runner). In - // the GC phase the barrier holds sched.lock to its end, so this - // blocks here, as the old global-lock design did. - ractor_sched_lock(vm, cr); - { - if (RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting) && - ntlist_stamp_if_uncounted(vm, add_th)) { - // the walk ran before this registration; count it in - RUBY_DEBUG_LOG("barrier_is_waiting"); - vm->ractor.sched.barrier_running_cnt++; - RUBY_VM_SET_VM_BARRIER_INTERRUPT(add_th->ec); - } - } - ractor_sched_unlock(vm, cr); - } - - sched->is_running = true; - - // taking a turn with waiters already queued needs the timeslice ticks - if (!ccan_list_empty(&sched->readyq)) { - timeslice_sched_link(vm, sched); - ractor_sched_lock(vm, cr); - { - if (vm->ractor.sched.timeslice_wait_inf) { - timer_thread_wakeup_locked(vm); - } - } - ractor_sched_unlock(vm, cr); - } - } -} - -static void -thread_sched_add_running_thread(struct rb_thread_sched *sched, rb_thread_t *th) -{ - ASSERT_thread_sched_locked(sched, th); - VM_ASSERT(sched->running == th); - - rb_vm_t *vm = th->vm; - thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL); -} - -static void -thread_sched_del_running_thread(struct rb_thread_sched *sched, rb_thread_t *th) -{ - ASSERT_thread_sched_locked(sched, th); - - rb_vm_t *vm = th->vm; - thread_sched_setup_running_threads(sched, th->ractor, vm, NULL, th); } -void -rb_add_running_thread(rb_thread_t *th) -{ - struct rb_thread_sched *sched = TH_SCHED(th); +static void native_thread_setup(struct rb_native_thread *nt); +static void native_thread_setup_on_thread(struct rb_native_thread *nt); - thread_sched_lock(sched, th); - { - thread_sched_add_running_thread(sched, th); - } - thread_sched_unlock(sched, th); -} +// Internal cache of page size: +static size_t RB_THREAD_PAGE_SIZE; void -rb_del_running_thread(rb_thread_t *th) -{ - struct rb_thread_sched *sched = TH_SCHED(th); - - thread_sched_lock(sched, th); - { - thread_sched_del_running_thread(sched, th); - } - thread_sched_unlock(sched, th); -} - -// setup current or next running thread -// sched->running should be set only on this function. -// -// if th is NULL, there is no running threads. -static void -thread_sched_set_running(struct rb_thread_sched *sched, rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%u->th:%u", rb_th_serial(sched->running), rb_th_serial(th)); - VM_ASSERT(sched->running != th); - - if (RUBY_DTRACE_RTS_SET_RUNNING_ENABLED()) { - RUBY_DTRACE_RTS_SET_RUNNING(sched, sched->running, th); - } - - sched->running = th; -} - -RBIMPL_ATTR_MAYBE_UNUSED() -static bool -thread_sched_readyq_contain_p(struct rb_thread_sched *sched, rb_thread_t *th) -{ - rb_thread_t *rth; - ccan_list_for_each(&sched->readyq, rth, sched.node.readyq) { - if (rth == th) { - VM_ASSERT(th->sched.node.is_ready); - return true; - } - } - VM_ASSERT(!th->sched.node.is_ready); - return false; -} - -// deque thread from the ready queue. -// if the ready queue is empty, return NULL. -// -// return deque'ed running thread (or NULL). -static rb_thread_t * -thread_sched_deq(struct rb_thread_sched *sched) -{ - ASSERT_thread_sched_locked(sched, NULL); - rb_thread_t *next_th; - - VM_ASSERT(sched->running != NULL); - - if (ccan_list_empty(&sched->readyq)) { - next_th = NULL; - } - else { - next_th = ccan_list_pop(&sched->readyq, rb_thread_t, sched.node.readyq); - VM_ASSERT(next_th->sched.node.is_ready); - next_th->sched.node.is_ready = false; - - VM_ASSERT(sched->readyq_cnt > 0); - sched->readyq_cnt--; - ccan_list_node_init(&next_th->sched.node.readyq); - } - - RUBY_DEBUG_LOG("next_th:%u readyq_cnt:%d", rb_th_serial(next_th), sched->readyq_cnt); - - return next_th; -} - -// enqueue ready thread to the ready queue. -static void -thread_sched_enq(struct rb_thread_sched *sched, rb_thread_t *ready_th) +Init_native_thread(rb_thread_t *main_th) { - ASSERT_thread_sched_locked(sched, NULL); - RUBY_DEBUG_LOG("ready_th:%u readyq_cnt:%d", rb_th_serial(ready_th), sched->readyq_cnt); - - VM_ASSERT(sched->running != NULL); - VM_ASSERT(!thread_sched_readyq_contain_p(sched, ready_th)); - - bool timeslice_onset = sched->is_running && ccan_list_empty(&sched->readyq); - - ccan_list_add_tail(&sched->readyq, &ready_th->sched.node.readyq); - ready_th->sched.node.is_ready = true; - sched->readyq_cnt++; - - if (timeslice_onset) { - // The running thread needs the timeslice ticks now. Linked before - // the check under sched.lock: either the timer's scan (same lock) - // sees the sched, or this sees timeslice_wait_inf. - rb_vm_t *vm = ready_th->vm; - timeslice_sched_link(vm, sched); - ractor_sched_lock(vm, NULL); - { - if (vm->ractor.sched.timeslice_wait_inf) { - timer_thread_wakeup_locked(vm); - } - } - ractor_sched_unlock(vm, NULL); - } -} + // Get the system page size for later use in stack allocation and stack overflow checks: + RB_THREAD_PAGE_SIZE = sysconf(_SC_PAGESIZE); -// DNT: kick condvar -// SNT: TODO -static void -thread_sched_wakeup_running_thread(struct rb_thread_sched *sched, rb_thread_t *next_th, bool will_switch) -{ - ASSERT_thread_sched_locked(sched, NULL); - VM_ASSERT(sched->running == next_th); - - if (next_th) { - if (next_th->nt) { - if (th_has_dedicated_nt(next_th)) { - RUBY_DEBUG_LOG("pinning th:%u", next_th->serial); - rb_native_cond_signal(&next_th->nt->readyq); - } - else { - // TODO - RUBY_DEBUG_LOG("th:%u is already running.", next_th->serial); - } - } - else { - if (will_switch) { - RUBY_DEBUG_LOG("th:%u (do nothing)", rb_th_serial(next_th)); - } - else { - RUBY_DEBUG_LOG("th:%u (enq)", rb_th_serial(next_th)); - ractor_sched_enq(next_th->vm, next_th->ractor); - } +#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) + if (condattr_monotonic) { + int r = pthread_condattr_init(condattr_monotonic); + if (r == 0) { + r = pthread_condattr_setclock(condattr_monotonic, CLOCK_MONOTONIC); } + if (r) condattr_monotonic = NULL; } - else { - RUBY_DEBUG_LOG("no waiting threads%s", ""); - } -} - -// waiting -> ready (locked) -static void -thread_sched_to_ready_common(struct rb_thread_sched *sched, rb_thread_t *th, bool wakeup, bool will_switch) -{ - RUBY_DEBUG_LOG("th:%u running:%u redyq_cnt:%d", rb_th_serial(th), rb_th_serial(sched->running), sched->readyq_cnt); - - VM_ASSERT(sched->running != th); - VM_ASSERT(!thread_sched_readyq_contain_p(sched, th)); - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_READY, th); +#endif - if (sched->running == NULL) { - thread_sched_set_running(sched, th); - if (wakeup) thread_sched_wakeup_running_thread(sched, th, will_switch); +#ifndef RB_THREAD_LOCAL_SPECIFIER + if (pthread_key_create(&ruby_native_thread_key, 0) == EAGAIN) { + rb_bug("pthread_key_create failed (ruby_native_thread_key)"); } - else { - thread_sched_enq(sched, th); + if (pthread_key_create(&ruby_current_ec_key, 0) == EAGAIN) { + rb_bug("pthread_key_create failed (ruby_current_ec_key)"); } -} +#endif + ruby_posix_signal(SIGVTALRM, null_func); -// waiting -> ready -// -// `th` had became "waiting" state by `thread_sched_to_waiting` -// and `thread_sched_to_ready` enqueue `th` to the thread ready queue. -RBIMPL_ATTR_MAYBE_UNUSED() -static void -thread_sched_to_ready(struct rb_thread_sched *sched, rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); - - thread_sched_lock(sched, th); - { - thread_sched_to_ready_common(sched, th, true, false); - } - thread_sched_unlock(sched, th); -} - -// wait until sched->running is `th`. `end` is an absolute deadline for a dedicated -static void -thread_sched_wait_running_turn(struct rb_thread_sched *sched, rb_thread_t *th, bool can_direct_transfer, const rb_hrtime_t *end) -{ - RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); - - ASSERT_thread_sched_locked(sched, th); - VM_ASSERT(th == rb_ec_thread_ptr(rb_current_ec_noinline())); - - bool timedout = false; - - if (th != sched->running) { - // TODO: This optimization should also be made to work for MN_THREADS - if (th->has_dedicated_nt && th == sched->runnable_hot_th && (sched->running == NULL || sched->running->has_dedicated_nt)) { - RUBY_DEBUG_LOG("(nt) stealing: hot-th:%u. running:%u", rb_th_serial(th), rb_th_serial(sched->running)); - - // th serves itself on its own nt, displacing the enqueued - // running thread back to the readyq: cancel the entry that was - // posted for it (a later dequeue would find this Ractor served - // and its next enqueue would double-list the node) - ractor_sched_cancel_enq(th->vm, sched); - - // If there is a thread set to run, move it back to the front of the readyq - if (sched->running != NULL) { - rb_thread_t *running = sched->running; - VM_ASSERT(!thread_sched_readyq_contain_p(sched, running)); - running->sched.node.is_ready = true; - ccan_list_add(&sched->readyq, &running->sched.node.readyq); - sched->readyq_cnt++; - } - - // Pull off the ready queue and start running. - if (th->sched.node.is_ready) { - VM_ASSERT(thread_sched_readyq_contain_p(sched, th)); - ccan_list_del_init(&th->sched.node.readyq); - th->sched.node.is_ready = false; - sched->readyq_cnt--; - } - thread_sched_set_running(sched, th); - rb_ractor_thread_switch(th->ractor, th, false); - } - else if (th == sched->runnable_hot_th) { - // The hot thread cannot steal the control (e.g. the running thread - // is an MN thread). It is going to sleep, so it is no longer spinning; - // drop the hint so that other threads don't yield the lock to it. - sched->runnable_hot_th = NULL; - sched->runnable_hot_th_waiting = 0; - } - - // already deleted from running threads - - - // wait for execution right - rb_thread_t *next_th; - while((next_th = sched->running) != th) { - if (th_has_dedicated_nt(th)) { - RUBY_DEBUG_LOG("(nt) sleep th:%u running:%u", rb_th_serial(th), rb_th_serial(sched->running)); - - thread_sched_set_unlocked(sched, th); - { - RUBY_DEBUG_LOG("nt:%d cond:%p", th->nt->serial, &th->nt->readyq); - rb_nativethread_cond_t *cond = &th->nt->readyq; - - // Once someone has queued this thread the deadline is spent: it - // is waiting for a turn, not for the time, and arming a kernel - // timer for every round of that costs more than the wait. - // Once someone has queued this thread the deadline is spent: it - // is waiting for a turn, not for the time, and arming a kernel - // timer for every round of that costs more than the wait. - if (end && !th->sched.node.is_ready) { - rb_hrtime_t abs = *end; - - if (!condattr_monotonic) { - // the condvar counts in another clock: restate it there - rb_hrtime_t now = rb_hrtime_now(); - abs = native_cond_timeout(cond, *end > now ? *end - now : 0); - } - timedout = native_cond_timedwait(cond, &sched->lock_, &abs) == ETIMEDOUT; - } - else { - rb_native_cond_wait(cond, &sched->lock_); - } - } - thread_sched_set_locked(sched, th); - - if (timedout && - sched->running != th && !th->sched.node.is_ready) { - // the deadline passed and nobody woke this thread: get back in - // line for the running turn, then wait for it without a deadline - thread_sched_to_ready_common(sched, th, false, false); - end = NULL; - } - - if (sched->runnable_hot_th != NULL && sched->runnable_hot_th_waiting) { - VM_ASSERT(sched->runnable_hot_th != th); - // Give the hot thread a chance to preempt, if it's actively spinning. - // On multicore, this reduces the rate of core-switching. On single-core it - // should mostly be a nop, since the other thread can't be concurrently spinning. - thread_sched_unlock(sched, th); - thread_sched_lock(sched, th); - } - - RUBY_DEBUG_LOG("(nt) wakeup %s", sched->running == th ? "success" : "failed"); - if (th == sched->running) { - rb_ractor_thread_switch(th->ractor, th, false); - } - } - else { - // search another ready thread - if (can_direct_transfer && - (next_th = sched->running) != NULL && - !next_th->nt // next_th is running or has dedicated nt - ) { - - RUBY_DEBUG_LOG("th:%u->%u (direct)", rb_th_serial(th), rb_th_serial(next_th)); - - thread_sched_set_unlocked(sched, th); - { - rb_ractor_set_current_ec(th->ractor, NULL); - thread_sched_switch(th, next_th); - } - thread_sched_set_locked(sched, th); - } - else { - // search another ready ractor - struct rb_native_thread *nt = th->nt; - native_thread_assign(NULL, th); - - RUBY_DEBUG_LOG("th:%u->%u (ractor scheduling)", rb_th_serial(th), rb_th_serial(next_th)); - - thread_sched_set_unlocked(sched, th); - { - rb_ractor_set_current_ec(th->ractor, NULL); - coroutine_transfer0(th->sched.context, nt->nt_context, false); - } - thread_sched_set_locked(sched, th); - } - - VM_ASSERT(rb_current_ec_noinline() == th->ec); - } - } - - VM_ASSERT(th->nt != NULL); - VM_ASSERT(rb_current_ec_noinline() == th->ec); - VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none); - - // add th to running threads - thread_sched_add_running_thread(sched, th); - } - - // Control transfer to the current thread is now complete. The original thread - // cannot steal control at this point. - sched->runnable_hot_th = NULL; - sched->runnable_hot_th_waiting = 0; - - - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_RESUMED, th); -} - -// waiting -> ready -> running (locked) -static void -thread_sched_to_running_common(struct rb_thread_sched *sched, rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%u dedicated:%d", rb_th_serial(th), th_has_dedicated_nt(th)); - - VM_ASSERT(sched->running != th); - VM_ASSERT(th_has_dedicated_nt(th)); - VM_ASSERT(GET_THREAD() == th); - - native_thread_dedicated_dec(th->vm, th->ractor, th->nt); - - // waiting -> ready - thread_sched_to_ready_common(sched, th, false, false); - - if (sched->running == th) { - thread_sched_add_running_thread(sched, th); - } - - // TODO: check SNT number - thread_sched_wait_running_turn(sched, th, false, NULL); -} - -// waiting -> ready -> running -// -// `th` had been waiting by `thread_sched_to_waiting()` -// and run a dedicated task (like waitpid and so on). -// After the dedicated task, this function is called -// to join a normal thread-scheduling. -static void -thread_sched_to_running(struct rb_thread_sched *sched, rb_thread_t *th) -{ - // We are reading and writing these sched fields without lock cover, but - // there are no correctness issues resulting from stale cache or delayed writeback. - // When it works, this causes the next-scheduled thread to yield the sched lock - // briefly so that we can grab it if we're still spinning (not descheduled yet). - if (sched->runnable_hot_th == th) { - sched->runnable_hot_th_waiting = 1; - } - thread_sched_lock(sched, th); - { - thread_sched_to_running_common(sched, th); - } - thread_sched_unlock(sched, th); -} - -// resume a next thread in the thread ready queue. -// -// deque next running thread from the ready thread queue and -// resume this thread if available. -// -// If the next therad has a dedicated native thraed, simply signal to resume. -// Otherwise, make the ractor ready and other nt will run the ractor and the thread. -static void -thread_sched_wakeup_next_thread(struct rb_thread_sched *sched, rb_thread_t *th, bool will_switch) -{ - ASSERT_thread_sched_locked(sched, th); - - VM_ASSERT(sched->running == th); - VM_ASSERT(sched->running->nt != NULL); - - rb_thread_t *next_th = thread_sched_deq(sched); - - RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th)); - VM_ASSERT(th != next_th); - - thread_sched_set_running(sched, next_th); - VM_ASSERT(next_th == sched->running); - thread_sched_wakeup_running_thread(sched, next_th, will_switch); - - if (th != next_th) { - thread_sched_del_running_thread(sched, th); - } -} - -// running -> dead (locked) -static void -thread_sched_to_dead_common(struct rb_thread_sched *sched, rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated); - - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); - - // A dying coroutine thread (will_switch=true here) does NOT wake the - // next thread now: it is still winding down (co_start's epilogue), and - // the same Ractor must not have two threads executing at once. The - // epilogue enqueues the Ractor after its last rb_ractor_t access. - thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th)); - - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_EXITED, th); -} - -// running -> dead -static void -thread_sched_to_dead(struct rb_thread_sched *sched, rb_thread_t *th) -{ - // wait out any pending wake here, while th's Ractor is still alive - timer_thread_wake_fence(th); - - thread_sched_lock(sched, th); - { - thread_sched_to_dead_common(sched, th); - } - thread_sched_unlock(sched, th); -} - -// running -> waiting (locked) -// -// This thread will run dedicated task (th->nt->dedicated++). -static void -thread_sched_to_waiting_common(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately) -{ - RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated); - - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); - - native_thread_dedicated_inc(th->vm, th->ractor, th->nt); - if (!yield_immediately) { - sched->runnable_hot_th = th; - sched->runnable_hot_th_waiting = 0; - } - thread_sched_wakeup_next_thread(sched, th, false); -} - -// running -> waiting -// -// This thread will run a dedicated task. -static void -thread_sched_to_waiting(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately) -{ - thread_sched_lock(sched, th); - { - thread_sched_to_waiting_common(sched, th, yield_immediately); - } - thread_sched_unlock(sched, th); -} - -// mini utility func -// return true if any there are any interrupts -static bool -ubf_set(rb_thread_t *th, rb_unblock_function_t *func, void *arg, rb_atomic_t *event_serial) -{ - VM_ASSERT(func != NULL); - - retry: - if (RUBY_VM_INTERRUPTED(th->ec)) { - RUBY_DEBUG_LOG("interrupted:0x%x", th->ec->interrupt_flag); - return true; - } - - rb_native_mutex_lock(&th->interrupt_lock); - { - if (!th->ec->raised_flag && RUBY_VM_INTERRUPTED(th->ec)) { - rb_native_mutex_unlock(&th->interrupt_lock); - goto retry; - } - - VM_ASSERT(th->unblock.func == NULL); - th->unblock.func = func; - th->unblock.arg = arg; - if (event_serial) { - rb_atomic_t prev_serial = RUBY_ATOMIC_FETCH_ADD(th->unblock.event_serial, 1); - *event_serial = prev_serial+1; - } - } - rb_native_mutex_unlock(&th->interrupt_lock); - - return false; -} - -static void -ubf_clear(rb_thread_t *th, bool clear_serial) -{ - rb_native_mutex_lock(&th->interrupt_lock); - { - th->unblock.func = NULL; - th->unblock.arg = NULL; - if (clear_serial) { - RUBY_ATOMIC_ADD(th->unblock.event_serial, 1); - } - } - rb_native_mutex_unlock(&th->interrupt_lock); -} - -static void -ubf_waiting(void *ptr) -{ - rb_thread_t *th = (rb_thread_t *)ptr; - struct rb_thread_sched *sched = TH_SCHED(th); - - // only once. it is safe because th->interrupt_lock is already acquired. - th->unblock.func = NULL; - th->unblock.arg = NULL; - - RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); - - thread_sched_lock(sched, th); - { - if (sched->running == th || th->sched.node.is_ready) { - // not sleeping yet, or a deadline already put it back in line - } - else { - thread_sched_to_ready_common(sched, th, true, false); - - // If the turn is taken, th stays parked until the running thread yields. - // For a timed wait, wake it early anyway: it re-parks at once, but its - // wakeup then runs on another core in parallel with the running thread, - // off the handoff path. An untimed wait has no post-wake bookkeeping - // worth pipelining, so it skips the extra futex round. - if (sched->running != th && th->sched.waiting_timed && - th->nt != NULL && th_has_dedicated_nt(th)) { - rb_native_cond_signal(&th->nt->readyq); - } - } - } - thread_sched_unlock(sched, th); -} - -// running -> waiting -// -// This thread will sleep until other thread wakeup the thread. `end` is an -// absolute deadline, NULL to sleep until woken; only a dedicated native thread, -// which parks on its own condvar, can take one. -static void -thread_sched_to_waiting_until_wakeup(struct rb_thread_sched *sched, rb_thread_t *th, const rb_hrtime_t *end) -{ - RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); - - VM_ASSERT(end == NULL || th_has_dedicated_nt(th)); - - RB_VM_SAVE_MACHINE_CONTEXT(th); - - - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); - - thread_sched_lock(sched, th); - { - // NOTE: there's a lock ordering inversion here with the ubf call, but it's benign. - if (ubf_set(th, ubf_waiting, (void *)th, NULL)) { - RUBY_DEBUG_LOG("th:%u interrupted", rb_th_serial(th)); - } - else { - bool can_direct_transfer = !th_has_dedicated_nt(th); - th->sched.waiting_timed = (end != NULL); // never true here for M:N (end is NULL) - // NOTE: th->status is set before and after this sleep outside of this function in `sleep_forever` - thread_sched_wakeup_next_thread(sched, th, can_direct_transfer); - thread_sched_wait_running_turn(sched, th, can_direct_transfer, end); - th->sched.waiting_timed = false; - } - } - thread_sched_unlock(sched, th); - - ubf_clear(th, false); -} - -// run another thread in the ready queue. -// continue to run if there are no ready threads. -static void -thread_sched_yield(struct rb_thread_sched *sched, rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%d sched->readyq_cnt:%d", (int)th->serial, sched->readyq_cnt); - - thread_sched_lock(sched, th); - { - if (!ccan_list_empty(&sched->readyq)) { - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); - thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th)); - bool can_direct_transfer = !th_has_dedicated_nt(th); - thread_sched_to_ready_common(sched, th, false, can_direct_transfer); - thread_sched_wait_running_turn(sched, th, can_direct_transfer, NULL); - th->status = THREAD_RUNNABLE; - } - else { - VM_ASSERT(sched->readyq_cnt == 0); - } - } - thread_sched_unlock(sched, th); -} - -void -rb_thread_sched_init(struct rb_thread_sched *sched, bool atfork) -{ - rb_native_mutex_initialize(&sched->lock_); - -#if VM_CHECK_MODE - sched->lock_owner = NULL; -#endif - - ccan_list_head_init(&sched->readyq); - sched->readyq_cnt = 0; - ccan_list_node_init(&sched->grq_node); // self-linked = not enqueued - ccan_list_node_init(&sched->timeslice_node); - -#if USE_MN_THREADS - if (!atfork) sched->enable_mn_threads = true; // MN is enabled on Ractors -#endif -} - -static void -coroutine_transfer0(struct coroutine_context *transfer_from, struct coroutine_context *transfer_to, bool to_dead) -{ -#ifdef RUBY_ASAN_ENABLED - void **fake_stack = to_dead ? NULL : &transfer_from->fake_stack; - __sanitizer_start_switch_fiber(fake_stack, transfer_to->stack_base, transfer_to->stack_size); -#endif - -#if defined(COROUTINE_SANITIZE_THREAD) - /* Tell TSan we are switching to transfer_to's fiber before the stack - * switch, so its per-thread shadow stack stays bound to the right - * coroutine. */ - __tsan_switch_to_fiber(transfer_to->tsan_fiber, 0); -#endif - - RBIMPL_ATTR_MAYBE_UNUSED() - struct coroutine_context *returning_from = coroutine_transfer(transfer_from, transfer_to); - - /* if to_dead was passed, the caller is promising that this coroutine is finished and it should - * never be resumed! */ - VM_ASSERT(!to_dead); -#ifdef RUBY_ASAN_ENABLED - __sanitizer_finish_switch_fiber(transfer_from->fake_stack, - (const void**)&returning_from->stack_base, &returning_from->stack_size); -#endif -} - -static void -thread_sched_switch0(struct coroutine_context *current_cont, rb_thread_t *next_th, struct rb_native_thread *nt, bool to_dead) -{ - VM_ASSERT(!nt->dedicated); - VM_ASSERT(next_th->nt == NULL); - - RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th)); - - // this direct transfer serves next_th without a dequeue; cancel its - // Ractor's outstanding grq entry (no-op when nothing is enqueued) - ractor_sched_cancel_enq(next_th->vm, TH_SCHED(next_th)); - - ruby_thread_set_native(next_th); - native_thread_assign(nt, next_th); - - coroutine_transfer0(current_cont, next_th->sched.context, to_dead); -} - -static void -thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th) -{ - struct rb_native_thread *nt = cth->nt; - native_thread_assign(NULL, cth); - RUBY_DEBUG_LOG("th:%u->%u on nt:%d", rb_th_serial(cth), rb_th_serial(next_th), nt->serial); - thread_sched_switch0(cth->sched.context, next_th, nt, cth->status == THREAD_KILLED); -} - -#if VM_CHECK_MODE > 0 -RBIMPL_ATTR_MAYBE_UNUSED() -static unsigned int -grq_size(rb_vm_t *vm, rb_ractor_t *cr) -{ - ASSERT_ractor_sched_locked(vm, cr); - - rb_ractor_t *r, *prev_r = NULL; - unsigned int i = 0; - - ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) { - i++; - - VM_ASSERT(r != prev_r); - prev_r = r; - } - return i; -} -#endif - -// A native thread enters/leaves an epilogue that outlives its Ractor: from -// the increment until the decrement, ruby_vm_destruct waits for it below. -// The increment must happen while the VM still counts the thread's Ractor, -// so that the two never look absent at the same time. -void -rb_thread_sched_winding_begin(rb_vm_t *vm) -{ - RUBY_ATOMIC_INC(vm->ractor.sched.winding_cnt); -} - -void -rb_thread_sched_winding_end(rb_vm_t *vm) -{ - VM_ASSERT(RUBY_ATOMIC_LOAD(vm->ractor.sched.winding_cnt) > 0); - RUBY_ATOMIC_DEC(vm->ractor.sched.winding_cnt); -} - -// ruby_vm_destruct: wait until no native thread is between a coroutine -// epilogue and its reclaim -- past that point the reclaim frees through the -// (about to be destroyed) objspace and reads the (about to be unset) VM. -// Runs without the VM lock, which the epilogue needs to progress. -void -rb_thread_sched_wait_winding(rb_vm_t *vm) -{ - while (RUBY_ATOMIC_LOAD(vm->ractor.sched.winding_cnt) > 0) { - native_thread_yield(); - } -} - -// A direct service of a runnable thread (direct transfer or the hot-thread -// steal) bypasses the grq; cancel the Ractor's outstanding entry so that -// "enqueued <=> runnable and unserved" keeps holding. The caller holds the -// per-Ractor sched lock, so no concurrent enqueue can relink the node: a -// self-linked read needs no lock (the common case -- direct switches whose -// transition never enqueued). A linked read can race only with a dequeue, -// hence the recheck under the grq lock. -static void -ractor_sched_cancel_enq(rb_vm_t *vm, struct rb_thread_sched *sched) -{ - if (sched->grq_node.next != &sched->grq_node) { - ractor_sched_lock(vm, NULL); - { - if (sched->grq_node.next != &sched->grq_node) { - ccan_list_del_init(&sched->grq_node); - VM_ASSERT(vm->ractor.sched.grq_cnt > 0); - vm->ractor.sched.grq_cnt--; - } - } - ractor_sched_unlock(vm, NULL); - } -} - -static void -ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r) -{ - struct rb_thread_sched *sched = &r->threads.sched; - rb_ractor_t *cr = NULL; // timer thread can call this function - - VM_ASSERT(sched->running != NULL); - VM_ASSERT(sched->running->nt == NULL); - - ractor_sched_lock(vm, cr); - { - // Precondition: not already enqueued (the grq_node is self-linked). - // This holds because every service of a runnable-but-unserved thread - // either dequeues the entry (the nt scheduling loop) or cancels it - // (direct transfers / the hot-thread steal; see - // ractor_sched_cancel_enq) -- re-adding a linked node would corrupt - // the queue, so check unconditionally (a CHECK-mode-only assert - // would miss it: the race needs timing that CHECK builds perturb). - if (sched->grq_node.next != &sched->grq_node) { - rb_bug("ractor_sched_enq: already enqueued"); - } - ccan_list_add_tail(&vm->ractor.sched.grq, &sched->grq_node); - vm->ractor.sched.grq_cnt++; - VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt); - - RUBY_DEBUG_LOG("r:%u th:%u grq_cnt:%u", rb_ractor_id(r), rb_th_serial(sched->running), vm->ractor.sched.grq_cnt); - - rb_native_cond_signal(&vm->ractor.sched.cond); - - // The signal reaches a parked snt, and a running one revisits the - // queue in ractor_sched_deq before it can wait (same lock as here). - // With every snt dedicated or retired, only the timer thread's - // timeout branch can serve the entry or widen the pool: wake it - // (a no-op unless it sleeps untimed). - if (RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt) == 0) { - timer_thread_wakeup_locked(vm); - } - - // ractor_sched_dump(vm); - } - ractor_sched_unlock(vm, cr); -} - - -#ifndef MINIMUM_SNT -// make at least MINIMUM_SNT snts for debug. -#define MINIMUM_SNT 0 -#endif - -/* A shared thread woken with nothing to run is one whose turn another thread - * took first. After this many in a row it gives itself back: the queue keeps - * running dry, so the pool is wider than the work. 0 retires on the first one - * and is too eager to be useful; a negative value keeps every thread. */ -#ifndef SNT_IDLE_RETIRE -#define SNT_IDLE_RETIRE 3 -#endif - -/* Never give the last shared thread back. With none left an enqueue has nobody - * to signal, and the only code that makes one runs on the timer thread's - * timeout branch, which is reached only once it has seen a backlog. */ -#define SNT_KEEP_MINIMUM (MINIMUM_SNT > 1 ? MINIMUM_SNT : 1) - -static rb_ractor_t * -ractor_sched_deq(rb_vm_t *vm, rb_ractor_t *cr) -{ - rb_ractor_t *r; - int idle_streak = 0; // consecutive pops that found the queue empty - - ractor_sched_lock(vm, cr); - { - RUBY_DEBUG_LOG("empty? %d", ccan_list_empty(&vm->ractor.sched.grq)); - // ractor_sched_dump(vm); - - VM_ASSERT(rb_current_execution_context(false) == NULL); - VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt); - - while ((r = ccan_list_pop(&vm->ractor.sched.grq, rb_ractor_t, threads.sched.grq_node)) == NULL) { - RUBY_DEBUG_LOG("wait grq_cnt:%d", (int)vm->ractor.sched.grq_cnt); - - if (SNT_IDLE_RETIRE >= 0 && ++idle_streak > SNT_IDLE_RETIRE && - (int)RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt) > SNT_KEEP_MINIMUM) { - RUBY_ATOMIC_DEC(vm->ractor.sched.snt_cnt); - RUBY_DEBUG_LOG("retire, snt_cnt:%d", (int)vm->ractor.sched.snt_cnt); - break; // returning NULL ends this nt; see the caller - } - - ractor_sched_set_unlocked(vm, cr); - rb_native_cond_wait(&vm->ractor.sched.cond, &vm->ractor.sched.lock); - ractor_sched_set_locked(vm, cr); - - RUBY_DEBUG_LOG("wakeup grq_cnt:%d", (int)vm->ractor.sched.grq_cnt); - } - - VM_ASSERT(rb_current_execution_context(false) == NULL); - - if (r) { - ccan_list_node_init(&r->threads.sched.grq_node); // back to self-linked - VM_ASSERT(vm->ractor.sched.grq_cnt > 0); - vm->ractor.sched.grq_cnt--; - RUBY_DEBUG_LOG("r:%d grq_cnt:%u", (int)rb_ractor_id(r), vm->ractor.sched.grq_cnt); - } - else { - // the retire branch is the only way out of the loop without a ractor - VM_ASSERT(idle_streak > SNT_IDLE_RETIRE); - } - } - ractor_sched_unlock(vm, cr); - - return r; -} - -void rb_ractor_lock_self(rb_ractor_t *r); -void rb_ractor_unlock_self(rb_ractor_t *r); - -// The current thread for a ractor is put to "sleep" (descheduled in the STOPPED_FOREVER state) waiting for -// a ractor action to wake it up. -void -rb_ractor_sched_wait(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf, void *ubf_arg) -{ - // ractor lock of cr is acquired - - RUBY_DEBUG_LOG("start%s", ""); - - rb_thread_t * volatile th = rb_ec_thread_ptr(ec); - struct rb_thread_sched *sched = TH_SCHED(th); - struct ractor_waiter *waiter = (struct ractor_waiter*)ubf_arg; - - if (ubf_set(th, ubf, ubf_arg, &waiter->event_serial)) { - // interrupted - return; - } - - thread_sched_lock(sched, th); - rb_ractor_unlock_self(cr); - { - // A dedicated native thread takes the deadline on the very condvar a wakeup - // signals. An M:N thread has no condvar of its own, so its deadline goes to - // the timer thread, which then wakes it the way rb_ractor_sched_wakeup() does. - bool dedicated = th_has_dedicated_nt(th); - const rb_hrtime_t *end_p = NULL; - bool armed = false, expired = false; - - if (waiter->end) { - if (dedicated) { - end_p = waiter->end; - } - else { - // the timer wheel takes a relative timeout - rb_hrtime_t now = rb_hrtime_now(); - rb_hrtime_t rel = *waiter->end > now ? *waiter->end - now : 0; - - armed = ractor_sched_timeout_arm(th, &rel); - expired = !armed; - } - } - - if (expired) { - RUBY_DEBUG_LOG("expired before sleep%s", ""); - } - else if (armed && th->sched.waiting_reason.flags == thread_sched_waiting_none) { - // the timer thread already took this thread out of the wheel; bump the - // serial so that it does not try to wake a thread that never slept - th->sched.event_serial++; - } - else { - // setup sleep - bool can_direct_transfer = !dedicated; - RB_VM_SAVE_MACHINE_CONTEXT(th); - th->status = THREAD_STOPPED_FOREVER; - th->sched.waiting_timed = (end_p != NULL); // never true here for M:N (end_p is NULL) - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); - thread_sched_wakeup_next_thread(sched, th, can_direct_transfer); - // sleep - thread_sched_wait_running_turn(sched, th, can_direct_transfer, end_p); - th->sched.waiting_timed = false; - th->status = THREAD_RUNNABLE; - - // whoever woke this thread took the timeout back first - VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none); - } - } - thread_sched_unlock(sched, th); - rb_ractor_lock_self(cr); - - ubf_clear(th, true); - - RUBY_DEBUG_LOG("end%s", ""); -} - -void -rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *r_th) -{ - // ractor lock of r acquired - struct rb_thread_sched *sched = TH_SCHED(r_th); - - RUBY_DEBUG_LOG("r:%u th:%d", (unsigned int)rb_ractor_id(r), r_th->serial); - - thread_sched_lock(sched, r_th); - { - if (r_th->status == THREAD_STOPPED_FOREVER) { - RUBY_ATOMIC_ADD(r_th->unblock.event_serial, 1); - - // r_th must not resume with a wheel entry left behind: take its timeout - // back, as ubf_event_waiting() does. Only r_th arms it, and it is - // parked here, so reading the flags without the timer lock is safe. - if (r_th->sched.waiting_reason.flags != thread_sched_waiting_none) { - ractor_sched_timeout_disarm(r_th); - } - - // a timeout that fired first may have made r_th runnable already: waking - // it twice would put it on the readyq twice - if (sched->running != r_th && !r_th->sched.node.is_ready) { - r_th->sched.event_serial++; // a timeout still armed must not wake it again - thread_sched_to_ready_common(sched, r_th, true, false); - } - } - } - thread_sched_unlock(sched, r_th); -} - -static bool -ractor_sched_barrier_completed_p(rb_vm_t *vm) -{ - // The snapshot barrier_running_cnt is taken by the barrier's walk and - // decremented by counted deregistrations; no rescan is needed here. - RUBY_DEBUG_LOG("run:%u wait:%u", vm->ractor.sched.barrier_running_cnt, vm->ractor.sched.barrier_joined_cnt); - VM_ASSERT(vm->ractor.sched.barrier_running_cnt - 1 >= vm->ractor.sched.barrier_joined_cnt); - - return (vm->ractor.sched.barrier_running_cnt - vm->ractor.sched.barrier_joined_cnt) == 1; -} - -void -rb_ractor_sched_barrier_start(rb_vm_t *vm, rb_ractor_t *cr) -{ - VM_ASSERT(cr == GET_RACTOR()); - VM_ASSERT(vm->ractor.sync.lock_owner == cr); // VM is locked - VM_ASSERT(!vm->ractor.sched.barrier_is_waiting); - VM_ASSERT(vm->ractor.sched.barrier_joined_cnt == 0); - VM_ASSERT(vm->ractor.sched.barrier_ractor == NULL); - VM_ASSERT(vm->ractor.sched.barrier_lock_rec == 0); - - RUBY_DEBUG_LOG("start serial:%u", vm->ractor.sched.barrier_serial); - - unsigned int lock_rec; - - ractor_sched_lock(vm, cr); - { - RUBY_ATOMIC_SET(vm->ractor.sched.barrier_is_waiting, 1); - vm->ractor.sched.barrier_ractor = cr; - vm->ractor.sched.barrier_lock_rec = vm->ractor.sync.lock_rec; - - // release VM lock - lock_rec = vm->ractor.sync.lock_rec; - vm->ractor.sync.lock_rec = 0; - vm->ractor.sync.lock_owner = NULL; - rb_native_mutex_unlock(&vm->ractor.sync.lock); - - // Interrupt all running threads: running_dnts plus each snt's running_th. - // A switch before this scan is visible to it; one after it sees - // barrier_is_waiting (set above) and waits. - // Interrupt and count every registered runner, stamping each nt so a - // deregistration during this barrier knows it was counted. - rb_thread_t *ith; - unsigned int running_cnt = 0; - uint32_t serial = vm->ractor.sched.barrier_serial; - - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); - { - struct rb_native_thread *dnt; - ccan_list_for_each(&vm->ractor.sched.ntlist.running_dnts, dnt, running_dnts_node) { - ith = dnt->running_thread; - dnt->barrier_counted_serial = serial; - running_cnt++; - if (ith->ractor != cr) { - RUBY_DEBUG_LOG("barrier request to th:%u", rb_th_serial(ith)); - RUBY_VM_SET_VM_BARRIER_INTERRUPT(ith->ec); - } - } - - struct rb_native_thread *nt; - ccan_list_for_each(&vm->ractor.sched.ntlist.snts, nt, snts_node) { - rb_native_mutex_lock(&nt->running_th_lock); - { - ith = nt->running_th; - if (ith != NULL) { - nt->barrier_counted_serial = serial; - running_cnt++; - if (ith->ractor != cr) { - RUBY_DEBUG_LOG("barrier request to th:%u", rb_th_serial(ith)); - RUBY_VM_SET_VM_BARRIER_INTERRUPT(ith->ec); - } - } - } - rb_native_mutex_unlock(&nt->running_th_lock); - } - } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); - - vm->ractor.sched.barrier_running_cnt = running_cnt; - - // wait for other ractors - while (!ractor_sched_barrier_completed_p(vm)) { - ractor_sched_set_unlocked(vm, cr); - rb_native_cond_wait(&vm->ractor.sched.barrier_complete_cond, &vm->ractor.sched.lock); - ractor_sched_set_locked(vm, cr); - } - - RUBY_DEBUG_LOG("completed seirial:%u", vm->ractor.sched.barrier_serial); - - // no other ractors are there - vm->ractor.sched.barrier_serial++; - vm->ractor.sched.barrier_joined_cnt = 0; - rb_native_cond_broadcast(&vm->ractor.sched.barrier_release_cond); - - // acquire VM lock - rb_native_mutex_lock(&vm->ractor.sync.lock); - vm->ractor.sync.lock_rec = lock_rec; - vm->ractor.sync.lock_owner = cr; - } - - // do not release ractor_sched_lock and there is no newly added (resumed) thread - // thread_sched_setup_running_threads -} - -// called from vm_lock_leave if the vm_lock used for barrierred -void -rb_ractor_sched_barrier_end(rb_vm_t *vm, rb_ractor_t *cr) -{ - RUBY_DEBUG_LOG("serial:%u", (unsigned int)vm->ractor.sched.barrier_serial - 1); - VM_ASSERT(vm->ractor.sched.barrier_is_waiting); - VM_ASSERT(vm->ractor.sched.barrier_ractor); - VM_ASSERT(vm->ractor.sched.barrier_lock_rec > 0); - - RUBY_ATOMIC_SET(vm->ractor.sched.barrier_is_waiting, 0); - vm->ractor.sched.barrier_ractor = NULL; - vm->ractor.sched.barrier_lock_rec = 0; - ractor_sched_unlock(vm, cr); -} - -static void -ractor_sched_barrier_join_signal_locked(rb_vm_t *vm) -{ - if (ractor_sched_barrier_completed_p(vm)) { - rb_native_cond_signal(&vm->ractor.sched.barrier_complete_cond); - } -} - -static void -ractor_sched_barrier_join_wait_locked(rb_vm_t *vm, rb_thread_t *th) -{ - VM_ASSERT(vm->ractor.sched.barrier_is_waiting); - - unsigned int barrier_serial = vm->ractor.sched.barrier_serial; - - while (vm->ractor.sched.barrier_serial == barrier_serial) { - RUBY_DEBUG_LOG("sleep serial:%u", barrier_serial); - RB_VM_SAVE_MACHINE_CONTEXT(th); - - rb_ractor_t *cr = th->ractor; - ractor_sched_set_unlocked(vm, cr); - rb_native_cond_wait(&vm->ractor.sched.barrier_release_cond, &vm->ractor.sched.lock); - ractor_sched_set_locked(vm, cr); - - RUBY_DEBUG_LOG("wakeup serial:%u", barrier_serial); - } -} - -void -rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr) -{ - VM_ASSERT(cr->threads.sched.running != NULL); // running ractor - VM_ASSERT(cr == GET_RACTOR()); - VM_ASSERT(vm->ractor.sync.lock_owner == NULL); // VM is locked, but owner == NULL - VM_ASSERT(vm->ractor.sched.barrier_is_waiting); // VM needs barrier sync - -#if USE_RUBY_DEBUG_LOG || VM_CHECK_MODE > 0 - unsigned int barrier_serial = vm->ractor.sched.barrier_serial; -#endif - - RUBY_DEBUG_LOG("join"); - - rb_native_mutex_unlock(&vm->ractor.sync.lock); - { - VM_ASSERT(vm->ractor.sched.barrier_is_waiting); // VM needs barrier sync - VM_ASSERT(vm->ractor.sched.barrier_serial == barrier_serial); - - ractor_sched_lock(vm, cr); - { - // running_cnt - /* Every joiner is a member of the running set: a dying thread - * leaves the living set before handing over its scheduler slot. */ - vm->ractor.sched.barrier_joined_cnt++; - RUBY_DEBUG_LOG("waiting_cnt:%u serial:%u", vm->ractor.sched.barrier_joined_cnt, barrier_serial); - - ractor_sched_barrier_join_signal_locked(vm); - ractor_sched_barrier_join_wait_locked(vm, cr->threads.sched.running); - } - ractor_sched_unlock(vm, cr); - } - - rb_native_mutex_lock(&vm->ractor.sync.lock); - // VM locked here -} - -// Called when the ractor holding this sched is freed. A drained sched can -// still be on timeslice.scheds (pruning is lazy); an unlisted node is -// self-linked (fork re-inits them all), making this del a no-op. -void -rb_thread_sched_destroy(struct rb_thread_sched *sched) -{ - rb_vm_t *vm = GET_VM(); - - rb_native_mutex_lock(&vm->ractor.sched.timeslice.lock); - { - ccan_list_del_init(&sched->timeslice_node); - } - rb_native_mutex_unlock(&vm->ractor.sched.timeslice.lock); -} - -#ifdef RB_THREAD_T_HAS_NATIVE_ID -static int -get_native_thread_id(void) -{ -#ifdef __linux__ - return (int)syscall(SYS_gettid); -#elif defined(__FreeBSD__) - return pthread_getthreadid_np(); -#endif -} -#endif - -#if defined(HAVE_WORKING_FORK) -static void rb_internal_thread_event_hooks_rw_lock_atfork(void); - -static void -thread_sched_atfork(struct rb_thread_sched *sched) -{ - current_fork_gen++; - rb_thread_sched_init(sched, true); - rb_thread_t *th = GET_THREAD(); - rb_vm_t *vm = GET_VM(); - - if (th_has_dedicated_nt(th)) { - vm->ractor.sched.snt_cnt = 0; -#if USE_RUBY_DEBUG_LOG - vm->ractor.sched.dnt_cnt = 1; -#endif - } - else { - vm->ractor.sched.snt_cnt = 1; -#if USE_RUBY_DEBUG_LOG - vm->ractor.sched.dnt_cnt = 0; -#endif - } - - rb_native_mutex_initialize(&vm->ractor.sched.lock); -#if VM_CHECK_MODE > 0 - vm->ractor.sched.lock_owner = NULL; - vm->ractor.sched.locked = false; -#endif - - // rb_native_cond_destroy(&vm->ractor.sched.cond); - rb_native_cond_initialize(&vm->ractor.sched.cond); - rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond); - rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond); - - ccan_list_head_init(&vm->ractor.sched.grq); - vm->ractor.sched.grq_cnt = 0; // the list was just emptied; reset the count with it - // A fork during a VM barrier leaves the child with barrier state that can - // never complete (the other ractors are gone); reset it like the rest. - vm->ractor.sched.barrier_is_waiting = 0; // single-threaded child - vm->ractor.sched.barrier_joined_cnt = 0; - vm->ractor.sched.barrier_ractor = NULL; - vm->ractor.sched.barrier_lock_rec = 0; - // Threads that were winding down in the parent do not exist in the child; - // without this reset the child's ruby_vm_destruct would wait for their - // reclaim (which never comes) forever. - vm->ractor.sched.winding_cnt = 0; - rb_native_mutex_initialize(&vm->ractor.sched.ntlist.lock); - ccan_list_head_init(&vm->ractor.sched.ntlist.running_dnts); - ccan_list_head_init(&vm->ractor.sched.ntlist.snts); // those nts are gone - rb_native_mutex_initialize(&vm->ractor.sched.timeslice.lock); - ccan_list_head_init(&vm->ractor.sched.timeslice.scheds); - rb_native_mutex_initialize(&th->nt->running_th_lock); // a scan could hold it at fork - // Fork can copy nodes linked (or torn mid-link); re-init every sched's - // node so rb_thread_sched_destroy's del_init stays a no-op for them. - rb_ractor_t *r; - ccan_list_for_each(&vm->ractor.set, r, vmlr_node) { - ccan_list_node_init(&r->threads.sched.timeslice_node); - } - ccan_list_for_each(&vm->ractor.terminated_set, r, vmlr_node) { - ccan_list_node_init(&r->threads.sched.timeslice_node); - } - // th re-records itself below; the parent's record did not survive the lists - if (th->nt && th->nt->dedicated == 0) { - // surviving on an snt: put that nt back on the (just emptied) snts - // list, or the scans could not see this thread's record - ccan_list_add(&vm->ractor.sched.ntlist.snts, &th->nt->snts_node); - } - -#if USE_MN_THREADS - nt_machine_stack_atfork(); -#endif - rb_internal_thread_event_hooks_rw_lock_atfork(); - - VM_ASSERT(sched->is_running); - - if (sched->running != th) { - thread_sched_to_running(sched, th); - } - else { - thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL); - } - -#ifdef RB_THREAD_T_HAS_NATIVE_ID - if (th->nt) { - th->nt->tid = get_native_thread_id(); - } -#endif -} - -#endif - -#ifdef RB_THREAD_LOCAL_SPECIFIER -static RB_THREAD_LOCAL_SPECIFIER rb_thread_t *ruby_native_thread; -#else -static pthread_key_t ruby_native_thread_key; -#endif - -static void -null_func(int i) -{ - /* null */ - // This function can be called from signal handler - // RUBY_DEBUG_LOG("i:%d", i); -} - -rb_thread_t * -ruby_thread_from_native(void) -{ -#ifdef RB_THREAD_LOCAL_SPECIFIER - return ruby_native_thread; -#else - return pthread_getspecific(ruby_native_thread_key); -#endif -} - -int -ruby_thread_set_native(rb_thread_t *th) -{ - if (th) { -#ifdef USE_UBF_LIST - ccan_list_node_init(&th->sched.node.ubf); -#endif - } - - // setup TLS - - if (th && th->ec) { - rb_ractor_set_current_ec(th->ractor, th->ec); - } -#ifdef RB_THREAD_LOCAL_SPECIFIER - ruby_native_thread = th; - return 1; -#else - return pthread_setspecific(ruby_native_thread_key, th) == 0; -#endif -} - -static void native_thread_setup(struct rb_native_thread *nt); -static void native_thread_setup_on_thread(struct rb_native_thread *nt); - -// Internal cache of page size: -static size_t RB_THREAD_PAGE_SIZE; - -void -Init_native_thread(rb_thread_t *main_th) -{ - // Get the system page size for later use in stack allocation and stack overflow checks: - RB_THREAD_PAGE_SIZE = sysconf(_SC_PAGESIZE); - -#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) - if (condattr_monotonic) { - int r = pthread_condattr_init(condattr_monotonic); - if (r == 0) { - r = pthread_condattr_setclock(condattr_monotonic, CLOCK_MONOTONIC); - } - if (r) condattr_monotonic = NULL; - } -#endif - -#ifndef RB_THREAD_LOCAL_SPECIFIER - if (pthread_key_create(&ruby_native_thread_key, 0) == EAGAIN) { - rb_bug("pthread_key_create failed (ruby_native_thread_key)"); - } - if (pthread_key_create(&ruby_current_ec_key, 0) == EAGAIN) { - rb_bug("pthread_key_create failed (ruby_current_ec_key)"); - } -#endif - ruby_posix_signal(SIGVTALRM, null_func); - - // setup vm - rb_vm_t *vm = main_th->vm; - rb_native_mutex_initialize(&vm->ractor.sched.lock); - rb_native_cond_initialize(&vm->ractor.sched.cond); - rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond); - rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond); - - ccan_list_head_init(&vm->ractor.sched.grq); - rb_native_mutex_initialize(&vm->ractor.sched.ntlist.lock); - ccan_list_head_init(&vm->ractor.sched.ntlist.running_dnts); - ccan_list_head_init(&vm->ractor.sched.ntlist.snts); - rb_native_mutex_initialize(&vm->ractor.sched.timeslice.lock); - ccan_list_head_init(&vm->ractor.sched.timeslice.scheds); + // setup vm + rb_vm_t *vm = main_th->vm; + thread_sched_init_vm(vm); // setup main thread main_th->nt->thread_id = pthread_self(); @@ -2150,135 +454,23 @@ Init_native_thread(rb_thread_t *main_th) native_thread_setup(main_th->nt); native_thread_setup_on_thread(main_th->nt); - TH_SCHED(main_th)->running = main_th; - main_th->has_dedicated_nt = 1; - - // setup main NT (before the record below: its kind decides where it goes) - main_th->nt->dedicated = 1; - main_th->nt->running_thread = main_th; - main_th->nt->vm = vm; - - thread_sched_setup_running_threads(TH_SCHED(main_th), main_th->ractor, vm, main_th, NULL); - - // setup mn -#if USE_RUBY_DEBUG_LOG - vm->ractor.sched.dnt_cnt = 1; -#endif -} - -extern int ruby_mn_threads_enabled; - -void -ruby_mn_threads_params(void) -{ - rb_vm_t *vm = GET_VM(); - rb_ractor_t *main_ractor = GET_RACTOR(); - - const char *mn_threads_cstr = getenv("RUBY_MN_THREADS"); - bool enable_mn_threads = false; - - if (USE_MN_THREADS && mn_threads_cstr && (enable_mn_threads = atoi(mn_threads_cstr) > 0)) { - // enabled - ruby_mn_threads_enabled = 1; - } - main_ractor->threads.sched.enable_mn_threads = enable_mn_threads; - - const char *max_cpu_cstr = getenv("RUBY_MAX_CPU"); -#if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN) - long nprocessors = sysconf(_SC_NPROCESSORS_ONLN); - const int default_max_cpu = (nprocessors > 0) ? (int)nprocessors : 8; -#else - const int default_max_cpu = 8; -#endif - int max_cpu = default_max_cpu; - - if (USE_MN_THREADS && max_cpu_cstr) { - int given_max_cpu = atoi(max_cpu_cstr); - if (given_max_cpu > 0) { - max_cpu = given_max_cpu; - } - } - - vm->ractor.sched.max_cpu = max_cpu; -} - -static void -native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt) -{ - RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated + 1); - - if (nt->dedicated == 0) { - // Lock-free; pairs with ractor_sched_enq (enq: grq_cnt up then read - // snt_cnt / here: snt_cnt down then read grq_cnt) against lost wakeups. - if (RUBY_ATOMIC_FETCH_SUB(vm->ractor.sched.snt_cnt, 1) == 1) { - // the last snt went dedicated; pending entries need the timer thread - ractor_sched_lock(vm, cr); - { - if (vm->ractor.sched.grq_cnt > 0) { - timer_thread_wakeup_locked(vm); - } - } - ractor_sched_unlock(vm, cr); - } -#if USE_RUBY_DEBUG_LOG - vm->ractor.sched.dnt_cnt++; -#endif - } - - nt->dedicated++; -} + TH_SCHED(main_th)->running = main_th; + main_th->has_dedicated_nt = 1; -static void -native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt) -{ - RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated - 1); - VM_ASSERT(nt->dedicated > 0); - nt->dedicated--; - - if (nt->dedicated == 0) { - // Rejoin under the max_cpu cap; with no room this nt retires and - // belongs to neither count until it ends. - while (1) { - rb_atomic_t snt = RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt); - if (snt < vm->ractor.sched.max_cpu || (int)snt <= MINIMUM_SNT) { - if (RUBY_ATOMIC_CAS(vm->ractor.sched.snt_cnt, snt, snt + 1) == snt) break; - } - else { - nt->retiring = true; - break; - } - } -#if USE_RUBY_DEBUG_LOG - vm->ractor.sched.dnt_cnt--; -#endif - } -} + // setup main NT (before the record below: its kind decides where it goes) + main_th->nt->dedicated = 1; + main_th->nt->running_thread = main_th; + main_th->nt->vm = vm; -static void -native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th) -{ + thread_sched_setup_running_threads(TH_SCHED(main_th), main_th->ractor, vm, main_th, NULL); + + // setup mn #if USE_RUBY_DEBUG_LOG - if (nt) { - if (th->nt) { - RUBY_DEBUG_LOG("th:%d nt:%d->%d", (int)th->serial, (int)th->nt->serial, (int)nt->serial); - } - else { - RUBY_DEBUG_LOG("th:%d nt:NULL->%d", (int)th->serial, (int)nt->serial); - } - } - else { - if (th->nt) { - RUBY_DEBUG_LOG("th:%d nt:%d->NULL", (int)th->serial, (int)th->nt->serial); - } - else { - RUBY_DEBUG_LOG("th:%d nt:NULL->NULL", (int)th->serial); - } - } + vm->ractor.sched.dnt_cnt = 1; #endif - - th->nt = nt; } + static void native_thread_destroy_atfork(struct rb_native_thread *nt) { @@ -2571,8 +763,6 @@ struct nt_param { struct rb_native_thread *nt; }; -static void * -nt_start(void *ptr); static int native_thread_create0(struct rb_native_thread *nt) @@ -2645,264 +835,8 @@ native_thread_alloc(void) return nt; } -static int -native_thread_create_dedicated(rb_thread_t *th) -{ - th->nt = native_thread_alloc(); - th->nt->vm = th->vm; - th->nt->running_thread = th; - th->nt->dedicated = 1; - - // vm stack - size_t vm_stack_word_size = th->vm->default_params.thread_vm_stack_size / sizeof(VALUE); - void *vm_stack = ruby_xmalloc(vm_stack_word_size * sizeof(VALUE)); - th->sched.malloc_stack = true; - rb_ec_initialize_vm_stack(th->ec, vm_stack, vm_stack_word_size); - th->sched.context_stack = vm_stack; - th->sched.context_stack_size = vm_stack_word_size; - - int err = native_thread_create0(th->nt); - if (!err) { - // setup - thread_sched_to_ready(TH_SCHED(th), th); - } - return err; -} - -static void -call_thread_start_func_2(rb_thread_t *th) -{ - /* Capture the address of a local in this stack frame to mark the beginning of the - machine stack for this thread. This is required even if we can tell the real - stack beginning from the pthread API in native_thread_init_stack, because - glibc stores some of its own data on the stack before calling into user code - on a new thread, and replacing that data on fiber-switch would break it (see - bug #13887) */ - VALUE stack_start = 0; - VALUE *stack_start_addr = asan_get_real_stack_addr(&stack_start); - - native_thread_init_stack(th, stack_start_addr); - thread_start_func_2(th, th->ec->machine.stack_start); -} - -static void * -nt_start(void *ptr) -{ - struct rb_native_thread *nt = (struct rb_native_thread *)ptr; - rb_vm_t *vm = nt->vm; - - native_thread_setup_on_thread(nt); - - // init tid -#ifdef RB_THREAD_T_HAS_NATIVE_ID - nt->tid = get_native_thread_id(); -#endif - -#if USE_RUBY_DEBUG_LOG && defined(RUBY_NT_SERIAL) - ruby_nt_serial = nt->serial; -#endif - - RUBY_DEBUG_LOG("nt:%u", nt->serial); - - bool in_snts = false; - - if (!nt->dedicated) { - coroutine_initialize_main(nt->nt_context); - - // join the snt list that the barrier/timeslice scans walk - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); - { - ccan_list_add(&vm->ractor.sched.ntlist.snts, &nt->snts_node); - } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); - in_snts = true; - } - - bool retired = false; - - while (1) { - if (nt->dedicated) { - // wait running turn - rb_thread_t *th = nt->running_thread; - struct rb_thread_sched *sched = TH_SCHED(th); - - RUBY_DEBUG_LOG("on dedicated th:%u", rb_th_serial(th)); - ruby_thread_set_native(th); - - thread_sched_lock(sched, th); - { - if (sched->running == th) { - thread_sched_add_running_thread(sched, th); - } - thread_sched_wait_running_turn(sched, th, false, NULL); - } - thread_sched_unlock(sched, th); - - // start threads - call_thread_start_func_2(th); - break; // TODO: allow to change to the SNT - } - else { - RUBY_DEBUG_LOG("check next"); - if (nt->retiring) { // came back with no room in the shared pool - retired = true; - break; - } - - rb_ractor_t *r = ractor_sched_deq(vm, NULL); - - if (r) { - struct rb_thread_sched *sched = &r->threads.sched; - - bool locked = true; - - thread_sched_lock(sched, NULL); - { - rb_thread_t *next_th = sched->running; - - if (next_th && next_th->nt == NULL) { - RUBY_DEBUG_LOG("nt:%d next_th:%d", (int)nt->serial, (int)next_th->serial); -#if USE_MN_THREADS - thread_sched_switch0(nt->nt_context, next_th, nt, false); - - // If a coroutine terminated during the transfer, co_start - // recorded it in nt->dead_co (switch0's return value is - // backend-dependent, unusable; see thread_pthread.h). - struct coroutine_context *dead_co = nt->dead_co; - nt->dead_co = NULL; - if (thread_sched_reclaim(dead_co)) { - // it already released the sched lock before its - // transfer (its Ractor may be gone): leave sched be. - locked = false; - } -#else - thread_sched_switch0(nt->nt_context, next_th, nt, false); -#endif - } - else { - RUBY_DEBUG_LOG("no schedulable threads -- next_th:%p", next_th); - } - } - if (locked) { - thread_sched_unlock(sched, NULL); - } - } - else { - // ractor_sched_deq retired this nt. - retired = true; - break; - } - - if (nt->dedicated) { - // SNT becomes DNT while running - break; - } - } - } - - if (in_snts) { - // Leaving the shared loop: every path back here deregistered first - // (park and death both precede the transfer), so only the snts entry - // is left to remove. - VM_ASSERT(nt->running_th == NULL); - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); - { - ccan_list_del_init(&nt->snts_node); - } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); - } - - if (retired) { - // The counts dropped this nt already; nothing can reference it now. - RUBY_DEBUG_LOG("retired nt:%u", nt->serial); - native_thread_destroy_self(nt); - } - - return NULL; -} - -static int native_thread_create_shared(rb_thread_t *th); - -#if USE_MN_THREADS -static void nt_free_stack(void *mstack); - - -// Reclaim the context a coroutine thread recorded in nt->dead_co before its -// final transfer (co_start's epilogue). Our running here proves that transfer's -// register save into the block completed. Returns true when a thread did -// terminate -- it RELEASED the sched lock before transferring; NULL/false means -// a live yield, where the loop still owns the lock. -static bool -thread_sched_reclaim(struct coroutine_context *dead_co) -{ - struct rb_thread_context *tctx = (struct rb_thread_context *)dead_co; - - if (tctx != NULL && tctx->dead) { - nt_free_stack(tctx->stack); - SIZED_FREE(tctx); - // pairs with the increment at the top of coroutine_thread_terminated: - // a waiting VM destruct may proceed once this reclaim is done - VM_ASSERT(RUBY_ATOMIC_LOAD(GET_VM()->ractor.sched.winding_cnt) > 0); - RUBY_ATOMIC_DEC(GET_VM()->ractor.sched.winding_cnt); - return true; - } - return false; -} -#endif - -void -rb_thread_wake_fence(rb_thread_t *th) -{ - timer_thread_wake_fence(th); -} - -void -rb_threadptr_sched_free(rb_thread_t *th) -{ - timer_thread_wake_fence(th); -#if USE_MN_THREADS - if (th->sched.malloc_stack) { - // has dedicated - SIZED_FREE_N((VALUE *)th->sched.context_stack, th->sched.context_stack_size); - native_thread_destroy(th->nt); - } - else if (th->sched.context != NULL) { - // a coroutine thread that never reached its epilogue (never started); - // a terminated one is reclaimed by whoever resumed from its final - // transfer (thread_sched_reclaim), and cleared this pointer. - struct rb_thread_context *tctx = (struct rb_thread_context *)th->sched.context; - nt_free_stack(tctx->stack); - SIZED_FREE(tctx); - th->sched.context = NULL; - // TODO: how to free nt and nt->altstack? - } -#else - SIZED_FREE_N((VALUE *)th->sched.context_stack, th->sched.context_stack_size); - native_thread_destroy(th->nt); -#endif - - th->nt = NULL; -} - - -static int -native_thread_create(rb_thread_t *th) -{ - VM_ASSERT(th->nt == 0); - RUBY_DEBUG_LOG("th:%d has_dnt:%d", th->serial, th->has_dedicated_nt); - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_STARTED, th); - if (!th->ractor->threads.sched.enable_mn_threads) { - th->has_dedicated_nt = 1; - } - if (th->has_dedicated_nt) { - return native_thread_create_dedicated(th); - } - else { - return native_thread_create_shared(th); - } -} #if USE_NATIVE_THREAD_PRIORITY @@ -2940,118 +874,27 @@ native_fd_select(int n, rb_fdset_t *readfds, rb_fdset_t *writefds, rb_fdset_t *e return rb_fd_select(n, readfds, writefds, exceptfds, timeout); } -#ifdef USE_UBF_LIST -static CCAN_LIST_HEAD(ubf_list_head); -static rb_nativethread_lock_t ubf_list_lock = RB_NATIVETHREAD_LOCK_INIT; - -static void -ubf_list_atfork(void) -{ - ccan_list_head_init(&ubf_list_head); - rb_native_mutex_initialize(&ubf_list_lock); -} - -RBIMPL_ATTR_MAYBE_UNUSED() -static bool -ubf_list_contain_p(rb_thread_t *th) -{ - rb_thread_t *list_th; - ccan_list_for_each(&ubf_list_head, list_th, sched.node.ubf) { - if (list_th == th) return true; - } - return false; -} - -/* The thread 'th' is registered to be trying unblock. */ -static void -register_ubf_list(rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); - struct ccan_list_node *node = &th->sched.node.ubf; - - VM_ASSERT(th->unblock.func != NULL); - - rb_native_mutex_lock(&ubf_list_lock); - { - // check not connected yet - if (ccan_list_empty((struct ccan_list_head*)node)) { - VM_ASSERT(!ubf_list_contain_p(th)); - ccan_list_add(&ubf_list_head, node); - } - } - rb_native_mutex_unlock(&ubf_list_lock); - - timer_thread_wakeup(); -} - -/* The thread 'th' is unblocked. It no longer need to be registered. */ -static void -unregister_ubf_list(rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); - struct ccan_list_node *node = &th->sched.node.ubf; - - /* we can't allow re-entry into ubf_list_head */ - VM_ASSERT(th->unblock.func == NULL); - - if (!ccan_list_empty((struct ccan_list_head*)node)) { - rb_native_mutex_lock(&ubf_list_lock); - { - VM_ASSERT(ubf_list_contain_p(th)); - ccan_list_del_init(node); - } - rb_native_mutex_unlock(&ubf_list_lock); - } -} - /* - * send a signal to intent that a target thread return from blocking syscall. - * Maybe any signal is ok, but we chose SIGVTALRM. + * Send a signal to make the target thread return from a blocking syscall. + * Maybe any signal is ok, but we chose SIGVTALRM (see null_func). */ static void -ubf_wakeup_thread(rb_thread_t *th) +native_thread_interrupt(rb_thread_t *th) { - RUBY_DEBUG_LOG("th:%u thread_id:%p", rb_th_serial(th), (void *)th->nt->thread_id); - pthread_kill(th->nt->thread_id, SIGVTALRM); } -static void -ubf_select(void *ptr) -{ - rb_thread_t *th = (rb_thread_t *)ptr; - RUBY_DEBUG_LOG("wakeup th:%u", rb_th_serial(th)); - ubf_wakeup_thread(th); - register_ubf_list(th); -} - -static bool -ubf_threads_empty(void) -{ - return ccan_list_empty(&ubf_list_head) != 0; -} - -static void -ubf_wakeup_all_threads(void) +static int +native_thread_default_max_cpu(void) { - rb_thread_t *th; - rb_native_mutex_lock(&ubf_list_lock); - { - ccan_list_for_each(&ubf_list_head, th, sched.node.ubf) { - ubf_wakeup_thread(th); - } - } - rb_native_mutex_unlock(&ubf_list_lock); +#if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN) + long nprocessors = sysconf(_SC_NPROCESSORS_ONLN); + return (nprocessors > 0) ? (int)nprocessors : 8; +#else + return 8; +#endif } -#else /* USE_UBF_LIST */ -#define register_ubf_list(th) (void)(th) -#define unregister_ubf_list(th) (void)(th) -#define ubf_select 0 -static void ubf_wakeup_all_threads(void) { return; } -static bool ubf_threads_empty(void) { return true; } -#define ubf_list_atfork() do {} while (0) -#endif /* USE_UBF_LIST */ #define TT_DEBUG 0 #define WRITE_CONST(fd, str) (void)(write((fd),(str),sizeof(str)-1)<0) @@ -3302,7 +1145,7 @@ static struct { #if USE_MN_THREADS /* Timed waiters, bucketed by deadline into a hierarchical timer wheel; * untimed (fd-only) waiters keep a plain list. Both under waiting_lock. - * The wheel operations all live in thread_pthread_mn.c (timer_wheel_*). */ + * The wheel operations all live in thread_sched_mn.c (timer_wheel_*). */ #define TIMER_WHEEL_LEVELS 4 #define TIMER_WHEEL_SLOT_BITS 6 #define TIMER_WHEEL_SLOTS (1 << TIMER_WHEEL_SLOT_BITS) @@ -3343,102 +1186,8 @@ static void timer_thread_check_timeslice(rb_vm_t *vm); static bool timeslice_scan(rb_vm_t *vm, bool interrupt); static int timer_thread_set_timeout(rb_vm_t *vm); -#include "thread_pthread_mn.c" - -static int -timer_thread_set_timeout(rb_vm_t *vm) -{ -#if 0 - return 10; // ms -#else - int timeout = -1; - - ractor_sched_lock(vm, NULL); - { - if ( timeslice_scan(vm, false) // (1-1) Provide time slice for active NTs - || !ubf_threads_empty() // (1-3) Periodic UBF - || vm->ractor.sched.grq_cnt > 0 // (1-4) Lazy GRQ deq start - ) { - - RUBY_DEBUG_LOG("ubf:%d grq:%d", - !ubf_threads_empty(), - (vm->ractor.sched.grq_cnt > 0)); - - timeout = 10; // ms - vm->ractor.sched.timeslice_wait_inf = false; - } - else { - vm->ractor.sched.timeslice_wait_inf = true; - } - } - ractor_sched_unlock(vm, NULL); - - timeout = timer_wheel_timeout(timeout); - - RUBY_DEBUG_LOG("timeout:%d inf:%d", timeout, (int)vm->ractor.sched.timeslice_wait_inf); - - // fprintf(stderr, "timeout:%d\n", timeout); - return timeout; -#endif -} - -static void -timer_thread_check_signal(rb_vm_t *vm) -{ - // ruby_sigchld_handler(vm); TODO - - int signum = rb_signal_buff_size(); - if (UNLIKELY(signum > 0) && vm->ractor.main_thread) { - RUBY_DEBUG_LOG("signum:%d", signum); - threadptr_trap_interrupt(vm->ractor.main_thread); - } -} - -// Tick (with `interrupt`) each listed sched's running thread and prune scheds -// whose readyq drained; returns whether any sched still needs ticks. -static bool -timeslice_scan(rb_vm_t *vm, bool interrupt) -{ - bool found = false; - struct rb_thread_sched *sched, *next; - - rb_native_mutex_lock(&vm->ractor.sched.timeslice.lock); - { - ccan_list_for_each_safe(&vm->ractor.sched.timeslice.scheds, sched, next, timeslice_node) { - // trylock: timeslice_sched_link nests sched.lock -> timeslice.lock, - // this scan holds the locks the other way around - if (rb_native_mutex_trylock(&sched->lock_) == 0) { - if (ccan_list_empty(&sched->readyq)) { - ccan_list_del_init(&sched->timeslice_node); // a later enq relinks it - } - else if (sched->is_running) { - VM_ASSERT(sched->running != NULL); - found = true; - if (interrupt) { - RUBY_DEBUG_LOG("timeslice th:%u", rb_th_serial(sched->running)); - RUBY_VM_SET_TIMER_INTERRUPT(sched->running->ec); - } - } - // else: waiters behind a blocked runner need no ticks; the - // add path wakes the timer when the sched runs again - rb_native_mutex_unlock(&sched->lock_); - } - else { - found = true; // busy switching; tick it on the next round - } - } - } - rb_native_mutex_unlock(&vm->ractor.sched.timeslice.lock); - - return found; -} +#include "thread_sched_mn.c" -static void -timer_thread_check_timeslice(rb_vm_t *vm) -{ - // TODO: check time - timeslice_scan(vm, true); -} void rb_assert_sig(void) @@ -3453,28 +1202,6 @@ rb_assert_sig(void) } } -static void * -timer_thread_func(void *ptr) -{ - rb_vm_t *vm = (rb_vm_t *)ptr; -#if defined(RUBY_NT_SERIAL) - ruby_nt_serial = (rb_atomic_t)-1; -#endif - - RUBY_DEBUG_LOG("started%s", ""); - - while (RUBY_ATOMIC_LOAD(system_working)) { - timer_thread_check_signal(vm); - timer_thread_check_timeout(vm); - ubf_wakeup_all_threads(); - - RUBY_DEBUG_LOG("system_working:%d", RUBY_ATOMIC_LOAD(system_working)); - timer_thread_polling(vm); - } - - RUBY_DEBUG_LOG("terminated"); - return NULL; -} /* only use signal-safe system calls here */ static void @@ -3517,34 +1244,6 @@ timer_thread_wakeup_force(void) signal_communication_pipe(timer_th.comm_fds[1]); } -static void -timer_thread_wakeup_locked(rb_vm_t *vm) -{ - // should be locked before. - ASSERT_ractor_sched_locked(vm, NULL); - - if (timer_th.created_fork_gen == current_fork_gen) { - if (vm->ractor.sched.timeslice_wait_inf) { - RUBY_DEBUG_LOG("wakeup with fd:%d", timer_th.comm_fds[1]); - timer_thread_wakeup_force(); - } - else { - RUBY_DEBUG_LOG("will be wakeup..."); - } - } -} - -static void -timer_thread_wakeup(void) -{ - rb_vm_t *vm = GET_VM(); - - ractor_sched_lock(vm, NULL); - { - timer_thread_wakeup_locked(vm); - } - ractor_sched_unlock(vm, NULL); -} static void rb_thread_create_timer_thread(void) @@ -3748,38 +1447,6 @@ ruby_ppoll(struct pollfd *fds, nfds_t nfds, # define ppoll(fds,nfds,ts,sigmask) ruby_ppoll((fds),(nfds),(ts),(sigmask)) #endif -static void -native_sleep(rb_thread_t *th, rb_hrtime_t *rel) -{ - struct rb_thread_sched *sched = TH_SCHED(th); - - RUBY_DEBUG_LOG("rel:%d", rel ? (int)*rel : 0); - - if (rel && !th_has_dedicated_nt(th)) { - // an M:N thread has no condvar of its own: the timer thread wakes it - thread_sched_wait_events(sched, th, -1, thread_sched_waiting_timeout, rel); - } - else if (rel) { - /* Solaris cond_timedwait() returns EINVAL if an argument is greater than - * current_time + 100,000,000. So cut up to 100,000,000. This is - * considered as a kind of spurious wakeup. The caller to native_sleep - * should care about spurious wakeup. - * - * See also [Bug #1341] [ruby-core:29702] - * http://download.oracle.com/docs/cd/E19683-01/816-0216/6m6ngupgv/index.html - */ - const rb_hrtime_t max = (rb_hrtime_t)100000000 * RB_HRTIME_PER_SEC; - if (*rel > max) *rel = max; - - rb_hrtime_t end = rb_hrtime_add(rb_hrtime_now(), *rel); - thread_sched_to_waiting_until_wakeup(sched, th, &end); - } - else { - thread_sched_to_waiting_until_wakeup(sched, th, NULL); - } - - RUBY_DEBUG_LOG("wakeup"); -} // fork read-write lock (only for pthread) static pthread_rwlock_t rb_thread_fork_rw_lock = PTHREAD_RWLOCK_INITIALIZER; @@ -3959,24 +1626,4 @@ rb_thread_execute_hooks(rb_event_flag_t event, rb_thread_t *th) } } -// return true if the current thread acquires DNT. -// return false if the current thread already acquires DNT. -bool -rb_thread_lock_native_thread(void) -{ - rb_thread_t *th = GET_THREAD(); - bool is_snt = th->nt->dedicated == 0; - native_thread_dedicated_inc(th->vm, th->ractor, th->nt); - - return is_snt; -} - -void -rb_thread_malloc_stack_set(rb_thread_t *th, void *stack, size_t stack_size) -{ - th->sched.malloc_stack = true; - th->sched.context_stack = stack; - th->sched.context_stack_size = stack_size; -} - #endif /* THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION */ diff --git a/thread_pthread.h b/thread_pthread.h index 040ae55be4697e..b8979be530cfbb 100644 --- a/thread_pthread.h +++ b/thread_pthread.h @@ -8,6 +8,9 @@ Copyright (C) 2004-2007 Koichi Sasada + This platform runs the common scheduler; see thread_sched.h for its data + structures and thread_sched.c for the implementation. + **********************************************************************/ #ifdef HAVE_PTHREAD_NP_H @@ -22,178 +25,7 @@ # define RB_THREAD_CURRENT_EC_NOINLINE #endif -// How a thread_sched_wait_events() wait ended. "unavailable" (could not be -// registered) is not "the event fired": the caller must fall back, not proceed. -enum thread_sched_wait_result { - thread_sched_wait_event, // an event the caller asked for fired - thread_sched_wait_timeout, // the timeout expired before any event - thread_sched_wait_unavailable, // not registered; the caller must fall back -}; - -// this data should be protected by timer_th.waiting_lock -struct rb_thread_sched_waiting { - enum thread_sched_waiting_flag { - thread_sched_waiting_none = 0x00, - thread_sched_waiting_timeout = 0x01, - thread_sched_waiting_io_read = 0x02, - thread_sched_waiting_io_write = 0x08, - thread_sched_waiting_io_force = 0x40, // ignore readable - } flags; - - struct { - // should be compat with hrtime.h -#ifdef MY_RUBY_BUILD_MAY_TIME_TRAVEL - int128_t timeout; -#else - uint64_t timeout; -#endif - uint32_t event_serial; - int fd; // -1 for timeout only - int result; - } data; - - // connected to a timer_th wheel slot (timed) or timer_th.waiting_untimed - struct ccan_list_node node; - - /* which wheel slot `node` is on; meaningful only while flags has - * thread_sched_waiting_timeout */ - uint8_t wheel_lvl; - uint8_t wheel_slot; - - // connected to rb_fd_waiters.waiters of data.fd - struct ccan_list_node fd_node; -}; - -// One entry per fd with waiters; fds stay dense, so a table indexed by fd fits. -// Entries live in fixed chunks: growing must not move a live list head. -struct rb_fd_waiters { - struct ccan_list_head waiters; // rb_thread_sched_waiting.fd_node - - // The io flags currently armed in epoll/kqueue for this fd: the union of - // what its waiters asked for. - uint32_t armed_flags; - - // Bumped on full disarm. Events carry the generation they were armed with, - // so one queued before the fd was disarmed (and reused) is recognised. - uint32_t generation; -}; - -// per-Thread scheduler helper data -struct rb_thread_sched_item { - struct { - struct ccan_list_node ubf; - - // connected to ractor->threads.sched.reqdyq - // locked by ractor->threads.sched.lock - struct ccan_list_node readyq; - // Indicates whether thread is on the readyq. - // There is no clear relationship between this and th->status. - bool is_ready; - - } node; - - struct rb_thread_sched_waiting waiting_reason; - uint32_t event_serial; - - // wakes pending on this thread (timer thread or an fd shard claim); - // under timer_th.wake_pending_lock - uint32_t wake_pending_cnt; - - // parked on its own condvar with a deadline; under the sched lock (see - // ubf_waiting). Always false for an M:N thread: its deadline lives on the - // timer wheel, and its early wake comes from the timer thread instead. - bool waiting_timed; - - bool malloc_stack; - void *context_stack; - size_t context_stack_size; - struct coroutine_context *context; -}; - -struct rb_native_thread { - rb_atomic_t serial; - struct rb_vm_struct *vm; - - rb_nativethread_id_t thread_id; - -#ifdef RB_THREAD_T_HAS_NATIVE_ID - int tid; -#endif - - struct rb_thread_struct *running_thread; - - // The running thread on this shared nt, for the barrier/timeslice scans. - // While a scan holds running_th_lock the thread cannot finish parking. - rb_nativethread_lock_t running_th_lock; - struct rb_thread_struct *running_th; - struct ccan_list_node snts_node; // in vm->ractor.sched.ntlist.snts - // in vm->ractor.sched.ntlist.running_dnts while running_thread runs - struct ccan_list_node running_dnts_node; - // barrier_serial stamped by the barrier's counting walk; this nt's - // deregistration during that barrier decrements the snapshot count - uint32_t barrier_counted_serial; - - // to control native thread; use sched->lock - rb_nativethread_cond_t readyq; - -#ifdef USE_SIGALTSTACK - void *altstack; -#endif - - struct coroutine_context *nt_context; - int dedicated; - - // set when this thread came back from a blocking region with no room left - // in the shared pool; it ends when it next asks for work - bool retiring; - - // A terminating coroutine records its context here before its final - // transfer; this nt's loop reclaims it. (Not via coroutine_transfer()'s - // return value: its meaning differs between the amd64 asm and ucontext.) - struct coroutine_context *dead_co; -}; - -#undef except -#undef try -#undef leave -#undef finally - -// per-Ractor -struct rb_thread_sched { - rb_nativethread_lock_t lock_; -#if VM_CHECK_MODE - struct rb_thread_struct *lock_owner; -#endif - struct rb_thread_struct *running; // running thread or NULL - // Most recently running thread or NULL. If this thread wakes up before the newly running - // thread completes the transfer of control, it can interrupt and resume running. - // The new thread clears this field when it takes control. - struct rb_thread_struct *runnable_hot_th; - int runnable_hot_th_waiting; - bool is_running; - - bool enable_mn_threads; - - struct ccan_list_head readyq; - int readyq_cnt; - // ractor scheduling - // When not linked in vm->ractor.sched.grq, this node is kept - // self-linked (ccan_list_node_init), so "linked?" can be read off the - // node itself: enqueuers assert it, and direct transfers cancel an - // outstanding entry (see ractor_sched_cancel_enq). - struct ccan_list_node grq_node; - struct ccan_list_node timeslice_node; // self-linked = not on timeslice.scheds -}; - -struct rb_thread_context; - -// A coroutine (M:N) thread's teardown runs coroutine_thread_terminated -// instead of the dedicated-thread path in thread_start_func_2; see the -// comments there and in thread_pthread_mn.c. th->sched.context is cleared in -// that epilogue, so this also reads as "did not tear down yet". -// (Only meaningful when USE_MN_THREADS -- gate uses accordingly; the macro -// itself is a plain pointer test and always compiles.) -#define th_has_coroutine(th) ((th)->sched.context != NULL) +#include "thread_sched.h" #ifdef RB_THREAD_LOCAL_SPECIFIER NOINLINE(void rb_current_ec_set(struct rb_execution_context_struct *)); @@ -228,9 +60,4 @@ native_tls_set(native_tls_key_t key, void *ptr) RUBY_EXTERN native_tls_key_t ruby_current_ec_key; #endif -struct rb_ractor_struct; -void rb_ractor_sched_wait(struct rb_execution_context_struct *ec, struct rb_ractor_struct *cr, rb_unblock_function_t *ubf, void *ptr); -void rb_ractor_sched_wakeup(struct rb_ractor_struct *r, struct rb_thread_struct *th); -void rb_thread_wake_fence(struct rb_thread_struct *th); - #endif /* RUBY_THREAD_PTHREAD_H */ diff --git a/thread_sched.c b/thread_sched.c new file mode 100644 index 00000000000000..a7d89cc1316eef --- /dev/null +++ b/thread_sched.c @@ -0,0 +1,2501 @@ +/* -*-c-*- */ +/********************************************************************** + + thread_sched.c - platform independent thread/ractor scheduler + + This file is #included from thread.c. It pulls in the platform + implementation (THREAD_IMPL_SRC: thread_pthread.c or thread_win32.c) + first, then builds the scheduler on top of the primitives that file + provides. It implements: + + - the per-Ractor thread scheduler (GVL): struct rb_thread_sched + - the Ractor scheduler: global ready queue (grq) and the VM barrier + - the native thread (NT) main loop + - the timer thread main loop and time slice management + - the unblocking function (UBF) list + + See thread_sched.h for the data structures and for the primitives the + platform layer has to supply. + +**********************************************************************/ + +/* ------------------------------------------------------------------------ + * The scheduler <-> platform contract. + * + * The platform implementation is included below, ahead of the scheduler + * body, so it can use every primitive it defines without declaring them. + * The traffic in the other direction -- the scheduler entry points the + * platform layer (and the M:N scheduler it includes) calls back into -- + * has to be declared here instead. + * ------------------------------------------------------------------------ */ + +#define thread_sched_dump(s) thread_sched_dump_(__FILE__, __LINE__, s) +#define ractor_sched_dump(s) ractor_sched_dump_(__FILE__, __LINE__, s) + +#define thread_sched_lock(a, b) thread_sched_lock_(a, b, __FILE__, __LINE__) +#define thread_sched_unlock(a, b) thread_sched_unlock_(a, b, __FILE__, __LINE__) +#define ractor_sched_lock(a, b) ractor_sched_lock_(a, b, __FILE__, __LINE__) +#define ractor_sched_unlock(a, b) ractor_sched_unlock_(a, b, __FILE__, __LINE__) + +#ifndef MINIMUM_SNT +// make at least MINIMUM_SNT snts for debug. +#define MINIMUM_SNT 0 +#endif + +struct coroutine_context; + +#include "probes.h" + +// thread.c +static void threadptr_trap_interrupt(rb_thread_t *); + +// thread scheduler (GVL) +static void thread_sched_to_running(struct rb_thread_sched *sched, rb_thread_t *th); +static void thread_sched_to_waiting(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately); +static void thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th); +static void coroutine_transfer0(struct coroutine_context *transfer_from, + struct coroutine_context *transfer_to, bool to_dead); +static void thread_sched_lock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line); +static void thread_sched_unlock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line); +static void thread_sched_unlock_no_log(struct rb_thread_sched *sched, rb_thread_t *th); +static void thread_sched_set_locked(struct rb_thread_sched *sched, rb_thread_t *th); +static void thread_sched_setup_running_threads(struct rb_thread_sched *sched, rb_ractor_t *cr, rb_vm_t *vm, + rb_thread_t *add_th, rb_thread_t *del_th); +static void thread_sched_add_running_thread(struct rb_thread_sched *sched, rb_thread_t *th); +static void thread_sched_to_ready(struct rb_thread_sched *sched, rb_thread_t *th); +static void thread_sched_to_ready_common(struct rb_thread_sched *sched, rb_thread_t *th, bool wakeup, bool will_switch); +static void thread_sched_to_dead_common(struct rb_thread_sched *sched, rb_thread_t *th); +static void thread_sched_to_waiting_until_wakeup(struct rb_thread_sched *sched, rb_thread_t *th, const rb_hrtime_t *end); +static void thread_sched_wait_running_turn(struct rb_thread_sched *sched, rb_thread_t *th, bool can_direct_transfer, const rb_hrtime_t *end); +static void thread_sched_wakeup_next_thread(struct rb_thread_sched *sched, rb_thread_t *th, bool will_switch); + +// native thread <-> ractor assignment +static void native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt); +static void native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt); +static void native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th); + +// ractor scheduler +static void ractor_sched_lock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line); +static void ractor_sched_unlock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line); +static void ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r); +static void ractor_sched_cancel_enq(rb_vm_t *vm, struct rb_thread_sched *sched); + +// VM wide scheduler state; the platform's Init_native_thread() calls this +static void thread_sched_init_vm(rb_vm_t *vm); + +// implemented by the M:N scheduler (thread_sched_mn.c) or stubbed out +static bool ractor_sched_timeout_arm(rb_thread_t *th, const rb_hrtime_t *rel); +static bool ractor_sched_timeout_disarm(rb_thread_t *th); +static void timer_thread_wake_fence(struct rb_thread_struct *th); + +// unblocking function (UBF) +static bool ubf_set(rb_thread_t *th, rb_unblock_function_t *func, void *arg, rb_atomic_t *event_serial); +static void ubf_clear(rb_thread_t *th, bool clear_serial); + +// native thread main loops +static void call_thread_start_func_2(rb_thread_t *th); +static void *nt_start(void *ptr); + +// timer thread +static void *timer_thread_func(void *ptr); +static int timer_thread_set_timeout(rb_vm_t *vm); +static void timer_thread_check_timeslice(rb_vm_t *vm); +static bool timeslice_scan(rb_vm_t *vm, bool interrupt); +static void timer_thread_wakeup(void); +static void timer_thread_wakeup_locked(rb_vm_t *vm); +static void timer_thread_wakeup_force(void); + +#include THREAD_IMPL_SRC + +// Defaults for what the platform above did not opt out of. + +#ifndef RB_NATIVE_MUTEX_TRYLOCK_DETECTS_SELF +// Whether rb_native_mutex_trylock() reports EBUSY when the calling thread is +// itself the owner. A recursive lock (a Windows CRITICAL_SECTION) grants it +// again instead, so it cannot back a "somebody holds this" assertion. +#define RB_NATIVE_MUTEX_TRYLOCK_DETECTS_SELF 1 +#endif + +/* ------------------------------------------------------------------------ + * The scheduler itself. + * ------------------------------------------------------------------------ */ + +static bool +th_has_dedicated_nt(const rb_thread_t *th) +{ + // TODO: th->has_dedicated_nt + return th->nt->dedicated > 0; +} + +RBIMPL_ATTR_MAYBE_UNUSED() +static void +thread_sched_dump_(const char *file, int line, struct rb_thread_sched *sched) +{ + fprintf(stderr, "@%s:%d running:%d\n", file, line, sched->running ? (int)sched->running->serial : -1); + rb_thread_t *th; + int i = 0; + ccan_list_for_each(&sched->readyq, th, sched.node.readyq) { + i++; if (i>10) rb_bug("too many"); + fprintf(stderr, " ready:%d (%sNT:%d)\n", th->serial, + th->nt ? (th->nt->dedicated ? "D" : "S") : "x", + th->nt ? (int)th->nt->serial : -1); + } +} + + +RBIMPL_ATTR_MAYBE_UNUSED() +static void +ractor_sched_dump_(const char *file, int line, rb_vm_t *vm) +{ + rb_ractor_t *r; + + fprintf(stderr, "ractor_sched_dump %s:%d\n", file, line); + + int i = 0; + ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) { + i++; + if (i>10) rb_bug("!!"); + fprintf(stderr, " %d ready:%d\n", i, rb_ractor_id(r)); + } +} + + +static void +thread_sched_set_locked(struct rb_thread_sched *sched, rb_thread_t *th) +{ +#if VM_CHECK_MODE > 0 + VM_ASSERT(sched->lock_owner == NULL); + + sched->lock_owner = th; +#endif +} + +static void +thread_sched_set_unlocked(struct rb_thread_sched *sched, rb_thread_t *th) +{ +#if VM_CHECK_MODE > 0 + VM_ASSERT(sched->lock_owner == th); + + sched->lock_owner = NULL; +#endif +} + +static void +thread_sched_lock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line) +{ + rb_native_mutex_lock(&sched->lock_); + +#if VM_CHECK_MODE + RUBY_DEBUG_LOG2(file, line, "r:%d th:%u", th ? (int)rb_ractor_id(th->ractor) : -1, rb_th_serial(th)); +#else + RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th)); +#endif + + thread_sched_set_locked(sched, th); +} + +static void +thread_sched_unlock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line) +{ + RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th)); + + thread_sched_set_unlocked(sched, th); + + rb_native_mutex_unlock(&sched->lock_); +} + +// Like thread_sched_unlock(), but never dereferences th (the debug log above +// reads th->serial). For the MN termination epilogue, which unlocks after th +// may already be collectable. Keep in sync with thread_sched_unlock_. +RBIMPL_ATTR_MAYBE_UNUSED() +static void +thread_sched_unlock_no_log(struct rb_thread_sched *sched, rb_thread_t *th) +{ + thread_sched_set_unlocked(sched, th); // pointer compare only + + rb_native_mutex_unlock(&sched->lock_); +} + +static void +ASSERT_thread_sched_locked(struct rb_thread_sched *sched, rb_thread_t *th) +{ +#if RB_NATIVE_MUTEX_TRYLOCK_DETECTS_SELF + VM_ASSERT(rb_native_mutex_trylock(&sched->lock_) == EBUSY); +#endif + +#if VM_CHECK_MODE + if (th) { + VM_ASSERT(sched->lock_owner == th); + } + else { + VM_ASSERT(sched->lock_owner != NULL); + } +#endif +} + + +RBIMPL_ATTR_MAYBE_UNUSED() +static unsigned int +rb_ractor_serial(const rb_ractor_t *r) +{ + if (r) { + return rb_ractor_id(r); + } + else { + return 0; + } +} + +static void +ractor_sched_set_locked(rb_vm_t *vm, rb_ractor_t *cr) +{ +#if VM_CHECK_MODE > 0 + VM_ASSERT(vm->ractor.sched.lock_owner == NULL); + VM_ASSERT(vm->ractor.sched.locked == false); + + vm->ractor.sched.lock_owner = cr; + vm->ractor.sched.locked = true; +#endif +} + +static void +ractor_sched_set_unlocked(rb_vm_t *vm, rb_ractor_t *cr) +{ +#if VM_CHECK_MODE > 0 + VM_ASSERT(vm->ractor.sched.locked); + VM_ASSERT(vm->ractor.sched.lock_owner == cr); + + vm->ractor.sched.locked = false; + vm->ractor.sched.lock_owner = NULL; +#endif +} + + +static void +ractor_sched_lock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line) +{ + rb_native_mutex_lock(&vm->ractor.sched.lock); + +#if VM_CHECK_MODE + RUBY_DEBUG_LOG2(file, line, "cr:%u prev_owner:%u", rb_ractor_serial(cr), rb_ractor_serial(vm->ractor.sched.lock_owner)); +#else + RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr)); +#endif + + ractor_sched_set_locked(vm, cr); +} + +static void +ractor_sched_unlock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line) +{ + RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr)); + + ractor_sched_set_unlocked(vm, cr); + rb_native_mutex_unlock(&vm->ractor.sched.lock); +} + +static void +ASSERT_ractor_sched_locked(rb_vm_t *vm, rb_ractor_t *cr) +{ +#if RB_NATIVE_MUTEX_TRYLOCK_DETECTS_SELF + VM_ASSERT(rb_native_mutex_trylock(&vm->ractor.sched.lock) == EBUSY); +#endif + VM_ASSERT(vm->ractor.sched.locked); + VM_ASSERT(cr == NULL || vm->ractor.sched.lock_owner == cr); +} + +static void ractor_sched_barrier_join_signal_locked(rb_vm_t *vm); + +/* ntlist registration: a thread that executes Ruby code is always registered, + * in its snt's nt->running_th or on running_dnts via its dedicated nt. The + * only unregistered execution is scheduler glue (parking, resuming), which + * touches no Ruby heap, and the barrier wait below. */ +static void +ntlist_add_running(rb_vm_t *vm, rb_thread_t *th) +{ + struct rb_native_thread *nt = th->nt; + + // a dedicated nt is not on the snts list the scans walk: running_dnts instead + if (nt != NULL && nt->dedicated == 0) { + rb_native_mutex_lock(&nt->running_th_lock); + { + VM_ASSERT(nt->running_th == NULL); + nt->running_th = th; + } + rb_native_mutex_unlock(&nt->running_th_lock); + } + else { + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + // an snt gone dedicated (rb_thread_lock_native_thread) has no + // creation-time running_thread: the registration supplies it + nt->running_thread = th; + ccan_list_add(&vm->ractor.sched.ntlist.running_dnts, &nt->running_dnts_node); + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); + } +} + +// Returns whether the active barrier's walk had counted this registration: +// such a deregistration owes the snapshot count a decrement. Read and +// cleared under the registration's own lock, so it pairs with the walk. +static bool +ntlist_del_running(rb_vm_t *vm, rb_thread_t *th) +{ + struct rb_native_thread *nt = th->nt; + uint32_t serial; + bool counted; + bool in_running_th; + + // The registration itself says where it is: nt->running_th holds th, or + // th's nt hangs on running_dnts. barrier_serial is read inside the + // registration's lock, ordered with the walk that stamped there. + rb_native_mutex_lock(&nt->running_th_lock); + { + in_running_th = (nt->running_th == th); + if (in_running_th) { + nt->running_th = NULL; + serial = vm->ractor.sched.barrier_serial; + counted = (nt->barrier_counted_serial == serial); + nt->barrier_counted_serial = serial - 1; // only once per barrier + } + } + rb_native_mutex_unlock(&nt->running_th_lock); + + if (!in_running_th) { + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + ccan_list_del_init(&nt->running_dnts_node); + serial = vm->ractor.sched.barrier_serial; + counted = (nt->barrier_counted_serial == serial); + nt->barrier_counted_serial = serial - 1; + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); + } + return counted; +} + +// Stamp a registration into the active barrier's snapshot unless the walk +// already counted it; returns whether it stamped. Called under sched.lock, +// so it is serialized with the walk: the stamp says exactly whether the +// registration came first. +static bool +ntlist_stamp_if_uncounted(rb_vm_t *vm, rb_thread_t *th) +{ + struct rb_native_thread *nt = th->nt; + uint32_t serial = vm->ractor.sched.barrier_serial; // sched.lock is held + bool stamped; + bool in_running_th; + + rb_native_mutex_lock(&nt->running_th_lock); + { + in_running_th = (nt->running_th == th); + if (in_running_th) { + stamped = (nt->barrier_counted_serial != serial); + nt->barrier_counted_serial = serial; + } + } + rb_native_mutex_unlock(&nt->running_th_lock); + + if (!in_running_th) { + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + stamped = (nt->barrier_counted_serial != serial); + nt->barrier_counted_serial = serial; + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); + } + return stamped; +} + +// Record a thread entering/leaving the running set, with no global lock and +// no count: the records themselves are what the barrier counts. Pairing: +// the barrier sets barrier_is_waiting and then walks the records under their +// locks; we move a record and then read the flag, so one side sees the other. +// List sched for the timer's timeslice ticks. The caller holds sched->lock_ +// with the readyq non-empty, so the timer cannot prune the entry meanwhile. +static void +timeslice_sched_link(rb_vm_t *vm, struct rb_thread_sched *sched) +{ + rb_native_mutex_lock(&vm->ractor.sched.timeslice.lock); + { + if (sched->timeslice_node.next == &sched->timeslice_node) { + ccan_list_add_tail(&vm->ractor.sched.timeslice.scheds, &sched->timeslice_node); + } + } + rb_native_mutex_unlock(&vm->ractor.sched.timeslice.lock); +} + +static void +thread_sched_setup_running_threads(struct rb_thread_sched *sched, rb_ractor_t *cr, rb_vm_t *vm, + rb_thread_t *add_th, rb_thread_t *del_th) +{ + RUBY_DEBUG_LOG("+:%u -:%u", rb_th_serial(add_th), rb_th_serial(del_th)); + + if (del_th) { + bool counted = ntlist_del_running(vm, del_th); + sched->is_running = false; + + // The first load is only a filter; the one under sched.lock decides. + // A missed flag means this deregistration preceded the barrier's walk. + if (UNLIKELY(RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting))) { + ractor_sched_lock(vm, cr); + { + if (RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting)) { + if (counted) { + VM_ASSERT(vm->ractor.sched.barrier_running_cnt > 0); + vm->ractor.sched.barrier_running_cnt--; + } + ractor_sched_barrier_join_signal_locked(vm); + } + } + ractor_sched_unlock(vm, cr); + } + } + + if (add_th) { + ntlist_add_running(vm, add_th); + + if (UNLIKELY(RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting))) { + // A stop-the-world section. In its waiting phase sched.lock is + // takable: join the snapshot count and take the interrupt (this + // thread joins at its next check, like any walked runner). In + // the GC phase the barrier holds sched.lock to its end, so this + // blocks here, as the old global-lock design did. + ractor_sched_lock(vm, cr); + { + if (RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting) && + ntlist_stamp_if_uncounted(vm, add_th)) { + // the walk ran before this registration; count it in + RUBY_DEBUG_LOG("barrier_is_waiting"); + vm->ractor.sched.barrier_running_cnt++; + RUBY_VM_SET_VM_BARRIER_INTERRUPT(add_th->ec); + } + } + ractor_sched_unlock(vm, cr); + } + + sched->is_running = true; + + // taking a turn with waiters already queued needs the timeslice ticks + if (!ccan_list_empty(&sched->readyq)) { + timeslice_sched_link(vm, sched); + ractor_sched_lock(vm, cr); + { + if (vm->ractor.sched.timeslice_wait_inf) { + timer_thread_wakeup_locked(vm); + } + } + ractor_sched_unlock(vm, cr); + } + } +} + +static void +thread_sched_add_running_thread(struct rb_thread_sched *sched, rb_thread_t *th) +{ + ASSERT_thread_sched_locked(sched, th); + VM_ASSERT(sched->running == th); + + rb_vm_t *vm = th->vm; + thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL); +} + +static void +thread_sched_del_running_thread(struct rb_thread_sched *sched, rb_thread_t *th) +{ + ASSERT_thread_sched_locked(sched, th); + + rb_vm_t *vm = th->vm; + thread_sched_setup_running_threads(sched, th->ractor, vm, NULL, th); +} + +void +rb_add_running_thread(rb_thread_t *th) +{ + struct rb_thread_sched *sched = TH_SCHED(th); + + thread_sched_lock(sched, th); + { + thread_sched_add_running_thread(sched, th); + } + thread_sched_unlock(sched, th); +} + +void +rb_del_running_thread(rb_thread_t *th) +{ + struct rb_thread_sched *sched = TH_SCHED(th); + + thread_sched_lock(sched, th); + { + thread_sched_del_running_thread(sched, th); + } + thread_sched_unlock(sched, th); +} + +// setup current or next running thread +// sched->running should be set only on this function. +// +// if th is NULL, there is no running threads. +static void +thread_sched_set_running(struct rb_thread_sched *sched, rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u->th:%u", rb_th_serial(sched->running), rb_th_serial(th)); + VM_ASSERT(sched->running != th); + + if (RUBY_DTRACE_RTS_SET_RUNNING_ENABLED()) { + RUBY_DTRACE_RTS_SET_RUNNING(sched, sched->running, th); + } + + sched->running = th; +} + +RBIMPL_ATTR_MAYBE_UNUSED() +static bool +thread_sched_readyq_contain_p(struct rb_thread_sched *sched, rb_thread_t *th) +{ + rb_thread_t *rth; + ccan_list_for_each(&sched->readyq, rth, sched.node.readyq) { + if (rth == th) { + VM_ASSERT(th->sched.node.is_ready); + return true; + } + } + VM_ASSERT(!th->sched.node.is_ready); + return false; +} + +// deque thread from the ready queue. +// if the ready queue is empty, return NULL. +// +// return deque'ed running thread (or NULL). +static rb_thread_t * +thread_sched_deq(struct rb_thread_sched *sched) +{ + ASSERT_thread_sched_locked(sched, NULL); + rb_thread_t *next_th; + + VM_ASSERT(sched->running != NULL); + + if (ccan_list_empty(&sched->readyq)) { + next_th = NULL; + } + else { + next_th = ccan_list_pop(&sched->readyq, rb_thread_t, sched.node.readyq); + VM_ASSERT(next_th->sched.node.is_ready); + next_th->sched.node.is_ready = false; + + VM_ASSERT(sched->readyq_cnt > 0); + sched->readyq_cnt--; + ccan_list_node_init(&next_th->sched.node.readyq); + } + + RUBY_DEBUG_LOG("next_th:%u readyq_cnt:%d", rb_th_serial(next_th), sched->readyq_cnt); + + return next_th; +} + +// enqueue ready thread to the ready queue. +static void +thread_sched_enq(struct rb_thread_sched *sched, rb_thread_t *ready_th) +{ + ASSERT_thread_sched_locked(sched, NULL); + RUBY_DEBUG_LOG("ready_th:%u readyq_cnt:%d", rb_th_serial(ready_th), sched->readyq_cnt); + + VM_ASSERT(sched->running != NULL); + VM_ASSERT(!thread_sched_readyq_contain_p(sched, ready_th)); + + bool timeslice_onset = sched->is_running && ccan_list_empty(&sched->readyq); + + ccan_list_add_tail(&sched->readyq, &ready_th->sched.node.readyq); + ready_th->sched.node.is_ready = true; + sched->readyq_cnt++; + + if (timeslice_onset) { + // The running thread needs the timeslice ticks now. Linked before + // the check under sched.lock: either the timer's scan (same lock) + // sees the sched, or this sees timeslice_wait_inf. + rb_vm_t *vm = ready_th->vm; + timeslice_sched_link(vm, sched); + ractor_sched_lock(vm, NULL); + { + if (vm->ractor.sched.timeslice_wait_inf) { + timer_thread_wakeup_locked(vm); + } + } + ractor_sched_unlock(vm, NULL); + } +} + +// DNT: kick condvar +// SNT: TODO +static void +thread_sched_wakeup_running_thread(struct rb_thread_sched *sched, rb_thread_t *next_th, bool will_switch) +{ + ASSERT_thread_sched_locked(sched, NULL); + VM_ASSERT(sched->running == next_th); + + if (next_th) { + if (next_th->nt) { + if (th_has_dedicated_nt(next_th)) { + RUBY_DEBUG_LOG("pinning th:%u", next_th->serial); + rb_native_cond_signal(&next_th->nt->readyq); + } + else { + // TODO + RUBY_DEBUG_LOG("th:%u is already running.", next_th->serial); + } + } + else { + if (will_switch) { + RUBY_DEBUG_LOG("th:%u (do nothing)", rb_th_serial(next_th)); + } + else { + RUBY_DEBUG_LOG("th:%u (enq)", rb_th_serial(next_th)); + ractor_sched_enq(next_th->vm, next_th->ractor); + } + } + } + else { + RUBY_DEBUG_LOG("no waiting threads%s", ""); + } +} + +// waiting -> ready (locked) +static void +thread_sched_to_ready_common(struct rb_thread_sched *sched, rb_thread_t *th, bool wakeup, bool will_switch) +{ + RUBY_DEBUG_LOG("th:%u running:%u redyq_cnt:%d", rb_th_serial(th), rb_th_serial(sched->running), sched->readyq_cnt); + + VM_ASSERT(sched->running != th); + VM_ASSERT(!thread_sched_readyq_contain_p(sched, th)); + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_READY, th); + + if (sched->running == NULL) { + thread_sched_set_running(sched, th); + if (wakeup) thread_sched_wakeup_running_thread(sched, th, will_switch); + } + else { + thread_sched_enq(sched, th); + } +} + +// waiting -> ready +// +// `th` had became "waiting" state by `thread_sched_to_waiting` +// and `thread_sched_to_ready` enqueue `th` to the thread ready queue. +RBIMPL_ATTR_MAYBE_UNUSED() +static void +thread_sched_to_ready(struct rb_thread_sched *sched, rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); + + thread_sched_lock(sched, th); + { + thread_sched_to_ready_common(sched, th, true, false); + } + thread_sched_unlock(sched, th); +} + +// wait until sched->running is `th`. `end` is an absolute deadline for a dedicated +static void +thread_sched_wait_running_turn(struct rb_thread_sched *sched, rb_thread_t *th, bool can_direct_transfer, const rb_hrtime_t *end) +{ + RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); + + ASSERT_thread_sched_locked(sched, th); + VM_ASSERT(th == rb_ec_thread_ptr(rb_current_ec_noinline())); + + bool timedout = false; + + if (th != sched->running) { + // TODO: This optimization should also be made to work for MN_THREADS + if (th->has_dedicated_nt && th == sched->runnable_hot_th && (sched->running == NULL || sched->running->has_dedicated_nt)) { + RUBY_DEBUG_LOG("(nt) stealing: hot-th:%u. running:%u", rb_th_serial(th), rb_th_serial(sched->running)); + + // th serves itself on its own nt, displacing the enqueued + // running thread back to the readyq: cancel the entry that was + // posted for it (a later dequeue would find this Ractor served + // and its next enqueue would double-list the node) + ractor_sched_cancel_enq(th->vm, sched); + + // If there is a thread set to run, move it back to the front of the readyq + if (sched->running != NULL) { + rb_thread_t *running = sched->running; + VM_ASSERT(!thread_sched_readyq_contain_p(sched, running)); + running->sched.node.is_ready = true; + ccan_list_add(&sched->readyq, &running->sched.node.readyq); + sched->readyq_cnt++; + } + + // Pull off the ready queue and start running. + if (th->sched.node.is_ready) { + VM_ASSERT(thread_sched_readyq_contain_p(sched, th)); + ccan_list_del_init(&th->sched.node.readyq); + th->sched.node.is_ready = false; + sched->readyq_cnt--; + } + thread_sched_set_running(sched, th); + rb_ractor_thread_switch(th->ractor, th, false); + } + else if (th == sched->runnable_hot_th) { + // The hot thread cannot steal the control (e.g. the running thread + // is an MN thread). It is going to sleep, so it is no longer spinning; + // drop the hint so that other threads don't yield the lock to it. + sched->runnable_hot_th = NULL; + sched->runnable_hot_th_waiting = 0; + } + + // already deleted from running threads + + + // wait for execution right + rb_thread_t *next_th; + while((next_th = sched->running) != th) { + if (th_has_dedicated_nt(th)) { + RUBY_DEBUG_LOG("(nt) sleep th:%u running:%u", rb_th_serial(th), rb_th_serial(sched->running)); + + thread_sched_set_unlocked(sched, th); + { + RUBY_DEBUG_LOG("nt:%d cond:%p", th->nt->serial, &th->nt->readyq); + rb_nativethread_cond_t *cond = &th->nt->readyq; + + // Once someone has queued this thread the deadline is spent: it + // is waiting for a turn, not for the time, and arming a kernel + // timer for every round of that costs more than the wait. + // Once someone has queued this thread the deadline is spent: it + // is waiting for a turn, not for the time, and arming a kernel + // timer for every round of that costs more than the wait. + if (end && !th->sched.node.is_ready) { + rb_hrtime_t abs = *end; + + if (!RB_NATIVE_COND_HRTIME_DEADLINE_P()) { + // the condvar counts in another clock: restate it there + rb_hrtime_t now = rb_hrtime_now(); + abs = native_cond_timeout(cond, *end > now ? *end - now : 0); + } + timedout = native_cond_timedwait(cond, &sched->lock_, &abs) == ETIMEDOUT; + } + else { + rb_native_cond_wait(cond, &sched->lock_); + } + } + thread_sched_set_locked(sched, th); + + if (timedout && + sched->running != th && !th->sched.node.is_ready) { + // the deadline passed and nobody woke this thread: get back in + // line for the running turn, then wait for it without a deadline + thread_sched_to_ready_common(sched, th, false, false); + end = NULL; + } + + if (sched->runnable_hot_th != NULL && sched->runnable_hot_th_waiting) { + VM_ASSERT(sched->runnable_hot_th != th); + // Give the hot thread a chance to preempt, if it's actively spinning. + // On multicore, this reduces the rate of core-switching. On single-core it + // should mostly be a nop, since the other thread can't be concurrently spinning. + thread_sched_unlock(sched, th); + thread_sched_lock(sched, th); + } + + RUBY_DEBUG_LOG("(nt) wakeup %s", sched->running == th ? "success" : "failed"); + if (th == sched->running) { + rb_ractor_thread_switch(th->ractor, th, false); + } + } + else { + // search another ready thread + if (can_direct_transfer && + (next_th = sched->running) != NULL && + !next_th->nt // next_th is running or has dedicated nt + ) { + + RUBY_DEBUG_LOG("th:%u->%u (direct)", rb_th_serial(th), rb_th_serial(next_th)); + + thread_sched_set_unlocked(sched, th); + { + rb_ractor_set_current_ec(th->ractor, NULL); + thread_sched_switch(th, next_th); + } + thread_sched_set_locked(sched, th); + } + else { + // search another ready ractor + struct rb_native_thread *nt = th->nt; + native_thread_assign(NULL, th); + + RUBY_DEBUG_LOG("th:%u->%u (ractor scheduling)", rb_th_serial(th), rb_th_serial(next_th)); + + thread_sched_set_unlocked(sched, th); + { + rb_ractor_set_current_ec(th->ractor, NULL); + coroutine_transfer0(th->sched.context, nt->nt_context, false); + } + thread_sched_set_locked(sched, th); + } + + VM_ASSERT(rb_current_ec_noinline() == th->ec); + } + } + + VM_ASSERT(th->nt != NULL); + VM_ASSERT(rb_current_ec_noinline() == th->ec); + VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none); + + // add th to running threads + thread_sched_add_running_thread(sched, th); + } + + // Control transfer to the current thread is now complete. The original thread + // cannot steal control at this point. + sched->runnable_hot_th = NULL; + sched->runnable_hot_th_waiting = 0; + + + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_RESUMED, th); +} + +// waiting -> ready -> running (locked) +static void +thread_sched_to_running_common(struct rb_thread_sched *sched, rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u dedicated:%d", rb_th_serial(th), th_has_dedicated_nt(th)); + + VM_ASSERT(sched->running != th); + VM_ASSERT(th_has_dedicated_nt(th)); + VM_ASSERT(GET_THREAD() == th); + + native_thread_dedicated_dec(th->vm, th->ractor, th->nt); + + // waiting -> ready + thread_sched_to_ready_common(sched, th, false, false); + + if (sched->running == th) { + thread_sched_add_running_thread(sched, th); + } + + // TODO: check SNT number + thread_sched_wait_running_turn(sched, th, false, NULL); +} + +// waiting -> ready -> running +// +// `th` had been waiting by `thread_sched_to_waiting()` +// and run a dedicated task (like waitpid and so on). +// After the dedicated task, this function is called +// to join a normal thread-scheduling. +static void +thread_sched_to_running(struct rb_thread_sched *sched, rb_thread_t *th) +{ + // We are reading and writing these sched fields without lock cover, but + // there are no correctness issues resulting from stale cache or delayed writeback. + // When it works, this causes the next-scheduled thread to yield the sched lock + // briefly so that we can grab it if we're still spinning (not descheduled yet). + if (sched->runnable_hot_th == th) { + sched->runnable_hot_th_waiting = 1; + } + thread_sched_lock(sched, th); + { + thread_sched_to_running_common(sched, th); + } + thread_sched_unlock(sched, th); +} + +// resume a next thread in the thread ready queue. +// +// deque next running thread from the ready thread queue and +// resume this thread if available. +// +// If the next therad has a dedicated native thraed, simply signal to resume. +// Otherwise, make the ractor ready and other nt will run the ractor and the thread. +static void +thread_sched_wakeup_next_thread(struct rb_thread_sched *sched, rb_thread_t *th, bool will_switch) +{ + ASSERT_thread_sched_locked(sched, th); + + VM_ASSERT(sched->running == th); + VM_ASSERT(sched->running->nt != NULL); + + rb_thread_t *next_th = thread_sched_deq(sched); + + RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th)); + VM_ASSERT(th != next_th); + + thread_sched_set_running(sched, next_th); + VM_ASSERT(next_th == sched->running); + thread_sched_wakeup_running_thread(sched, next_th, will_switch); + + if (th != next_th) { + thread_sched_del_running_thread(sched, th); + } +} + +// running -> dead (locked) +static void +thread_sched_to_dead_common(struct rb_thread_sched *sched, rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated); + + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); + + // A dying coroutine thread (will_switch=true here) does NOT wake the + // next thread now: it is still winding down (co_start's epilogue), and + // the same Ractor must not have two threads executing at once. The + // epilogue enqueues the Ractor after its last rb_ractor_t access. + thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th)); + + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_EXITED, th); +} + +// running -> dead +static void +thread_sched_to_dead(struct rb_thread_sched *sched, rb_thread_t *th) +{ + // wait out any pending wake here, while th's Ractor is still alive + timer_thread_wake_fence(th); + + thread_sched_lock(sched, th); + { + thread_sched_to_dead_common(sched, th); + } + thread_sched_unlock(sched, th); +} + +// running -> waiting (locked) +// +// This thread will run dedicated task (th->nt->dedicated++). +static void +thread_sched_to_waiting_common(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately) +{ + RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated); + + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); + + native_thread_dedicated_inc(th->vm, th->ractor, th->nt); + if (!yield_immediately) { + sched->runnable_hot_th = th; + sched->runnable_hot_th_waiting = 0; + } + thread_sched_wakeup_next_thread(sched, th, false); +} + +// running -> waiting +// +// This thread will run a dedicated task. +static void +thread_sched_to_waiting(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately) +{ + thread_sched_lock(sched, th); + { + thread_sched_to_waiting_common(sched, th, yield_immediately); + } + thread_sched_unlock(sched, th); +} + +// mini utility func +// return true if any there are any interrupts +static bool +ubf_set(rb_thread_t *th, rb_unblock_function_t *func, void *arg, rb_atomic_t *event_serial) +{ + VM_ASSERT(func != NULL); + + retry: + if (RUBY_VM_INTERRUPTED(th->ec)) { + RUBY_DEBUG_LOG("interrupted:0x%x", th->ec->interrupt_flag); + return true; + } + + rb_native_mutex_lock(&th->interrupt_lock); + { + if (!th->ec->raised_flag && RUBY_VM_INTERRUPTED(th->ec)) { + rb_native_mutex_unlock(&th->interrupt_lock); + goto retry; + } + + VM_ASSERT(th->unblock.func == NULL); + th->unblock.func = func; + th->unblock.arg = arg; + if (event_serial) { + rb_atomic_t prev_serial = RUBY_ATOMIC_FETCH_ADD(th->unblock.event_serial, 1); + *event_serial = prev_serial+1; + } + } + rb_native_mutex_unlock(&th->interrupt_lock); + + return false; +} + +static void +ubf_clear(rb_thread_t *th, bool clear_serial) +{ + rb_native_mutex_lock(&th->interrupt_lock); + { + th->unblock.func = NULL; + th->unblock.arg = NULL; + if (clear_serial) { + RUBY_ATOMIC_ADD(th->unblock.event_serial, 1); + } + } + rb_native_mutex_unlock(&th->interrupt_lock); +} + +static void +ubf_waiting(void *ptr) +{ + rb_thread_t *th = (rb_thread_t *)ptr; + struct rb_thread_sched *sched = TH_SCHED(th); + + // only once. it is safe because th->interrupt_lock is already acquired. + th->unblock.func = NULL; + th->unblock.arg = NULL; + + RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); + + thread_sched_lock(sched, th); + { + if (sched->running == th || th->sched.node.is_ready) { + // not sleeping yet, or a deadline already put it back in line + } + else { + thread_sched_to_ready_common(sched, th, true, false); + + // If the turn is taken, th stays parked until the running thread yields. + // For a timed wait, wake it early anyway: it re-parks at once, but its + // wakeup then runs on another core in parallel with the running thread, + // off the handoff path. An untimed wait has no post-wake bookkeeping + // worth pipelining, so it skips the extra futex round. + if (sched->running != th && th->sched.waiting_timed && + th->nt != NULL && th_has_dedicated_nt(th)) { + rb_native_cond_signal(&th->nt->readyq); + } + } + } + thread_sched_unlock(sched, th); +} + +// running -> waiting +// +// This thread will sleep until other thread wakeup the thread. `end` is an +// absolute deadline, NULL to sleep until woken; only a dedicated native thread, +// which parks on its own condvar, can take one. +static void +thread_sched_to_waiting_until_wakeup(struct rb_thread_sched *sched, rb_thread_t *th, const rb_hrtime_t *end) +{ + RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); + + VM_ASSERT(end == NULL || th_has_dedicated_nt(th)); + + RB_VM_SAVE_MACHINE_CONTEXT(th); + + + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); + + thread_sched_lock(sched, th); + { + // NOTE: there's a lock ordering inversion here with the ubf call, but it's benign. + if (ubf_set(th, ubf_waiting, (void *)th, NULL)) { + RUBY_DEBUG_LOG("th:%u interrupted", rb_th_serial(th)); + } + else { + bool can_direct_transfer = !th_has_dedicated_nt(th); + th->sched.waiting_timed = (end != NULL); // never true here for M:N (end is NULL) + // NOTE: th->status is set before and after this sleep outside of this function in `sleep_forever` + thread_sched_wakeup_next_thread(sched, th, can_direct_transfer); + thread_sched_wait_running_turn(sched, th, can_direct_transfer, end); + th->sched.waiting_timed = false; + } + } + thread_sched_unlock(sched, th); + + ubf_clear(th, false); +} + +// run another thread in the ready queue. +// continue to run if there are no ready threads. +static void +thread_sched_yield(struct rb_thread_sched *sched, rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%d sched->readyq_cnt:%d", (int)th->serial, sched->readyq_cnt); + + thread_sched_lock(sched, th); + { + if (!ccan_list_empty(&sched->readyq)) { + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); + thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th)); + bool can_direct_transfer = !th_has_dedicated_nt(th); + thread_sched_to_ready_common(sched, th, false, can_direct_transfer); + thread_sched_wait_running_turn(sched, th, can_direct_transfer, NULL); + th->status = THREAD_RUNNABLE; + } + else { + VM_ASSERT(sched->readyq_cnt == 0); + } + } + thread_sched_unlock(sched, th); +} + +void +rb_thread_sched_init(struct rb_thread_sched *sched, bool atfork) +{ + rb_native_mutex_initialize(&sched->lock_); + +#if VM_CHECK_MODE + sched->lock_owner = NULL; +#endif + + ccan_list_head_init(&sched->readyq); + sched->readyq_cnt = 0; + ccan_list_node_init(&sched->grq_node); // self-linked = not enqueued + ccan_list_node_init(&sched->timeslice_node); + +#if USE_MN_THREADS + if (!atfork) sched->enable_mn_threads = true; // MN is enabled on Ractors +#endif +} + +static void +coroutine_transfer0(struct coroutine_context *transfer_from, struct coroutine_context *transfer_to, bool to_dead) +{ +#ifdef RUBY_ASAN_ENABLED + void **fake_stack = to_dead ? NULL : &transfer_from->fake_stack; + __sanitizer_start_switch_fiber(fake_stack, transfer_to->stack_base, transfer_to->stack_size); +#endif + +#if defined(COROUTINE_SANITIZE_THREAD) + /* Tell TSan we are switching to transfer_to's fiber before the stack + * switch, so its per-thread shadow stack stays bound to the right + * coroutine. */ + __tsan_switch_to_fiber(transfer_to->tsan_fiber, 0); +#endif + + RBIMPL_ATTR_MAYBE_UNUSED() + struct coroutine_context *returning_from = coroutine_transfer(transfer_from, transfer_to); + + /* if to_dead was passed, the caller is promising that this coroutine is finished and it should + * never be resumed! */ + VM_ASSERT(!to_dead); +#ifdef RUBY_ASAN_ENABLED + __sanitizer_finish_switch_fiber(transfer_from->fake_stack, + (const void**)&returning_from->stack_base, &returning_from->stack_size); +#endif +} + +static void +thread_sched_switch0(struct coroutine_context *current_cont, rb_thread_t *next_th, struct rb_native_thread *nt, bool to_dead) +{ + VM_ASSERT(!nt->dedicated); + VM_ASSERT(next_th->nt == NULL); + + RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th)); + + // this direct transfer serves next_th without a dequeue; cancel its + // Ractor's outstanding grq entry (no-op when nothing is enqueued) + ractor_sched_cancel_enq(next_th->vm, TH_SCHED(next_th)); + + ruby_thread_set_native(next_th); + native_thread_assign(nt, next_th); + + coroutine_transfer0(current_cont, next_th->sched.context, to_dead); +} + +static void +thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th) +{ + struct rb_native_thread *nt = cth->nt; + native_thread_assign(NULL, cth); + RUBY_DEBUG_LOG("th:%u->%u on nt:%d", rb_th_serial(cth), rb_th_serial(next_th), nt->serial); + thread_sched_switch0(cth->sched.context, next_th, nt, cth->status == THREAD_KILLED); +} + +#if VM_CHECK_MODE > 0 +RBIMPL_ATTR_MAYBE_UNUSED() +static unsigned int +grq_size(rb_vm_t *vm, rb_ractor_t *cr) +{ + ASSERT_ractor_sched_locked(vm, cr); + + rb_ractor_t *r, *prev_r = NULL; + unsigned int i = 0; + + ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) { + i++; + + VM_ASSERT(r != prev_r); + prev_r = r; + } + return i; +} +#endif + +// A native thread enters/leaves an epilogue that outlives its Ractor: from +// the increment until the decrement, ruby_vm_destruct waits for it below. +// The increment must happen while the VM still counts the thread's Ractor, +// so that the two never look absent at the same time. +void +rb_thread_sched_winding_begin(rb_vm_t *vm) +{ + RUBY_ATOMIC_INC(vm->ractor.sched.winding_cnt); +} + +void +rb_thread_sched_winding_end(rb_vm_t *vm) +{ + VM_ASSERT(RUBY_ATOMIC_LOAD(vm->ractor.sched.winding_cnt) > 0); + RUBY_ATOMIC_DEC(vm->ractor.sched.winding_cnt); +} + +// ruby_vm_destruct: wait until no native thread is between a coroutine +// epilogue and its reclaim -- past that point the reclaim frees through the +// (about to be destroyed) objspace and reads the (about to be unset) VM. +// Runs without the VM lock, which the epilogue needs to progress. +void +rb_thread_sched_wait_winding(rb_vm_t *vm) +{ + while (RUBY_ATOMIC_LOAD(vm->ractor.sched.winding_cnt) > 0) { + native_thread_yield(); + } +} + +// A direct service of a runnable thread (direct transfer or the hot-thread +// steal) bypasses the grq; cancel the Ractor's outstanding entry so that +// "enqueued <=> runnable and unserved" keeps holding. The caller holds the +// per-Ractor sched lock, so no concurrent enqueue can relink the node: a +// self-linked read needs no lock (the common case -- direct switches whose +// transition never enqueued). A linked read can race only with a dequeue, +// hence the recheck under the grq lock. +static void +ractor_sched_cancel_enq(rb_vm_t *vm, struct rb_thread_sched *sched) +{ + if (sched->grq_node.next != &sched->grq_node) { + ractor_sched_lock(vm, NULL); + { + if (sched->grq_node.next != &sched->grq_node) { + ccan_list_del_init(&sched->grq_node); + VM_ASSERT(vm->ractor.sched.grq_cnt > 0); + vm->ractor.sched.grq_cnt--; + } + } + ractor_sched_unlock(vm, NULL); + } +} + +static void +ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r) +{ + struct rb_thread_sched *sched = &r->threads.sched; + rb_ractor_t *cr = NULL; // timer thread can call this function + + VM_ASSERT(sched->running != NULL); + VM_ASSERT(sched->running->nt == NULL); + + ractor_sched_lock(vm, cr); + { + // Precondition: not already enqueued (the grq_node is self-linked). + // This holds because every service of a runnable-but-unserved thread + // either dequeues the entry (the nt scheduling loop) or cancels it + // (direct transfers / the hot-thread steal; see + // ractor_sched_cancel_enq) -- re-adding a linked node would corrupt + // the queue, so check unconditionally (a CHECK-mode-only assert + // would miss it: the race needs timing that CHECK builds perturb). + if (sched->grq_node.next != &sched->grq_node) { + rb_bug("ractor_sched_enq: already enqueued"); + } + ccan_list_add_tail(&vm->ractor.sched.grq, &sched->grq_node); + vm->ractor.sched.grq_cnt++; + VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt); + + RUBY_DEBUG_LOG("r:%u th:%u grq_cnt:%u", rb_ractor_id(r), rb_th_serial(sched->running), vm->ractor.sched.grq_cnt); + + rb_native_cond_signal(&vm->ractor.sched.cond); + + // The signal reaches a parked snt, and a running one revisits the + // queue in ractor_sched_deq before it can wait (same lock as here). + // With every snt dedicated or retired, only the timer thread's + // timeout branch can serve the entry or widen the pool: wake it + // (a no-op unless it sleeps untimed). + if (RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt) == 0) { + timer_thread_wakeup_locked(vm); + } + + // ractor_sched_dump(vm); + } + ractor_sched_unlock(vm, cr); +} + + +#ifndef MINIMUM_SNT +// make at least MINIMUM_SNT snts for debug. +#define MINIMUM_SNT 0 +#endif + +/* A shared thread woken with nothing to run is one whose turn another thread + * took first. After this many in a row it gives itself back: the queue keeps + * running dry, so the pool is wider than the work. 0 retires on the first one + * and is too eager to be useful; a negative value keeps every thread. */ +#ifndef SNT_IDLE_RETIRE +#define SNT_IDLE_RETIRE 3 +#endif + +/* Never give the last shared thread back. With none left an enqueue has nobody + * to signal, and the only code that makes one runs on the timer thread's + * timeout branch, which is reached only once it has seen a backlog. */ +#define SNT_KEEP_MINIMUM (MINIMUM_SNT > 1 ? MINIMUM_SNT : 1) + +static rb_ractor_t * +ractor_sched_deq(rb_vm_t *vm, rb_ractor_t *cr) +{ + rb_ractor_t *r; + int idle_streak = 0; // consecutive pops that found the queue empty + + ractor_sched_lock(vm, cr); + { + RUBY_DEBUG_LOG("empty? %d", ccan_list_empty(&vm->ractor.sched.grq)); + // ractor_sched_dump(vm); + + VM_ASSERT(rb_current_execution_context(false) == NULL); + VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt); + + while ((r = ccan_list_pop(&vm->ractor.sched.grq, rb_ractor_t, threads.sched.grq_node)) == NULL) { + RUBY_DEBUG_LOG("wait grq_cnt:%d", (int)vm->ractor.sched.grq_cnt); + + if (SNT_IDLE_RETIRE >= 0 && ++idle_streak > SNT_IDLE_RETIRE && + (int)RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt) > SNT_KEEP_MINIMUM) { + RUBY_ATOMIC_DEC(vm->ractor.sched.snt_cnt); + RUBY_DEBUG_LOG("retire, snt_cnt:%d", (int)vm->ractor.sched.snt_cnt); + break; // returning NULL ends this nt; see the caller + } + + ractor_sched_set_unlocked(vm, cr); + rb_native_cond_wait(&vm->ractor.sched.cond, &vm->ractor.sched.lock); + ractor_sched_set_locked(vm, cr); + + RUBY_DEBUG_LOG("wakeup grq_cnt:%d", (int)vm->ractor.sched.grq_cnt); + } + + VM_ASSERT(rb_current_execution_context(false) == NULL); + + if (r) { + ccan_list_node_init(&r->threads.sched.grq_node); // back to self-linked + VM_ASSERT(vm->ractor.sched.grq_cnt > 0); + vm->ractor.sched.grq_cnt--; + RUBY_DEBUG_LOG("r:%d grq_cnt:%u", (int)rb_ractor_id(r), vm->ractor.sched.grq_cnt); + } + else { + // the retire branch is the only way out of the loop without a ractor + VM_ASSERT(idle_streak > SNT_IDLE_RETIRE); + } + } + ractor_sched_unlock(vm, cr); + + return r; +} + +void rb_ractor_lock_self(rb_ractor_t *r); +void rb_ractor_unlock_self(rb_ractor_t *r); + +// The current thread for a ractor is put to "sleep" (descheduled in the STOPPED_FOREVER state) waiting for +// a ractor action to wake it up. +void +rb_ractor_sched_wait(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf, void *ubf_arg) +{ + // ractor lock of cr is acquired + + RUBY_DEBUG_LOG("start%s", ""); + + rb_thread_t * volatile th = rb_ec_thread_ptr(ec); + struct rb_thread_sched *sched = TH_SCHED(th); + struct ractor_waiter *waiter = (struct ractor_waiter*)ubf_arg; + + if (ubf_set(th, ubf, ubf_arg, &waiter->event_serial)) { + // interrupted + return; + } + + thread_sched_lock(sched, th); + rb_ractor_unlock_self(cr); + { + // A dedicated native thread takes the deadline on the very condvar a wakeup + // signals. An M:N thread has no condvar of its own, so its deadline goes to + // the timer thread, which then wakes it the way rb_ractor_sched_wakeup() does. + bool dedicated = th_has_dedicated_nt(th); + const rb_hrtime_t *end_p = NULL; + bool armed = false, expired = false; + + if (waiter->end) { + if (dedicated) { + end_p = waiter->end; + } + else { + // the timer wheel takes a relative timeout + rb_hrtime_t now = rb_hrtime_now(); + rb_hrtime_t rel = *waiter->end > now ? *waiter->end - now : 0; + + armed = ractor_sched_timeout_arm(th, &rel); + expired = !armed; + } + } + + if (expired) { + RUBY_DEBUG_LOG("expired before sleep%s", ""); + } + else if (armed && th->sched.waiting_reason.flags == thread_sched_waiting_none) { + // the timer thread already took this thread out of the wheel; bump the + // serial so that it does not try to wake a thread that never slept + th->sched.event_serial++; + } + else { + // setup sleep + bool can_direct_transfer = !dedicated; + RB_VM_SAVE_MACHINE_CONTEXT(th); + th->status = THREAD_STOPPED_FOREVER; + th->sched.waiting_timed = (end_p != NULL); // never true here for M:N (end_p is NULL) + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); + thread_sched_wakeup_next_thread(sched, th, can_direct_transfer); + // sleep + thread_sched_wait_running_turn(sched, th, can_direct_transfer, end_p); + th->sched.waiting_timed = false; + th->status = THREAD_RUNNABLE; + + // whoever woke this thread took the timeout back first + VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none); + } + } + thread_sched_unlock(sched, th); + rb_ractor_lock_self(cr); + + ubf_clear(th, true); + + RUBY_DEBUG_LOG("end%s", ""); +} + +void +rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *r_th) +{ + // ractor lock of r acquired + struct rb_thread_sched *sched = TH_SCHED(r_th); + + RUBY_DEBUG_LOG("r:%u th:%d", (unsigned int)rb_ractor_id(r), r_th->serial); + + thread_sched_lock(sched, r_th); + { + if (r_th->status == THREAD_STOPPED_FOREVER) { + RUBY_ATOMIC_ADD(r_th->unblock.event_serial, 1); + + // r_th must not resume with a wheel entry left behind: take its timeout + // back, as ubf_event_waiting() does. Only r_th arms it, and it is + // parked here, so reading the flags without the timer lock is safe. + if (r_th->sched.waiting_reason.flags != thread_sched_waiting_none) { + ractor_sched_timeout_disarm(r_th); + } + + // a timeout that fired first may have made r_th runnable already: waking + // it twice would put it on the readyq twice + if (sched->running != r_th && !r_th->sched.node.is_ready) { + r_th->sched.event_serial++; // a timeout still armed must not wake it again + thread_sched_to_ready_common(sched, r_th, true, false); + } + } + } + thread_sched_unlock(sched, r_th); +} + +static bool +ractor_sched_barrier_completed_p(rb_vm_t *vm) +{ + // The snapshot barrier_running_cnt is taken by the barrier's walk and + // decremented by counted deregistrations; no rescan is needed here. + RUBY_DEBUG_LOG("run:%u wait:%u", vm->ractor.sched.barrier_running_cnt, vm->ractor.sched.barrier_joined_cnt); + VM_ASSERT(vm->ractor.sched.barrier_running_cnt - 1 >= vm->ractor.sched.barrier_joined_cnt); + + return (vm->ractor.sched.barrier_running_cnt - vm->ractor.sched.barrier_joined_cnt) == 1; +} + +void +rb_ractor_sched_barrier_start(rb_vm_t *vm, rb_ractor_t *cr) +{ + VM_ASSERT(cr == GET_RACTOR()); + VM_ASSERT(vm->ractor.sync.lock_owner == cr); // VM is locked + VM_ASSERT(!vm->ractor.sched.barrier_is_waiting); + VM_ASSERT(vm->ractor.sched.barrier_joined_cnt == 0); + VM_ASSERT(vm->ractor.sched.barrier_ractor == NULL); + VM_ASSERT(vm->ractor.sched.barrier_lock_rec == 0); + + RUBY_DEBUG_LOG("start serial:%u", vm->ractor.sched.barrier_serial); + + unsigned int lock_rec; + + ractor_sched_lock(vm, cr); + { + RUBY_ATOMIC_SET(vm->ractor.sched.barrier_is_waiting, 1); + vm->ractor.sched.barrier_ractor = cr; + vm->ractor.sched.barrier_lock_rec = vm->ractor.sync.lock_rec; + + // release VM lock + lock_rec = vm->ractor.sync.lock_rec; + vm->ractor.sync.lock_rec = 0; + vm->ractor.sync.lock_owner = NULL; + rb_native_mutex_unlock(&vm->ractor.sync.lock); + + // Interrupt all running threads: running_dnts plus each snt's running_th. + // A switch before this scan is visible to it; one after it sees + // barrier_is_waiting (set above) and waits. + // Interrupt and count every registered runner, stamping each nt so a + // deregistration during this barrier knows it was counted. + rb_thread_t *ith; + unsigned int running_cnt = 0; + uint32_t serial = vm->ractor.sched.barrier_serial; + + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + struct rb_native_thread *dnt; + ccan_list_for_each(&vm->ractor.sched.ntlist.running_dnts, dnt, running_dnts_node) { + ith = dnt->running_thread; + dnt->barrier_counted_serial = serial; + running_cnt++; + if (ith->ractor != cr) { + RUBY_DEBUG_LOG("barrier request to th:%u", rb_th_serial(ith)); + RUBY_VM_SET_VM_BARRIER_INTERRUPT(ith->ec); + } + } + + struct rb_native_thread *nt; + ccan_list_for_each(&vm->ractor.sched.ntlist.snts, nt, snts_node) { + rb_native_mutex_lock(&nt->running_th_lock); + { + ith = nt->running_th; + if (ith != NULL) { + nt->barrier_counted_serial = serial; + running_cnt++; + if (ith->ractor != cr) { + RUBY_DEBUG_LOG("barrier request to th:%u", rb_th_serial(ith)); + RUBY_VM_SET_VM_BARRIER_INTERRUPT(ith->ec); + } + } + } + rb_native_mutex_unlock(&nt->running_th_lock); + } + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); + + vm->ractor.sched.barrier_running_cnt = running_cnt; + + // wait for other ractors + while (!ractor_sched_barrier_completed_p(vm)) { + ractor_sched_set_unlocked(vm, cr); + rb_native_cond_wait(&vm->ractor.sched.barrier_complete_cond, &vm->ractor.sched.lock); + ractor_sched_set_locked(vm, cr); + } + + RUBY_DEBUG_LOG("completed seirial:%u", vm->ractor.sched.barrier_serial); + + // no other ractors are there + vm->ractor.sched.barrier_serial++; + vm->ractor.sched.barrier_joined_cnt = 0; + rb_native_cond_broadcast(&vm->ractor.sched.barrier_release_cond); + + // acquire VM lock + rb_native_mutex_lock(&vm->ractor.sync.lock); + vm->ractor.sync.lock_rec = lock_rec; + vm->ractor.sync.lock_owner = cr; + } + + // do not release ractor_sched_lock and there is no newly added (resumed) thread + // thread_sched_setup_running_threads +} + +// called from vm_lock_leave if the vm_lock used for barrierred +void +rb_ractor_sched_barrier_end(rb_vm_t *vm, rb_ractor_t *cr) +{ + RUBY_DEBUG_LOG("serial:%u", (unsigned int)vm->ractor.sched.barrier_serial - 1); + VM_ASSERT(vm->ractor.sched.barrier_is_waiting); + VM_ASSERT(vm->ractor.sched.barrier_ractor); + VM_ASSERT(vm->ractor.sched.barrier_lock_rec > 0); + + RUBY_ATOMIC_SET(vm->ractor.sched.barrier_is_waiting, 0); + vm->ractor.sched.barrier_ractor = NULL; + vm->ractor.sched.barrier_lock_rec = 0; + ractor_sched_unlock(vm, cr); +} + +static void +ractor_sched_barrier_join_signal_locked(rb_vm_t *vm) +{ + if (ractor_sched_barrier_completed_p(vm)) { + rb_native_cond_signal(&vm->ractor.sched.barrier_complete_cond); + } +} + +static void +ractor_sched_barrier_join_wait_locked(rb_vm_t *vm, rb_thread_t *th) +{ + VM_ASSERT(vm->ractor.sched.barrier_is_waiting); + + unsigned int barrier_serial = vm->ractor.sched.barrier_serial; + + while (vm->ractor.sched.barrier_serial == barrier_serial) { + RUBY_DEBUG_LOG("sleep serial:%u", barrier_serial); + RB_VM_SAVE_MACHINE_CONTEXT(th); + + rb_ractor_t *cr = th->ractor; + ractor_sched_set_unlocked(vm, cr); + rb_native_cond_wait(&vm->ractor.sched.barrier_release_cond, &vm->ractor.sched.lock); + ractor_sched_set_locked(vm, cr); + + RUBY_DEBUG_LOG("wakeup serial:%u", barrier_serial); + } +} + +void +rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr) +{ + VM_ASSERT(cr->threads.sched.running != NULL); // running ractor + VM_ASSERT(cr == GET_RACTOR()); + VM_ASSERT(vm->ractor.sync.lock_owner == NULL); // VM is locked, but owner == NULL + VM_ASSERT(vm->ractor.sched.barrier_is_waiting); // VM needs barrier sync + +#if USE_RUBY_DEBUG_LOG || VM_CHECK_MODE > 0 + unsigned int barrier_serial = vm->ractor.sched.barrier_serial; +#endif + + RUBY_DEBUG_LOG("join"); + + rb_native_mutex_unlock(&vm->ractor.sync.lock); + { + VM_ASSERT(vm->ractor.sched.barrier_is_waiting); // VM needs barrier sync + VM_ASSERT(vm->ractor.sched.barrier_serial == barrier_serial); + + ractor_sched_lock(vm, cr); + { + // running_cnt + /* Every joiner is a member of the running set: a dying thread + * leaves the living set before handing over its scheduler slot. */ + vm->ractor.sched.barrier_joined_cnt++; + RUBY_DEBUG_LOG("waiting_cnt:%u serial:%u", vm->ractor.sched.barrier_joined_cnt, barrier_serial); + + ractor_sched_barrier_join_signal_locked(vm); + ractor_sched_barrier_join_wait_locked(vm, cr->threads.sched.running); + } + ractor_sched_unlock(vm, cr); + } + + rb_native_mutex_lock(&vm->ractor.sync.lock); + // VM locked here +} + +// Called when the ractor holding this sched is freed. A drained sched can +// still be on timeslice.scheds (pruning is lazy); an unlisted node is +// self-linked (fork re-inits them all), making this del a no-op. +void +rb_thread_sched_destroy(struct rb_thread_sched *sched) +{ + rb_vm_t *vm = GET_VM(); + + rb_native_mutex_lock(&vm->ractor.sched.timeslice.lock); + { + ccan_list_del_init(&sched->timeslice_node); + } + rb_native_mutex_unlock(&vm->ractor.sched.timeslice.lock); +} + +#if defined(HAVE_WORKING_FORK) +static void rb_internal_thread_event_hooks_rw_lock_atfork(void); + +static void +thread_sched_atfork(struct rb_thread_sched *sched) +{ + current_fork_gen++; + rb_thread_sched_init(sched, true); + rb_thread_t *th = GET_THREAD(); + rb_vm_t *vm = GET_VM(); + + if (th_has_dedicated_nt(th)) { + vm->ractor.sched.snt_cnt = 0; +#if USE_RUBY_DEBUG_LOG + vm->ractor.sched.dnt_cnt = 1; +#endif + } + else { + vm->ractor.sched.snt_cnt = 1; +#if USE_RUBY_DEBUG_LOG + vm->ractor.sched.dnt_cnt = 0; +#endif + } + + rb_native_mutex_initialize(&vm->ractor.sched.lock); +#if VM_CHECK_MODE > 0 + vm->ractor.sched.lock_owner = NULL; + vm->ractor.sched.locked = false; +#endif + + // rb_native_cond_destroy(&vm->ractor.sched.cond); + rb_native_cond_initialize(&vm->ractor.sched.cond); + rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond); + rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond); + + ccan_list_head_init(&vm->ractor.sched.grq); + vm->ractor.sched.grq_cnt = 0; // the list was just emptied; reset the count with it + // A fork during a VM barrier leaves the child with barrier state that can + // never complete (the other ractors are gone); reset it like the rest. + vm->ractor.sched.barrier_is_waiting = 0; // single-threaded child + vm->ractor.sched.barrier_joined_cnt = 0; + vm->ractor.sched.barrier_ractor = NULL; + vm->ractor.sched.barrier_lock_rec = 0; + // Threads that were winding down in the parent do not exist in the child; + // without this reset the child's ruby_vm_destruct would wait for their + // reclaim (which never comes) forever. + vm->ractor.sched.winding_cnt = 0; + rb_native_mutex_initialize(&vm->ractor.sched.ntlist.lock); + ccan_list_head_init(&vm->ractor.sched.ntlist.running_dnts); + ccan_list_head_init(&vm->ractor.sched.ntlist.snts); // those nts are gone + rb_native_mutex_initialize(&vm->ractor.sched.timeslice.lock); + ccan_list_head_init(&vm->ractor.sched.timeslice.scheds); + rb_native_mutex_initialize(&th->nt->running_th_lock); // a scan could hold it at fork + // Fork can copy nodes linked (or torn mid-link); re-init every sched's + // node so rb_thread_sched_destroy's del_init stays a no-op for them. + rb_ractor_t *r; + ccan_list_for_each(&vm->ractor.set, r, vmlr_node) { + ccan_list_node_init(&r->threads.sched.timeslice_node); + } + ccan_list_for_each(&vm->ractor.terminated_set, r, vmlr_node) { + ccan_list_node_init(&r->threads.sched.timeslice_node); + } + // th re-records itself below; the parent's record did not survive the lists + if (th->nt && th->nt->dedicated == 0) { + // surviving on an snt: put that nt back on the (just emptied) snts + // list, or the scans could not see this thread's record + ccan_list_add(&vm->ractor.sched.ntlist.snts, &th->nt->snts_node); + } + +#if USE_MN_THREADS + nt_machine_stack_atfork(); +#endif + rb_internal_thread_event_hooks_rw_lock_atfork(); + + VM_ASSERT(sched->is_running); + + if (sched->running != th) { + thread_sched_to_running(sched, th); + } + else { + thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL); + } + +#ifdef RB_THREAD_T_HAS_NATIVE_ID + if (th->nt) { + th->nt->tid = get_native_thread_id(); + } +#endif +} + +#endif + +extern int ruby_mn_threads_enabled; + +void +ruby_mn_threads_params(void) +{ + rb_vm_t *vm = GET_VM(); + rb_ractor_t *main_ractor = GET_RACTOR(); + + const char *mn_threads_cstr = getenv("RUBY_MN_THREADS"); + bool enable_mn_threads = false; + + if (USE_MN_THREADS && mn_threads_cstr && (enable_mn_threads = atoi(mn_threads_cstr) > 0)) { + // enabled + ruby_mn_threads_enabled = 1; + } + main_ractor->threads.sched.enable_mn_threads = enable_mn_threads; + + const char *max_cpu_cstr = getenv("RUBY_MAX_CPU"); + int max_cpu = native_thread_default_max_cpu(); + + if (USE_MN_THREADS && max_cpu_cstr) { + int given_max_cpu = atoi(max_cpu_cstr); + if (given_max_cpu > 0) { + max_cpu = given_max_cpu; + } + } + + vm->ractor.sched.max_cpu = max_cpu; +} + +static void +native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt) +{ + RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated + 1); + + if (nt->dedicated == 0) { + // Lock-free; pairs with ractor_sched_enq (enq: grq_cnt up then read + // snt_cnt / here: snt_cnt down then read grq_cnt) against lost wakeups. + if (RUBY_ATOMIC_FETCH_SUB(vm->ractor.sched.snt_cnt, 1) == 1) { + // the last snt went dedicated; pending entries need the timer thread + ractor_sched_lock(vm, cr); + { + if (vm->ractor.sched.grq_cnt > 0) { + timer_thread_wakeup_locked(vm); + } + } + ractor_sched_unlock(vm, cr); + } +#if USE_RUBY_DEBUG_LOG + vm->ractor.sched.dnt_cnt++; +#endif + } + + nt->dedicated++; +} + +static void +native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt) +{ + RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated - 1); + VM_ASSERT(nt->dedicated > 0); + nt->dedicated--; + + if (nt->dedicated == 0) { + // Rejoin under the max_cpu cap; with no room this nt retires and + // belongs to neither count until it ends. + while (1) { + rb_atomic_t snt = RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt); + if (snt < vm->ractor.sched.max_cpu || (int)snt <= MINIMUM_SNT) { + if (RUBY_ATOMIC_CAS(vm->ractor.sched.snt_cnt, snt, snt + 1) == snt) break; + } + else { + nt->retiring = true; + break; + } + } +#if USE_RUBY_DEBUG_LOG + vm->ractor.sched.dnt_cnt--; +#endif + } +} + +static void +native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th) +{ +#if USE_RUBY_DEBUG_LOG + if (nt) { + if (th->nt) { + RUBY_DEBUG_LOG("th:%d nt:%d->%d", (int)th->serial, (int)th->nt->serial, (int)nt->serial); + } + else { + RUBY_DEBUG_LOG("th:%d nt:NULL->%d", (int)th->serial, (int)nt->serial); + } + } + else { + if (th->nt) { + RUBY_DEBUG_LOG("th:%d nt:%d->NULL", (int)th->serial, (int)th->nt->serial); + } + else { + RUBY_DEBUG_LOG("th:%d nt:NULL->NULL", (int)th->serial); + } + } +#endif + + th->nt = nt; +} + +static int +native_thread_create_dedicated(rb_thread_t *th) +{ + th->nt = native_thread_alloc(); + th->nt->vm = th->vm; + th->nt->running_thread = th; + th->nt->dedicated = 1; + + // vm stack + size_t vm_stack_word_size = th->vm->default_params.thread_vm_stack_size / sizeof(VALUE); + void *vm_stack = ruby_xmalloc(vm_stack_word_size * sizeof(VALUE)); + th->sched.malloc_stack = true; + rb_ec_initialize_vm_stack(th->ec, vm_stack, vm_stack_word_size); + th->sched.context_stack = vm_stack; + th->sched.context_stack_size = vm_stack_word_size; + + int err = native_thread_create0(th->nt); + if (!err) { + // setup + thread_sched_to_ready(TH_SCHED(th), th); + } + return err; +} + +static void +call_thread_start_func_2(rb_thread_t *th) +{ + /* Capture the address of a local in this stack frame to mark the beginning of the + machine stack for this thread. This is required even if we can tell the real + stack beginning from the pthread API in native_thread_init_stack, because + glibc stores some of its own data on the stack before calling into user code + on a new thread, and replacing that data on fiber-switch would break it (see + bug #13887) */ + VALUE stack_start = 0; + VALUE *stack_start_addr = asan_get_real_stack_addr(&stack_start); + + native_thread_init_stack(th, stack_start_addr); + thread_start_func_2(th, th->ec->machine.stack_start); +} + +static void * +nt_start(void *ptr) +{ + struct rb_native_thread *nt = (struct rb_native_thread *)ptr; + rb_vm_t *vm = nt->vm; + + native_thread_setup_on_thread(nt); + + // init tid +#ifdef RB_THREAD_T_HAS_NATIVE_ID + nt->tid = get_native_thread_id(); +#endif + +#if USE_RUBY_DEBUG_LOG && defined(RUBY_NT_SERIAL) + ruby_nt_serial = nt->serial; +#endif + + RUBY_DEBUG_LOG("nt:%u", nt->serial); + + bool in_snts = false; + + if (!nt->dedicated) { + coroutine_initialize_main(nt->nt_context); + + // join the snt list that the barrier/timeslice scans walk + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + ccan_list_add(&vm->ractor.sched.ntlist.snts, &nt->snts_node); + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); + in_snts = true; + } + + bool retired = false; + + while (1) { + if (nt->dedicated) { + // wait running turn + rb_thread_t *th = nt->running_thread; + struct rb_thread_sched *sched = TH_SCHED(th); + + RUBY_DEBUG_LOG("on dedicated th:%u", rb_th_serial(th)); + ruby_thread_set_native(th); + + thread_sched_lock(sched, th); + { + if (sched->running == th) { + thread_sched_add_running_thread(sched, th); + } + thread_sched_wait_running_turn(sched, th, false, NULL); + } + thread_sched_unlock(sched, th); + + // start threads + call_thread_start_func_2(th); + break; // TODO: allow to change to the SNT + } + else { + RUBY_DEBUG_LOG("check next"); + if (nt->retiring) { // came back with no room in the shared pool + retired = true; + break; + } + + rb_ractor_t *r = ractor_sched_deq(vm, NULL); + + if (r) { + struct rb_thread_sched *sched = &r->threads.sched; + + bool locked = true; + + thread_sched_lock(sched, NULL); + { + rb_thread_t *next_th = sched->running; + + if (next_th && next_th->nt == NULL) { + RUBY_DEBUG_LOG("nt:%d next_th:%d", (int)nt->serial, (int)next_th->serial); +#if USE_MN_THREADS + thread_sched_switch0(nt->nt_context, next_th, nt, false); + + // If a coroutine terminated during the transfer, co_start + // recorded it in nt->dead_co (switch0's return value is + // backend-dependent, unusable; see thread_pthread.h). + struct coroutine_context *dead_co = nt->dead_co; + nt->dead_co = NULL; + if (thread_sched_reclaim(dead_co)) { + // it already released the sched lock before its + // transfer (its Ractor may be gone): leave sched be. + locked = false; + } +#else + thread_sched_switch0(nt->nt_context, next_th, nt, false); +#endif + } + else { + RUBY_DEBUG_LOG("no schedulable threads -- next_th:%p", next_th); + } + } + if (locked) { + thread_sched_unlock(sched, NULL); + } + } + else { + // ractor_sched_deq retired this nt. + retired = true; + break; + } + + if (nt->dedicated) { + // SNT becomes DNT while running + break; + } + } + } + + if (in_snts) { + // Leaving the shared loop: every path back here deregistered first + // (park and death both precede the transfer), so only the snts entry + // is left to remove. + VM_ASSERT(nt->running_th == NULL); + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + ccan_list_del_init(&nt->snts_node); + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); + } + + if (retired) { + // The counts dropped this nt already; nothing can reference it now. + RUBY_DEBUG_LOG("retired nt:%u", nt->serial); + native_thread_destroy_self(nt); + } + + return NULL; +} + +static int native_thread_create_shared(rb_thread_t *th); + +#if USE_MN_THREADS +static void nt_free_stack(void *mstack); + + +// Reclaim the context a coroutine thread recorded in nt->dead_co before its +// final transfer (co_start's epilogue). Our running here proves that transfer's +// register save into the block completed. Returns true when a thread did +// terminate -- it RELEASED the sched lock before transferring; NULL/false means +// a live yield, where the loop still owns the lock. +static bool +thread_sched_reclaim(struct coroutine_context *dead_co) +{ + struct rb_thread_context *tctx = (struct rb_thread_context *)dead_co; + + if (tctx != NULL && tctx->dead) { + nt_free_stack(tctx->stack); + SIZED_FREE(tctx); + // pairs with the increment at the top of coroutine_thread_terminated: + // a waiting VM destruct may proceed once this reclaim is done + VM_ASSERT(RUBY_ATOMIC_LOAD(GET_VM()->ractor.sched.winding_cnt) > 0); + RUBY_ATOMIC_DEC(GET_VM()->ractor.sched.winding_cnt); + return true; + } + return false; +} +#endif + +void +rb_thread_wake_fence(rb_thread_t *th) +{ + timer_thread_wake_fence(th); +} + +void +rb_threadptr_sched_free(rb_thread_t *th) +{ + timer_thread_wake_fence(th); +#if USE_MN_THREADS + if (th->sched.malloc_stack) { + // has dedicated + SIZED_FREE_N((VALUE *)th->sched.context_stack, th->sched.context_stack_size); + native_thread_destroy(th->nt); + } + else if (th->sched.context != NULL) { + // a coroutine thread that never reached its epilogue (never started); + // a terminated one is reclaimed by whoever resumed from its final + // transfer (thread_sched_reclaim), and cleared this pointer. + struct rb_thread_context *tctx = (struct rb_thread_context *)th->sched.context; + nt_free_stack(tctx->stack); + SIZED_FREE(tctx); + th->sched.context = NULL; + // TODO: how to free nt and nt->altstack? + } +#else + SIZED_FREE_N((VALUE *)th->sched.context_stack, th->sched.context_stack_size); + native_thread_destroy(th->nt); +#endif + + th->nt = NULL; +} + + +static int +native_thread_create(rb_thread_t *th) +{ + VM_ASSERT(th->nt == 0); + RUBY_DEBUG_LOG("th:%d has_dnt:%d", th->serial, th->has_dedicated_nt); + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_STARTED, th); + + if (!th->ractor->threads.sched.enable_mn_threads) { + th->has_dedicated_nt = 1; + } + + if (th->has_dedicated_nt) { + return native_thread_create_dedicated(th); + } + else { + return native_thread_create_shared(th); + } +} + +#ifdef USE_UBF_LIST +static CCAN_LIST_HEAD(ubf_list_head); +#ifdef RB_NATIVETHREAD_LOCK_INIT +static rb_nativethread_lock_t ubf_list_lock = RB_NATIVETHREAD_LOCK_INIT; +#else +// no static initializer on this platform; thread_sched_init_vm() does it +static rb_nativethread_lock_t ubf_list_lock; +#endif + +static void +ubf_list_atfork(void) +{ + ccan_list_head_init(&ubf_list_head); + rb_native_mutex_initialize(&ubf_list_lock); +} + +RBIMPL_ATTR_MAYBE_UNUSED() +static bool +ubf_list_contain_p(rb_thread_t *th) +{ + rb_thread_t *list_th; + ccan_list_for_each(&ubf_list_head, list_th, sched.node.ubf) { + if (list_th == th) return true; + } + return false; +} + +/* The thread 'th' is registered to be trying unblock. */ +static void +register_ubf_list(rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); + struct ccan_list_node *node = &th->sched.node.ubf; + + VM_ASSERT(th->unblock.func != NULL); + + rb_native_mutex_lock(&ubf_list_lock); + { + // check not connected yet + if (ccan_list_empty((struct ccan_list_head*)node)) { + VM_ASSERT(!ubf_list_contain_p(th)); + ccan_list_add(&ubf_list_head, node); + } + } + rb_native_mutex_unlock(&ubf_list_lock); + + timer_thread_wakeup(); +} + +/* The thread 'th' is unblocked. It no longer need to be registered. */ +static void +unregister_ubf_list(rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); + struct ccan_list_node *node = &th->sched.node.ubf; + + /* we can't allow re-entry into ubf_list_head */ + VM_ASSERT(th->unblock.func == NULL); + + if (!ccan_list_empty((struct ccan_list_head*)node)) { + rb_native_mutex_lock(&ubf_list_lock); + { + VM_ASSERT(ubf_list_contain_p(th)); + ccan_list_del_init(node); + } + rb_native_mutex_unlock(&ubf_list_lock); + } +} + +/* + * Poke the target thread so that it returns from a blocking syscall. + * How that is done is up to the platform (native_thread_interrupt). + */ +static void +ubf_wakeup_thread(rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u thread_id:%p", rb_th_serial(th), (void *)th->nt->thread_id); + + native_thread_interrupt(th); +} + +static void +ubf_select(void *ptr) +{ + rb_thread_t *th = (rb_thread_t *)ptr; + RUBY_DEBUG_LOG("wakeup th:%u", rb_th_serial(th)); + ubf_wakeup_thread(th); + register_ubf_list(th); +} + +static bool +ubf_threads_empty(void) +{ + return ccan_list_empty(&ubf_list_head) != 0; +} + +static void +ubf_wakeup_all_threads(void) +{ + rb_thread_t *th; + rb_native_mutex_lock(&ubf_list_lock); + { + ccan_list_for_each(&ubf_list_head, th, sched.node.ubf) { + ubf_wakeup_thread(th); + } + } + rb_native_mutex_unlock(&ubf_list_lock); +} + +#else /* USE_UBF_LIST */ +#define register_ubf_list(th) (void)(th) +#define unregister_ubf_list(th) (void)(th) +#define ubf_select 0 +static void ubf_wakeup_all_threads(void) { return; } +static bool ubf_threads_empty(void) { return true; } +#define ubf_list_atfork() do {} while (0) +#endif /* USE_UBF_LIST */ + +static int +timer_thread_set_timeout(rb_vm_t *vm) +{ +#if 0 + return 10; // ms +#else + int timeout = -1; + + ractor_sched_lock(vm, NULL); + { + if ( timeslice_scan(vm, false) // (1-1) Provide time slice for active NTs + || !ubf_threads_empty() // (1-3) Periodic UBF + || vm->ractor.sched.grq_cnt > 0 // (1-4) Lazy GRQ deq start + ) { + + RUBY_DEBUG_LOG("ubf:%d grq:%d", + !ubf_threads_empty(), + (vm->ractor.sched.grq_cnt > 0)); + + timeout = 10; // ms + vm->ractor.sched.timeslice_wait_inf = false; + } + else { + vm->ractor.sched.timeslice_wait_inf = true; + } + } + ractor_sched_unlock(vm, NULL); + + timeout = timer_wheel_timeout(timeout); + + RUBY_DEBUG_LOG("timeout:%d inf:%d", timeout, (int)vm->ractor.sched.timeslice_wait_inf); + + // fprintf(stderr, "timeout:%d\n", timeout); + return timeout; +#endif +} + +static void +timer_thread_check_signal(rb_vm_t *vm) +{ + // ruby_sigchld_handler(vm); TODO + + int signum = rb_signal_buff_size(); + if (UNLIKELY(signum > 0) && vm->ractor.main_thread) { + RUBY_DEBUG_LOG("signum:%d", signum); + threadptr_trap_interrupt(vm->ractor.main_thread); + } +} + +// Tick (with `interrupt`) each listed sched's running thread and prune scheds +// whose readyq drained; returns whether any sched still needs ticks. +static bool +timeslice_scan(rb_vm_t *vm, bool interrupt) +{ + bool found = false; + struct rb_thread_sched *sched, *next; + + rb_native_mutex_lock(&vm->ractor.sched.timeslice.lock); + { + ccan_list_for_each_safe(&vm->ractor.sched.timeslice.scheds, sched, next, timeslice_node) { + // trylock: timeslice_sched_link nests sched.lock -> timeslice.lock, + // this scan holds the locks the other way around + if (rb_native_mutex_trylock(&sched->lock_) == 0) { + if (ccan_list_empty(&sched->readyq)) { + ccan_list_del_init(&sched->timeslice_node); // a later enq relinks it + } + else if (sched->is_running) { + VM_ASSERT(sched->running != NULL); + found = true; + if (interrupt) { + RUBY_DEBUG_LOG("timeslice th:%u", rb_th_serial(sched->running)); + RUBY_VM_SET_TIMER_INTERRUPT(sched->running->ec); + } + } + // else: waiters behind a blocked runner need no ticks; the + // add path wakes the timer when the sched runs again + rb_native_mutex_unlock(&sched->lock_); + } + else { + found = true; // busy switching; tick it on the next round + } + } + } + rb_native_mutex_unlock(&vm->ractor.sched.timeslice.lock); + + return found; +} + +static void +timer_thread_check_timeslice(rb_vm_t *vm) +{ + // TODO: check time + timeslice_scan(vm, true); +} + +static void * +timer_thread_func(void *ptr) +{ + rb_vm_t *vm = (rb_vm_t *)ptr; +#if defined(RUBY_NT_SERIAL) + ruby_nt_serial = (rb_atomic_t)-1; +#endif + + RUBY_DEBUG_LOG("started%s", ""); + + while (RUBY_ATOMIC_LOAD(system_working)) { + timer_thread_check_signal(vm); + timer_thread_check_timeout(vm); + ubf_wakeup_all_threads(); + + RUBY_DEBUG_LOG("system_working:%d", RUBY_ATOMIC_LOAD(system_working)); + timer_thread_polling(vm); + } + + RUBY_DEBUG_LOG("terminated"); + return NULL; +} + +static void +timer_thread_wakeup_locked(rb_vm_t *vm) +{ + // should be locked before. + ASSERT_ractor_sched_locked(vm, NULL); + + if (TIMER_THREAD_CREATED_P()) { + if (vm->ractor.sched.timeslice_wait_inf) { + RUBY_DEBUG_LOG("wakeup%s", ""); + timer_thread_wakeup_force(); + } + else { + RUBY_DEBUG_LOG("will be wakeup..."); + } + } +} + +static void +timer_thread_wakeup(void) +{ + rb_vm_t *vm = GET_VM(); + + ractor_sched_lock(vm, NULL); + { + timer_thread_wakeup_locked(vm); + } + ractor_sched_unlock(vm, NULL); +} + +static void +native_sleep(rb_thread_t *th, rb_hrtime_t *rel) +{ + struct rb_thread_sched *sched = TH_SCHED(th); + + RUBY_DEBUG_LOG("rel:%d", rel ? (int)*rel : 0); + + if (rel && !th_has_dedicated_nt(th)) { + // an M:N thread has no condvar of its own: the timer thread wakes it + thread_sched_wait_events(sched, th, -1, thread_sched_waiting_timeout, rel); + } + else if (rel) { + /* Solaris cond_timedwait() returns EINVAL if an argument is greater than + * current_time + 100,000,000. So cut up to 100,000,000. This is + * considered as a kind of spurious wakeup. The caller to native_sleep + * should care about spurious wakeup. + * + * See also [Bug #1341] [ruby-core:29702] + * http://download.oracle.com/docs/cd/E19683-01/816-0216/6m6ngupgv/index.html + */ + const rb_hrtime_t max = (rb_hrtime_t)100000000 * RB_HRTIME_PER_SEC; + if (*rel > max) *rel = max; + + rb_hrtime_t end = rb_hrtime_add(rb_hrtime_now(), *rel); + thread_sched_to_waiting_until_wakeup(sched, th, &end); + } + else { + thread_sched_to_waiting_until_wakeup(sched, th, NULL); + } + + RUBY_DEBUG_LOG("wakeup"); +} + + +// return true if the current thread acquires DNT. +// return false if the current thread already acquires DNT. +bool +rb_thread_lock_native_thread(void) +{ + rb_thread_t *th = GET_THREAD(); + bool is_snt = th->nt->dedicated == 0; + native_thread_dedicated_inc(th->vm, th->ractor, th->nt); + + return is_snt; +} + +void +rb_thread_malloc_stack_set(rb_thread_t *th, void *stack, size_t stack_size) +{ + th->sched.malloc_stack = true; + th->sched.context_stack = stack; + th->sched.context_stack_size = stack_size; +} + +// VM wide scheduler state, shared by every platform. Called from +// Init_native_thread() before the main thread is recorded. +static void +thread_sched_init_vm(rb_vm_t *vm) +{ + rb_native_mutex_initialize(&vm->ractor.sched.lock); + rb_native_cond_initialize(&vm->ractor.sched.cond); + rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond); + rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond); + + ccan_list_head_init(&vm->ractor.sched.grq); + rb_native_mutex_initialize(&vm->ractor.sched.ntlist.lock); + ccan_list_head_init(&vm->ractor.sched.ntlist.running_dnts); + ccan_list_head_init(&vm->ractor.sched.ntlist.snts); + rb_native_mutex_initialize(&vm->ractor.sched.timeslice.lock); + ccan_list_head_init(&vm->ractor.sched.timeslice.scheds); + +#ifndef RB_NATIVETHREAD_LOCK_INIT + // ubf_list_lock could not be initialized statically + ubf_list_atfork(); +#endif +} diff --git a/thread_sched.h b/thread_sched.h new file mode 100644 index 00000000000000..0fe5bd951133d3 --- /dev/null +++ b/thread_sched.h @@ -0,0 +1,264 @@ +#ifndef RUBY_THREAD_SCHED_H +#define RUBY_THREAD_SCHED_H +/********************************************************************** + + thread_sched.h - data structures of the thread/ractor scheduler + + The scheduler itself lives in thread_sched.c and is shared by every + platform. This header holds the types it works on; it is included from + the platform header (thread_pthread.h / thread_win32.h), which adds the + platform specific members and the thread local storage plumbing. + + == platform primitive layer == + + thread_sched.c is built on top of the following, which the platform + implementation (THREAD_IMPL_SRC) has to provide before including it: + + * rb_native_mutex_*() / rb_native_cond_*() (ruby/thread_native.h) + * native_thread_create0() / native_thread_destroy() and friends + * native_thread_interrupt() -- poke a thread out of a blocking call + * timer_thread_polling() -- the timer thread's blocking wait + * timer_thread_wakeup_force() -- wake that wait up + * TIMER_THREAD_CREATED_P() + * USE_MN_THREADS -- 1 enables the M:N scheduler + (when 0 the platform supplies the stubs listed at the bottom of + thread_sched_mn.c) + +**********************************************************************/ + +// How a thread_sched_wait_events() wait ended. "unavailable" (could not be +// registered) is not "the event fired": the caller must fall back, not proceed. +enum thread_sched_wait_result { + thread_sched_wait_event, // an event the caller asked for fired + thread_sched_wait_timeout, // the timeout expired before any event + thread_sched_wait_unavailable, // not registered; the caller must fall back +}; + +// this data should be protected by timer_th.waiting_lock +struct rb_thread_sched_waiting { + enum thread_sched_waiting_flag { + thread_sched_waiting_none = 0x00, + thread_sched_waiting_timeout = 0x01, + thread_sched_waiting_io_read = 0x02, + thread_sched_waiting_io_write = 0x08, + thread_sched_waiting_io_force = 0x40, // ignore readable + } flags; + + struct { + // should be compat with hrtime.h +#ifdef MY_RUBY_BUILD_MAY_TIME_TRAVEL + int128_t timeout; +#else + uint64_t timeout; +#endif + uint32_t event_serial; + int fd; // -1 for timeout only + int result; + } data; + + // connected to a timer_th wheel slot (timed) or timer_th.waiting_untimed + struct ccan_list_node node; + + /* which wheel slot `node` is on; meaningful only while flags has + * thread_sched_waiting_timeout */ + uint8_t wheel_lvl; + uint8_t wheel_slot; + + // connected to rb_fd_waiters.waiters of data.fd + struct ccan_list_node fd_node; +}; + +// One entry per fd with waiters; fds stay dense, so a table indexed by fd fits. +// Entries live in fixed chunks: growing must not move a live list head. +struct rb_fd_waiters { + struct ccan_list_head waiters; // rb_thread_sched_waiting.fd_node + + // The io flags currently armed in epoll/kqueue for this fd: the union of + // what its waiters asked for. + uint32_t armed_flags; + + // Bumped on full disarm. Events carry the generation they were armed with, + // so one queued before the fd was disarmed (and reused) is recognised. + uint32_t generation; +}; + +// per-Thread scheduler helper data +struct rb_thread_sched_item { + struct { + struct ccan_list_node ubf; + + // connected to ractor->threads.sched.reqdyq + // locked by ractor->threads.sched.lock + struct ccan_list_node readyq; + // Indicates whether thread is on the readyq. + // There is no clear relationship between this and th->status. + bool is_ready; + + } node; + + struct rb_thread_sched_waiting waiting_reason; + uint32_t event_serial; + + // wakes pending on this thread (timer thread or an fd shard claim); + // under timer_th.wake_pending_lock + uint32_t wake_pending_cnt; + + // parked on its own condvar with a deadline; under the sched lock (see + // ubf_waiting). Always false for an M:N thread: its deadline lives on the + // timer wheel, and its early wake comes from the timer thread instead. + bool waiting_timed; + + bool malloc_stack; + void *context_stack; + size_t context_stack_size; + struct coroutine_context *context; +}; + +struct rb_native_thread { + rb_atomic_t serial; + struct rb_vm_struct *vm; + + rb_nativethread_id_t thread_id; + +#ifdef RB_THREAD_T_HAS_NATIVE_ID + int tid; +#endif + +#if defined(_WIN32) + // signalled by native_thread_interrupt() to break this thread out of a + // blocking w32_wait_events() + HANDLE interrupt_event; +#endif + + struct rb_thread_struct *running_thread; + + // The running thread on this shared nt, for the barrier/timeslice scans. + // While a scan holds running_th_lock the thread cannot finish parking. + rb_nativethread_lock_t running_th_lock; + struct rb_thread_struct *running_th; + struct ccan_list_node snts_node; // in vm->ractor.sched.ntlist.snts + // in vm->ractor.sched.ntlist.running_dnts while running_thread runs + struct ccan_list_node running_dnts_node; + // barrier_serial stamped by the barrier's counting walk; this nt's + // deregistration during that barrier decrements the snapshot count + uint32_t barrier_counted_serial; + + // to control native thread; use sched->lock + rb_nativethread_cond_t readyq; + +#ifdef USE_SIGALTSTACK + void *altstack; +#endif + + struct coroutine_context *nt_context; + int dedicated; + + // set when this thread came back from a blocking region with no room left + // in the shared pool; it ends when it next asks for work + bool retiring; + + // A terminating coroutine records its context here before its final + // transfer; this nt's loop reclaims it. (Not via coroutine_transfer()'s + // return value: its meaning differs between the amd64 asm and ucontext.) + struct coroutine_context *dead_co; +}; + +// defines these as macros, and the field names below (and in the +// rest of the interpreter) would be rewritten by them. +#undef except +#undef try +#undef leave +#undef finally + +// per-Ractor +struct rb_thread_sched { + rb_nativethread_lock_t lock_; +#if VM_CHECK_MODE + struct rb_thread_struct *lock_owner; +#endif + struct rb_thread_struct *running; // running thread or NULL + // Most recently running thread or NULL. If this thread wakes up before the newly running + // thread completes the transfer of control, it can interrupt and resume running. + // The new thread clears this field when it takes control. + struct rb_thread_struct *runnable_hot_th; + int runnable_hot_th_waiting; + bool is_running; + + bool enable_mn_threads; + + struct ccan_list_head readyq; + int readyq_cnt; + // ractor scheduling + // When not linked in vm->ractor.sched.grq, this node is kept + // self-linked (ccan_list_node_init), so "linked?" can be read off the + // node itself: enqueuers assert it, and direct transfers cancel an + // outstanding entry (see ractor_sched_cancel_enq). + struct ccan_list_node grq_node; + struct ccan_list_node timeslice_node; // self-linked = not on timeslice.scheds +}; + +struct rb_thread_context; + +// A coroutine (M:N) thread's teardown runs coroutine_thread_terminated +// instead of the dedicated-thread path in thread_start_func_2; see the +// comments there and in thread_sched_mn.c. th->sched.context is cleared in +// that epilogue, so this also reads as "did not tear down yet". +// (Only meaningful when USE_MN_THREADS -- gate uses accordingly; the macro +// itself is a plain pointer test and always compiles.) +#define th_has_coroutine(th) ((th)->sched.context != NULL) + +struct rb_ractor_struct; + +// VM wide: what schedules Ractors onto native threads. One per VM, in +// rb_vm_struct.ractor.sched. +struct rb_ractor_sched { + rb_nativethread_lock_t lock; + struct rb_ractor_struct *lock_owner; + bool locked; + + rb_nativethread_cond_t cond; // GRQ + rb_atomic_t snt_cnt; // count of shared NTs; lock-free (see native_thread_dedicated_inc) + unsigned int dnt_cnt; // count of dedicated NTs; logging only (USE_RUBY_DEBUG_LOG), not atomic + + unsigned int max_cpu; + struct ccan_list_head grq; // // Global Ready Queue + rb_atomic_t winding_cnt; // native threads between a coroutine epilogue and its reclaim; ruby_vm_destruct waits for 0 + unsigned int grq_cnt; + + // What the barrier walk visits: threads running on dedicated + // nts, and the shared nts (whose running_th fields hold the rest). + struct { + rb_nativethread_lock_t lock; + struct ccan_list_head running_dnts; + struct ccan_list_head snts; + } ntlist; + + // scheds whose readyq holds waiters: the timer ticks their + // running thread (timeslice_scan) and prunes drained entries. + struct { + rb_nativethread_lock_t lock; + struct ccan_list_head scheds; + } timeslice; + + // true if timeslice timer is not enable + bool timeslice_wait_inf; + + // barrier + rb_nativethread_cond_t barrier_complete_cond; + rb_nativethread_cond_t barrier_release_cond; + // bool; nonzero while a stop-the-world section is active. Set + // before the barrier walks the running records; a record moved + // after the walk sees it (thread_sched_setup_running_threads). + rb_atomic_t barrier_is_waiting; + unsigned int barrier_joined_cnt; // threads joined so far; under sched.lock + unsigned int barrier_running_cnt; // runners counted by the barrier's walk; under sched.lock + unsigned int barrier_serial; + struct rb_ractor_struct *barrier_ractor; + unsigned int barrier_lock_rec; +}; + +void rb_ractor_sched_wait(struct rb_execution_context_struct *ec, struct rb_ractor_struct *cr, rb_unblock_function_t *ptr, void *arg); +void rb_ractor_sched_wakeup(struct rb_ractor_struct *r, struct rb_thread_struct *th); +void rb_thread_wake_fence(struct rb_thread_struct *th); + +#endif /* RUBY_THREAD_SCHED_H */ diff --git a/thread_pthread_mn.c b/thread_sched_mn.c similarity index 97% rename from thread_pthread_mn.c rename to thread_sched_mn.c index a57008da7b0f31..ddf35b459b8602 100644 --- a/thread_pthread_mn.c +++ b/thread_sched_mn.c @@ -1,4 +1,22 @@ -// included by "thread_pthread.c" +/* -*-c-*- */ +/********************************************************************** + + thread_sched_mn.c - the M:N scheduler + + Included by the platform implementation (currently only thread_pthread.c) + when USE_MN_THREADS is 1. A platform that cannot run coroutine threads + defines it to 0 and supplies the stubs at the bottom of this file itself + (see thread_win32.c). + + Most of what is here is platform independent: the coroutine threads + themselves, the native thread stack pool, the timer wheel, and the + fd -> waiters map. The part that is not is the readiness backend -- + arming an fd and waiting for events -- which is epoll on Linux and kqueue + elsewhere. Those pieces are marked "backend" below; they are the natural + seam for a thread_sched_epoll.c / thread_sched_kqueue.c split, and for an + IOCP backend that would let Windows run M:N threads too. + +**********************************************************************/ #if USE_MN_THREADS @@ -1358,6 +1376,14 @@ verify_waiting_list(void) #endif } +/* ------------------------------------------------------------------------ + * backend: the readiness notification mechanism (epoll / kqueue). + * + * Everything below that names epoll or kqueue is this backend; the rest of + * the M:N scheduler only asks it to arm an fd (fd_waiters_arm) and to wait + * for what fired (event_wait / timer_thread_polling). + * ------------------------------------------------------------------------ */ + #if HAVE_SYS_EVENT_H // kqueue helpers static enum thread_sched_waiting_flag diff --git a/thread_win32.c b/thread_win32.c index a7163fa8066acf..456ad8114bd86d 100644 --- a/thread_win32.c +++ b/thread_win32.c @@ -7,6 +7,10 @@ Copyright (C) 2004-2007 Koichi Sasada + Windows platform primitives for the common thread/ractor scheduler. The + scheduler itself is in thread_sched.c, which includes this file and then + builds on the primitives below; see thread_sched.h for the contract. + **********************************************************************/ #ifdef THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION @@ -14,21 +18,43 @@ #include "internal/sanitizers.h" #include -#define TIME_QUANTUM_USEC (10 * 1000) -#define RB_CONDATTR_CLOCK_MONOTONIC 1 /* no effect */ - #undef Sleep #define native_thread_yield() Sleep(0) -#define unregister_ubf_list(th) -#define ubf_wakeup_all_threads() do {} while (0) -#define ubf_threads_empty() (1) -#define ubf_timer_disarm() do {} while (0) -#define ubf_list_atfork() do {} while (0) + +// A CRITICAL_SECTION is recursive, so trylock cannot tell "held by me" from +// "free"; see thread_sched.c. +#define RB_NATIVE_MUTEX_TRYLOCK_DETECTS_SELF 0 + +// M:N threads need an event backend to park a coroutine on (epoll/kqueue on +// the POSIX side; IOCP would be the Windows counterpart). Until there is one +// every thread here is dedicated, and this file supplies the stubs that +// thread_sched_mn.c provides elsewhere. +#define USE_MN_THREADS 0 + +// Interruption is delivered through a per-native-thread event object rather +// than a signal, but the bookkeeping is the same as everywhere else. +#define USE_UBF_LIST 1 + +#include COROUTINE_H + +// Thread event hooks are not implemented on this platform. +#define RB_INTERNAL_THREAD_HOOK(event, th) ((void)0) + +// No fork(), so this never advances; it only keeps TIMER_THREAD_CREATED_P() +// spelled the same way on both platforms. +static rb_serial_t current_fork_gen = 1; + +// Always: native_cond_timedwait() below takes the rb_hrtime_t deadline and +// converts it to the relative timeout the Win32 wait wants itself. +#define RB_NATIVE_COND_HRTIME_DEADLINE_P() 1 static volatile DWORD ruby_native_thread_key = TLS_OUT_OF_INDEXES; static int w32_wait_events(HANDLE *events, int count, DWORD timeout, rb_thread_t *th); +static void native_thread_destroy(struct rb_native_thread *nt); +static void timer_thread_wakeup_force(void); +static void ubf_select(void *ptr); // thread_sched.c rb_internal_thread_event_hook_t * rb_internal_thread_add_event_hook(rb_internal_thread_event_callback callback, rb_event_flag_t internal_event, void *user_data) @@ -129,159 +155,6 @@ w32_mutex_create(void) return lock; } -#define GVL_DEBUG 0 - -static void -thread_sched_to_running(struct rb_thread_sched *sched, rb_thread_t *th) -{ - w32_mutex_lock(sched->lock, false); - if (GVL_DEBUG) fprintf(stderr, "gvl acquire (%p): acquire\n", th); -} - -static void -thread_sched_to_waiting(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately) -{ - ReleaseMutex(sched->lock); -} - -static void -thread_sched_to_dead(struct rb_thread_sched *sched, rb_thread_t *th) -{ - thread_sched_to_waiting(sched, th, true); -} - -static void -thread_sched_yield(struct rb_thread_sched *sched, rb_thread_t *th) -{ - thread_sched_to_waiting(sched, th, true); - native_thread_yield(); - thread_sched_to_running(sched, th); -} - -void -rb_thread_sched_init(struct rb_thread_sched *sched, bool atfork) -{ - if (GVL_DEBUG) fprintf(stderr, "sched init\n"); - sched->lock = w32_mutex_create(); -} - -#if 0 -// per-ractor -void -rb_thread_sched_destroy(struct rb_thread_sched *sched) -{ - if (GVL_DEBUG) fprintf(stderr, "sched destroy\n"); - CloseHandle(sched->lock); -} -#endif - -rb_thread_t * -ruby_thread_from_native(void) -{ - return TlsGetValue(ruby_native_thread_key); -} - -int -ruby_thread_set_native(rb_thread_t *th) -{ - if (th && th->ec) { - rb_ractor_set_current_ec(th->ractor, th->ec); - } - return TlsSetValue(ruby_native_thread_key, th); -} - -void -Init_native_thread(rb_thread_t *main_th) -{ - if ((ruby_current_ec_key = TlsAlloc()) == TLS_OUT_OF_INDEXES) { - rb_bug("TlsAlloc() for ruby_current_ec_key fails"); - } - if ((ruby_native_thread_key = TlsAlloc()) == TLS_OUT_OF_INDEXES) { - rb_bug("TlsAlloc() for ruby_native_thread_key fails"); - } - - // setup main thread - - ruby_thread_set_native(main_th); - main_th->nt->interrupt_event = CreateEvent(0, TRUE, FALSE, 0); - - DuplicateHandle(GetCurrentProcess(), - GetCurrentThread(), - GetCurrentProcess(), - &main_th->nt->thread_id, 0, FALSE, DUPLICATE_SAME_ACCESS); - - RUBY_DEBUG_LOG("initial thread th:%u thid:%p, event: %p", - rb_th_serial(main_th), - main_th->nt->thread_id, - main_th->nt->interrupt_event); -} - -void -ruby_mn_threads_params(void) -{ -} - -static int -w32_wait_events(HANDLE *events, int count, DWORD timeout, rb_thread_t *th) -{ - HANDLE *targets = events; - HANDLE intr; - const int initcount = count; - DWORD ret; - - w32_event_debug("events:%p, count:%d, timeout:%ld, th:%u\n", - events, count, timeout, th ? rb_th_serial(th) : UINT_MAX); - - if (th && (intr = th->nt->interrupt_event)) { - if (ResetEvent(intr) && (!RUBY_VM_INTERRUPTED(th->ec) || SetEvent(intr))) { - targets = ALLOCA_N(HANDLE, count + 1); - memcpy(targets, events, sizeof(HANDLE) * count); - - targets[count++] = intr; - w32_event_debug("handle:%p (count:%d, intr)\n", intr, count); - } - else if (intr == th->nt->interrupt_event) { - w32_error("w32_wait_events"); - } - } - - w32_event_debug("WaitForMultipleObjects start count:%d\n", count); - ret = WaitForMultipleObjects(count, targets, FALSE, timeout); - w32_event_debug("WaitForMultipleObjects end ret:%lu\n", ret); - - if (ret == (DWORD)(WAIT_OBJECT_0 + initcount) && th) { - errno = EINTR; - } - if (ret == WAIT_FAILED && W32_EVENT_DEBUG) { - int i; - DWORD dmy; - for (i = 0; i < count; i++) { - w32_event_debug("i:%d %s\n", i, GetHandleInformation(targets[i], &dmy) ? "OK" : "NG"); - } - } - return ret; -} - -static void ubf_handle(void *ptr); -#define ubf_select ubf_handle - -int -rb_w32_wait_events_blocking(HANDLE *events, int num, DWORD timeout) -{ - return w32_wait_events(events, num, timeout, ruby_thread_from_native()); -} - -int -rb_w32_wait_events(HANDLE *events, int num, DWORD timeout) -{ - int ret; - rb_thread_t *th = GET_THREAD(); - - BLOCKING_REGION(th, ret = rb_w32_wait_events_blocking(events, num, timeout), - ubf_handle, ruby_thread_from_native(), FALSE); - return ret; -} - static void w32_close_handle(HANDLE handle) { @@ -290,90 +163,9 @@ w32_close_handle(HANDLE handle) } } -static void -w32_resume_thread(HANDLE handle) -{ - if (ResumeThread(handle) == (DWORD)-1) { - w32_error("w32_resume_thread"); - } -} - -#ifdef _MSC_VER -#define HAVE__BEGINTHREADEX 1 -#else -#undef HAVE__BEGINTHREADEX -#endif - -#ifdef HAVE__BEGINTHREADEX -#define start_thread (HANDLE)_beginthreadex -#define thread_errno errno -typedef unsigned long (__stdcall *w32_thread_start_func)(void*); -#else -#define start_thread CreateThread -#define thread_errno rb_w32_map_errno(GetLastError()) -typedef LPTHREAD_START_ROUTINE w32_thread_start_func; -#endif - -static HANDLE -w32_create_thread(DWORD stack_size, w32_thread_start_func func, void *val) -{ - return start_thread(0, stack_size, func, val, CREATE_SUSPENDED | STACK_SIZE_PARAM_IS_A_RESERVATION, 0); -} - -int -rb_w32_sleep(unsigned long msec) -{ - return w32_wait_events(0, 0, msec, ruby_thread_from_native()); -} - -int WINAPI -rb_w32_Sleep(unsigned long msec) -{ - int ret; - rb_thread_t *th = GET_THREAD(); - - BLOCKING_REGION(th, ret = rb_w32_sleep(msec), - ubf_handle, ruby_thread_from_native(), FALSE); - return ret; -} - -static DWORD -hrtime2msec(rb_hrtime_t hrt) -{ - return (DWORD)hrt / (DWORD)RB_HRTIME_PER_MSEC; -} - -static void -native_sleep(rb_thread_t *th, rb_hrtime_t *rel) -{ - const volatile DWORD msec = rel ? hrtime2msec(*rel) : INFINITE; - - THREAD_BLOCKING_BEGIN(th); - { - DWORD ret; - - rb_native_mutex_lock(&th->interrupt_lock); - th->unblock.func = ubf_handle; - th->unblock.arg = th; - rb_native_mutex_unlock(&th->interrupt_lock); - - if (RUBY_VM_INTERRUPTED(th->ec)) { - /* interrupted. return immediate */ - } - else { - RUBY_DEBUG_LOG("start msec:%lu", msec); - ret = w32_wait_events(0, 0, msec, th); - RUBY_DEBUG_LOG("done ret:%lu", ret); - (void)ret; - } - - rb_native_mutex_lock(&th->interrupt_lock); - th->unblock.func = 0; - th->unblock.arg = 0; - rb_native_mutex_unlock(&th->interrupt_lock); - } - THREAD_BLOCKING_END(th); -} +/* ------------------------------------------------------------------------- + * native mutex / condition variable + * ------------------------------------------------------------------------- */ void rb_native_mutex_lock(rb_nativethread_lock_t *lock) @@ -510,75 +302,38 @@ rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex) native_cond_timedwait_ms(cond, mutex, INFINITE); } -static unsigned long -abs_timespec_to_timeout_ms(const struct timespec *ts) +void +rb_native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, unsigned long msec) { - struct timeval tv; - struct timeval now; - - gettimeofday(&now, NULL); - tv.tv_sec = ts->tv_sec; - tv.tv_usec = ts->tv_nsec / 1000; - - if (!rb_w32_time_subtract(&tv, &now)) - return 0; - - return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); + native_cond_timedwait_ms(cond, mutex, msec); } -static int -native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, const struct timespec *ts) +// The scheduler parks threads with an absolute deadline; on this platform the +// wait itself is relative, so the conversion happens here. +static rb_hrtime_t +native_cond_timeout(rb_nativethread_cond_t *cond, const rb_hrtime_t rel) { - unsigned long timeout_ms; - - timeout_ms = abs_timespec_to_timeout_ms(ts); - if (!timeout_ms) - return ETIMEDOUT; - - return native_cond_timedwait_ms(cond, mutex, timeout_ms); + if (rel > 0) { + rb_hrtime_t now = rb_hrtime_now(); + return (rel > RB_HRTIME_MAX - now) ? RB_HRTIME_MAX : now + rel; + } + return rb_hrtime_now(); } -static struct timespec native_cond_timeout(rb_nativethread_cond_t *cond, struct timespec timeout_rel); - -void -rb_native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, unsigned long msec) +static int +native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, const rb_hrtime_t *abs) { - struct timespec rel = { - .tv_sec = msec / 1000, - .tv_nsec = (msec % 1000) * 1000 * 1000, - }; - struct timespec ts = native_cond_timeout(cond, rel); - native_cond_timedwait(cond, mutex, &ts); -} + rb_hrtime_t now = rb_hrtime_now(); -static struct timespec -native_cond_timeout(rb_nativethread_cond_t *cond, struct timespec timeout_rel) -{ - int ret; - struct timeval tv; - struct timespec timeout; - struct timespec now; - - ret = gettimeofday(&tv, 0); - if (ret != 0) - rb_sys_fail(0); - now.tv_sec = tv.tv_sec; - now.tv_nsec = tv.tv_usec * 1000; - - timeout.tv_sec = now.tv_sec; - timeout.tv_nsec = now.tv_nsec; - timeout.tv_sec += timeout_rel.tv_sec; - timeout.tv_nsec += timeout_rel.tv_nsec; - - if (timeout.tv_nsec >= 1000*1000*1000) { - timeout.tv_sec++; - timeout.tv_nsec -= 1000*1000*1000; - } + if (*abs <= now) return ETIMEDOUT; - if (timeout.tv_sec < now.tv_sec) - timeout.tv_sec = TIMET_MAX; + rb_hrtime_t rel = *abs - now; + unsigned long msec = (unsigned long)(rel / RB_HRTIME_PER_MSEC); - return timeout; + // do not busy loop on a sub-millisecond deadline + if (msec == 0) msec = 1; + + return native_cond_timedwait_ms(cond, mutex, msec); } void @@ -594,109 +349,292 @@ rb_native_cond_destroy(rb_nativethread_cond_t *cond) /* */ } +/* ------------------------------------------------------------------------- + * thread local storage + * ------------------------------------------------------------------------- */ -#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < 0x602 -/* declared in processthreadsapi.h only when _WIN32_WINNT >= 0x0602, - * but exported from kernel32.dll since Windows 8 */ -WINBASEAPI VOID WINAPI GetCurrentThreadStackLimits(PULONG_PTR, PULONG_PTR); -#endif +rb_thread_t * +ruby_thread_from_native(void) +{ + return TlsGetValue(ruby_native_thread_key); +} -static void -native_thread_init_stack(rb_thread_t *th, void *local_in_parent_frame) +int +ruby_thread_set_native(rb_thread_t *th) { - ULONG_PTR low, high; - SIZE_T size, space; + if (th) { + ccan_list_node_init(&th->sched.node.ubf); + } - /* VirtualQuery against the current stack pointer may return a region - * that does not span the whole stack when the interpreter is - * initialized deep in the stack, which makes stack_check() misfire. - * [Bug #11438] */ - GetCurrentThreadStackLimits(&low, &high); - size = high - low; - space = size / 5; - if (space > 1024*1024) space = 1024*1024; - th->ec->machine.stack_start = (VALUE *)high - 1; - th->ec->machine.stack_maxsize = size - space; + if (th && th->ec) { + rb_ractor_set_current_ec(th->ractor, th->ec); + } + return TlsSetValue(ruby_native_thread_key, th); } -static void -native_thread_destroy_atfork(struct rb_native_thread *nt) +/* ------------------------------------------------------------------------- + * waiting on Windows objects, with interruption + * ------------------------------------------------------------------------- */ + +static int +w32_wait_events(HANDLE *events, int count, DWORD timeout, rb_thread_t *th) { - /* no-op */ -} + HANDLE *targets = events; + HANDLE intr; + const int initcount = count; + DWORD ret; + + w32_event_debug("events:%p, count:%d, timeout:%ld, th:%u\n", + events, count, timeout, th ? rb_th_serial(th) : UINT_MAX); + + if (th && (intr = th->nt->interrupt_event)) { + if (ResetEvent(intr) && (!RUBY_VM_INTERRUPTED(th->ec) || SetEvent(intr))) { + targets = ALLOCA_N(HANDLE, count + 1); + memcpy(targets, events, sizeof(HANDLE) * count); + + targets[count++] = intr; + w32_event_debug("handle:%p (count:%d, intr)\n", intr, count); + } + else if (intr == th->nt->interrupt_event) { + w32_error("w32_wait_events"); + } + } + + w32_event_debug("WaitForMultipleObjects start count:%d\n", count); + ret = WaitForMultipleObjects(count, targets, FALSE, timeout); + w32_event_debug("WaitForMultipleObjects end ret:%lu\n", ret); + + if (ret == (DWORD)(WAIT_OBJECT_0 + initcount) && th) { + errno = EINTR; + } + if (ret == WAIT_FAILED && W32_EVENT_DEBUG) { + int i; + DWORD dmy; + for (i = 0; i < count; i++) { + w32_event_debug("i:%d %s\n", i, GetHandleInformation(targets[i], &dmy) ? "OK" : "NG"); + } + } + return ret; +} + +int +rb_w32_wait_events_blocking(HANDLE *events, int num, DWORD timeout) +{ + return w32_wait_events(events, num, timeout, ruby_thread_from_native()); +} + +int +rb_w32_wait_events(HANDLE *events, int num, DWORD timeout) +{ + int ret; + rb_thread_t *th = GET_THREAD(); + + BLOCKING_REGION(th, ret = rb_w32_wait_events_blocking(events, num, timeout), + ubf_select, ruby_thread_from_native(), FALSE); + return ret; +} + +int +rb_w32_sleep(unsigned long msec) +{ + return w32_wait_events(0, 0, msec, ruby_thread_from_native()); +} + +int WINAPI +rb_w32_Sleep(unsigned long msec) +{ + int ret; + rb_thread_t *th = GET_THREAD(); + + BLOCKING_REGION(th, ret = rb_w32_sleep(msec), + ubf_select, ruby_thread_from_native(), FALSE); + return ret; +} + +/* @internal */ +int +rb_w32_check_interrupt(rb_thread_t *th) +{ + return w32_wait_events(0, 0, 0, th); +} + +/* + * Pull the target thread out of a blocking w32_wait_events(). This is the + * counterpart of the SIGVTALRM the POSIX implementation sends. + */ +static void +native_thread_interrupt(rb_thread_t *th) +{ + // the caller (ubf_wakeup_thread) logs this + if (!SetEvent(th->nt->interrupt_event)) { + w32_error("native_thread_interrupt"); + } +} + +/* ------------------------------------------------------------------------- + * native thread + * ------------------------------------------------------------------------- */ + +static void +w32_resume_thread(HANDLE handle) +{ + if (ResumeThread(handle) == (DWORD)-1) { + w32_error("w32_resume_thread"); + } +} + +#ifdef _MSC_VER +#define HAVE__BEGINTHREADEX 1 +#else +#undef HAVE__BEGINTHREADEX +#endif + +#ifdef HAVE__BEGINTHREADEX +#define start_thread (HANDLE)_beginthreadex +#define thread_errno errno +typedef unsigned long (__stdcall *w32_thread_start_func)(void*); +#else +#define start_thread CreateThread +#define thread_errno rb_w32_map_errno(GetLastError()) +typedef LPTHREAD_START_ROUTINE w32_thread_start_func; +#endif + +static HANDLE +w32_create_thread(DWORD stack_size, w32_thread_start_func func, void *val) +{ + return start_thread(0, stack_size, func, val, CREATE_SUSPENDED | STACK_SIZE_PARAM_IS_A_RESERVATION, 0); +} + +static void +native_thread_join(HANDLE th) +{ + w32_wait_events(&th, 1, INFINITE, 0); +} + +#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < 0x602 +/* declared in processthreadsapi.h only when _WIN32_WINNT >= 0x0602, + * but exported from kernel32.dll since Windows 8 */ +WINBASEAPI VOID WINAPI GetCurrentThreadStackLimits(PULONG_PTR, PULONG_PTR); +#endif + +static void +native_thread_init_stack(rb_thread_t *th, void *local_in_parent_frame) +{ + ULONG_PTR low, high; + SIZE_T size, space; + + /* VirtualQuery against the current stack pointer may return a region + * that does not span the whole stack when the interpreter is + * initialized deep in the stack, which makes stack_check() misfire. + * [Bug #11438] */ + GetCurrentThreadStackLimits(&low, &high); + size = high - low; + space = size / 5; + if (space > 1024*1024) space = 1024*1024; + th->ec->machine.stack_start = (VALUE *)high - 1; + th->ec->machine.stack_maxsize = size - space; +} + +static void +native_thread_setup(struct rb_native_thread *nt) +{ + rb_native_cond_initialize(&nt->readyq); + rb_native_mutex_initialize(&nt->running_th_lock); + + // Created here rather than on the new thread itself: ubf can fire before + // that thread gets a chance to run. + nt->interrupt_event = CreateEvent(0, TRUE, FALSE, 0); + if (nt->interrupt_event == NULL) { + w32_error("native_thread_setup"); + } +} + +static void +native_thread_setup_on_thread(struct rb_native_thread *nt) +{ + // nothing to do: there is no altstack and no thread id to cache +} + +static struct rb_native_thread * +native_thread_alloc(void) +{ + struct rb_native_thread *nt = ZALLOC(struct rb_native_thread); + native_thread_setup(nt); + +#if USE_RUBY_DEBUG_LOG + static rb_atomic_t nt_serial = 2; + nt->serial = RUBY_ATOMIC_FETCH_ADD(nt_serial, 1); +#endif + return nt; +} + +static void +native_thread_destroy_atfork(struct rb_native_thread *nt) +{ + /* no fork() on this platform */ +} #ifndef InterlockedExchangePointer #define InterlockedExchangePointer(t, v) \ (void *)InterlockedExchange((long *)(t), (long)(v)) #endif + static void native_thread_destroy(struct rb_native_thread *nt) { if (nt) { HANDLE intr = InterlockedExchangePointer(&nt->interrupt_event, 0); RUBY_DEBUG_LOG("close handle intr:%p, thid:%p\n", intr, nt->thread_id); - w32_close_handle(intr); + if (intr) w32_close_handle(intr); + + rb_native_cond_destroy(&nt->readyq); + rb_native_mutex_destroy(&nt->running_th_lock); + + ruby_xfree(nt); } } -static unsigned long __stdcall -thread_start_func_1(void *th_ptr) +static void +native_thread_destroy_self(struct rb_native_thread *nt) { - rb_thread_t *th = th_ptr; - volatile HANDLE thread_id = th->nt->thread_id; - - native_thread_init_stack(th, &th); - th->nt->interrupt_event = CreateEvent(0, TRUE, FALSE, 0); - - /* run */ - RUBY_DEBUG_LOG("thread created th:%u, thid: %p, event: %p", - rb_th_serial(th), th->nt->thread_id, th->nt->interrupt_event); + native_thread_destroy(nt); +} - thread_sched_to_running(TH_SCHED(th), th); - ruby_thread_set_native(th); +static unsigned long __stdcall +nt_start_trampoline(void *nt_ptr) +{ + struct rb_native_thread *nt = (struct rb_native_thread *)nt_ptr; + HANDLE thread_id = nt->thread_id; - // kick threads - thread_start_func_2(th, th->ec->machine.stack_start); + nt_start(nt); w32_close_handle(thread_id); - RUBY_DEBUG_LOG("thread deleted th:%u", rb_th_serial(th)); - return 0; } static int -native_thread_create(rb_thread_t *th) +native_thread_create0(struct rb_native_thread *nt) { - // setup nt - const size_t stack_size = th->vm->default_params.thread_machine_stack_size; - th->nt = ZALLOC(struct rb_native_thread); - th->nt->thread_id = w32_create_thread(stack_size, thread_start_func_1, th); + const size_t stack_size = nt->vm->default_params.thread_machine_stack_size; - // setup vm stack - size_t vm_stack_word_size = th->vm->default_params.thread_vm_stack_size / sizeof(VALUE); - void *vm_stack = ruby_xmalloc(vm_stack_word_size * sizeof(VALUE)); - th->sched.vm_stack = vm_stack; - rb_ec_initialize_vm_stack(th->ec, vm_stack, vm_stack_word_size); - - if ((th->nt->thread_id) == 0) { + nt->thread_id = w32_create_thread(stack_size, nt_start_trampoline, nt); + if (nt->thread_id == 0) { return thread_errno; } - w32_resume_thread(th->nt->thread_id); + w32_resume_thread(nt->thread_id); - if (USE_RUBY_DEBUG_LOG) { - Sleep(0); - RUBY_DEBUG_LOG("th:%u thid:%p intr:%p), stack size: %"PRIuSIZE"", - rb_th_serial(th), th->nt->thread_id, - th->nt->interrupt_event, stack_size); - } + RUBY_DEBUG_LOG("nt:%u thid:%p stack size:%"PRIuSIZE"", + nt->serial, nt->thread_id, stack_size); return 0; } -static void -native_thread_join(HANDLE th) +static int +native_thread_default_max_cpu(void) { - w32_wait_events(&th, 1, INFINITE, 0); + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwNumberOfProcessors > 0 ? (int)si.dwNumberOfProcessors : 8; } #if USE_NATIVE_THREAD_PRIORITY @@ -741,66 +679,166 @@ native_fd_select(int n, rb_fdset_t *readfds, rb_fdset_t *writefds, rb_fdset_t *e return rb_w32_select_with_thread(n, r, w, e, timeout, th); } -/* @internal */ -int -rb_w32_check_interrupt(rb_thread_t *th) +int rb_w32_set_thread_description(HANDLE th, const WCHAR *name); +int rb_w32_set_thread_description_str(HANDLE th, VALUE name); +#define native_set_another_thread_name rb_w32_set_thread_description_str + +static void +native_set_thread_name(rb_thread_t *th) { - return w32_wait_events(0, 0, 0, th); } -static void -ubf_handle(void *ptr) +static VALUE +native_thread_native_thread_id(rb_thread_t *th) { - rb_thread_t *th = (rb_thread_t *)ptr; - RUBY_DEBUG_LOG("th:%u\n", rb_th_serial(th)); + DWORD tid = GetThreadId(th->nt->thread_id); + if (tid == 0) rb_sys_fail("GetThreadId"); + return ULONG2NUM(tid); +} +#define USE_NATIVE_THREAD_NATIVE_THREAD_ID 1 - if (!SetEvent(th->nt->interrupt_event)) { - w32_error("ubf_handle"); +void +Init_native_thread(rb_thread_t *main_th) +{ + if ((ruby_current_ec_key = TlsAlloc()) == TLS_OUT_OF_INDEXES) { + rb_bug("TlsAlloc() for ruby_current_ec_key fails"); + } + if ((ruby_native_thread_key = TlsAlloc()) == TLS_OUT_OF_INDEXES) { + rb_bug("TlsAlloc() for ruby_native_thread_key fails"); } + + // setup vm + rb_vm_t *vm = main_th->vm; + thread_sched_init_vm(vm); + + // setup main thread + native_thread_setup(main_th->nt); + DuplicateHandle(GetCurrentProcess(), + GetCurrentThread(), + GetCurrentProcess(), + &main_th->nt->thread_id, 0, FALSE, DUPLICATE_SAME_ACCESS); + main_th->nt->serial = 1; + ruby_thread_set_native(main_th); + + TH_SCHED(main_th)->running = main_th; + main_th->has_dedicated_nt = 1; + + // setup main NT (before the record below: its kind decides where it goes) + main_th->nt->dedicated = 1; + main_th->nt->running_thread = main_th; + main_th->nt->vm = vm; + + thread_sched_setup_running_threads(TH_SCHED(main_th), main_th->ractor, vm, main_th, NULL); + +#if USE_RUBY_DEBUG_LOG + vm->ractor.sched.dnt_cnt = 1; +#endif + + RUBY_DEBUG_LOG("initial thread th:%u thid:%p, event: %p", + rb_th_serial(main_th), + main_th->nt->thread_id, + main_th->nt->interrupt_event); } -int rb_w32_set_thread_description(HANDLE th, const WCHAR *name); -int rb_w32_set_thread_description_str(HANDLE th, VALUE name); -#define native_set_another_thread_name rb_w32_set_thread_description_str +/* ------------------------------------------------------------------------- + * timer thread + * ------------------------------------------------------------------------- */ static struct { - HANDLE id; - HANDLE lock; -} timer_thread; -#define TIMER_THREAD_CREATED_P() (timer_thread.id != 0) + rb_serial_t created_fork_gen; + HANDLE thread_id; + HANDLE wakeup_event; // manual reset; the "comm pipe" of this platform +} timer_th = { + .created_fork_gen = 0, +}; -static unsigned long __stdcall -timer_thread_func(void *dummy) +#define TIMER_THREAD_CREATED_P() (timer_th.created_fork_gen == current_fork_gen) + +static void +timer_thread_wakeup_force(void) { - rb_vm_t *vm = GET_VM(); - RUBY_DEBUG_LOG("start"); - rb_w32_set_thread_description(GetCurrentThread(), L"ruby-timer-thread"); - while (WaitForSingleObject(timer_thread.lock, - TIME_QUANTUM_USEC/1000) == WAIT_TIMEOUT) { - vm->clock++; - rb_threadptr_check_signal(vm->ractor.main_thread); + if (timer_th.wakeup_event) { + SetEvent(timer_th.wakeup_event); } - RUBY_DEBUG_LOG("end"); - return 0; } void rb_thread_wakeup_timer_thread(int sig) { - /* do nothing */ + timer_thread_wakeup_force(); + + if (RUBY_ATOMIC_LOAD(system_working)) { + rb_vm_t *vm = GET_VM(); + rb_thread_t *main_th = vm->ractor.main_thread; + + if (main_th) { + volatile rb_execution_context_t *main_th_ec = ACCESS_ONCE(rb_execution_context_t *, main_th->ec); + + if (main_th_ec) { + RUBY_VM_SET_TRAP_INTERRUPT(main_th_ec); + + if (vm->ubf_async_safe && main_th->unblock.func) { + (main_th->unblock.func)(main_th->unblock.arg); + } + } + } + } +} + +// The blocking part of the timer thread loop: this is what +// timer_thread_wakeup_force() interrupts. +static void +timer_thread_polling(rb_vm_t *vm) +{ + int timeout = timer_thread_set_timeout(vm); + DWORD msec = (timeout < 0) ? INFINITE : (DWORD)timeout; + + DWORD ret = WaitForSingleObject(timer_th.wakeup_event, msec); + + switch (ret) { + case WAIT_TIMEOUT: + ractor_sched_lock(vm, NULL); + { + timer_thread_check_timeslice(vm); + } + ractor_sched_unlock(vm, NULL); + break; + + case WAIT_OBJECT_0: + ResetEvent(timer_th.wakeup_event); + break; + + default: + w32_error("timer_thread_polling"); + } +} + +static unsigned long __stdcall +timer_thread_trampoline(void *vm_ptr) +{ + rb_w32_set_thread_description(GetCurrentThread(), L"ruby-timer-thread"); + timer_thread_func(vm_ptr); + return 0; } static void rb_thread_create_timer_thread(void) { - if (timer_thread.id == 0) { - if (!timer_thread.lock) { - timer_thread.lock = CreateEvent(0, TRUE, FALSE, 0); + timer_th.created_fork_gen = current_fork_gen; + + if (timer_th.wakeup_event == NULL) { + timer_th.wakeup_event = CreateEvent(0, TRUE, FALSE, 0); + if (timer_th.wakeup_event == NULL) { + w32_error("rb_thread_create_timer_thread"); } - timer_thread.id = w32_create_thread(1024 + (USE_RUBY_DEBUG_LOG ? BUFSIZ : 0), - timer_thread_func, 0); - w32_resume_thread(timer_thread.id); } + + timer_th.thread_id = w32_create_thread(1024 + (USE_RUBY_DEBUG_LOG ? BUFSIZ : 0), + timer_thread_trampoline, GET_VM()); + if (timer_th.thread_id == 0) { + rb_bug("rb_thread_create_timer_thread: failed to create the timer thread"); + } + w32_resume_thread(timer_th.thread_id); } static int @@ -808,10 +846,11 @@ native_stop_timer_thread(void) { RUBY_ATOMIC_SET(system_working, 0); - SetEvent(timer_thread.lock); - native_thread_join(timer_thread.id); - CloseHandle(timer_thread.lock); - timer_thread.lock = 0; + timer_thread_wakeup_force(); + native_thread_join(timer_th.thread_id); + + w32_close_handle(timer_th.wakeup_event); + timer_th.wakeup_event = NULL; return 1; } @@ -819,12 +858,67 @@ native_stop_timer_thread(void) static void native_reset_timer_thread(void) { - if (timer_thread.id) { - CloseHandle(timer_thread.id); - timer_thread.id = 0; + if (timer_th.thread_id) { + CloseHandle(timer_th.thread_id); + timer_th.thread_id = 0; } } +/* ------------------------------------------------------------------------- + * M:N scheduler stubs + * + * These are what thread_sched_mn.c provides on platforms that have an event + * backend. Every thread here is dedicated, so the scheduler never reaches + * the ones that rb_bug(). + * ------------------------------------------------------------------------- */ + +static int +native_thread_create_shared(rb_thread_t *th) +{ + rb_bug("unreachable"); +} + +static enum thread_sched_wait_result +thread_sched_wait_events(struct rb_thread_sched *sched, rb_thread_t *th, int fd, + enum thread_sched_waiting_flag events, rb_hrtime_t *rel) +{ + return thread_sched_wait_unavailable; +} + +static bool +ractor_sched_timeout_arm(rb_thread_t *th, const rb_hrtime_t *rel) +{ + rb_bug("unreachable"); +} + +static bool +ractor_sched_timeout_disarm(rb_thread_t *th) +{ + rb_bug("unreachable"); +} + +static int +timer_wheel_timeout(int timeout) +{ + return timeout; // no M:N threads, no timed waiters +} + +static void +timer_thread_wake_fence(rb_thread_t *th) +{ + // no timer wheel, no wake batches +} + +static void +timer_thread_check_timeout(rb_vm_t *vm) +{ + // no M:N threads, no timed waiters +} + +/* ------------------------------------------------------------------------- + * misc + * ------------------------------------------------------------------------- */ + int ruby_stack_overflowed_p(const rb_thread_t *th, const void *addr) { @@ -856,6 +950,7 @@ ruby_alloca_chkstk(size_t len, void *sp) } } #endif + int rb_reserved_fd_p(int fd) { @@ -868,175 +963,10 @@ rb_nativethread_self(void) return GetCurrentThread(); } -static void -native_set_thread_name(rb_thread_t *th) -{ -} - -static VALUE -native_thread_native_thread_id(rb_thread_t *th) -{ - DWORD tid = GetThreadId(th->nt->thread_id); - if (tid == 0) rb_sys_fail("GetThreadId"); - return ULONG2NUM(tid); -} -#define USE_NATIVE_THREAD_NATIVE_THREAD_ID 1 - -void -rb_add_running_thread(rb_thread_t *th) -{ - // do nothing -} - -void -rb_del_running_thread(rb_thread_t *th) -{ - // do nothing -} - -static bool -th_has_dedicated_nt(const rb_thread_t *th) -{ - return true; -} - -void -rb_threadptr_sched_free(rb_thread_t *th) -{ - native_thread_destroy(th->nt); - ruby_xfree(th->nt); - ruby_xfree(th->sched.vm_stack); -} - - - -static bool -vm_barrier_finish_p(rb_vm_t *vm) -{ - RUBY_DEBUG_LOG("cnt:%u living:%u blocking:%u", - vm->ractor.blocking_cnt == vm->ractor.cnt, - vm->ractor.sync.barrier_cnt, - vm->ractor.cnt, - vm->ractor.blocking_cnt); - - VM_ASSERT(vm->ractor.blocking_cnt <= vm->ractor.cnt); - - return vm->ractor.blocking_cnt == vm->ractor.cnt; -} - -void -rb_ractor_sched_barrier_start(rb_vm_t *vm, rb_ractor_t *cr) -{ - vm->ractor.sync.barrier_waiting = true; - - RUBY_DEBUG_LOG("barrier start. cnt:%u living:%u blocking:%u", - vm->ractor.sync.barrier_cnt, - vm->ractor.cnt, - vm->ractor.blocking_cnt); - - rb_vm_ractor_blocking_cnt_inc(vm, cr, __FILE__, __LINE__); - - // send signal - rb_ractor_t *r = 0; - ccan_list_for_each(&vm->ractor.set, r, vmlr_node) { - if (r != cr) { - rb_ractor_vm_barrier_interrupt_running_thread(r); - } - } - - // wait - while (!vm_barrier_finish_p(vm)) { - rb_vm_cond_wait(vm, &vm->ractor.sync.barrier_complete_cond); - } - - RUBY_DEBUG_LOG("cnt:%u barrier success", vm->ractor.sync.barrier_cnt); - - rb_vm_ractor_blocking_cnt_dec(vm, cr, __FILE__, __LINE__); - - vm->ractor.sync.barrier_waiting = false; - vm->ractor.sync.barrier_cnt++; - - rb_native_cond_broadcast(&vm->ractor.sync.barrier_release_cond); -} - -void -rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr) -{ - vm->ractor.sync.lock_owner = cr; - unsigned int barrier_cnt = vm->ractor.sync.barrier_cnt; - rb_thread_t *th = GET_THREAD(); - bool running; - - RB_VM_SAVE_MACHINE_CONTEXT(th); - - if (rb_ractor_status_p(cr, ractor_running)) { - rb_vm_ractor_blocking_cnt_inc(vm, cr, __FILE__, __LINE__); - running = true; - } - else { - running = false; - } - VM_ASSERT(rb_ractor_status_p(cr, ractor_blocking)); - - if (vm_barrier_finish_p(vm)) { - RUBY_DEBUG_LOG("wakeup barrier owner"); - rb_native_cond_signal(&vm->ractor.sync.barrier_complete_cond); - } - else { - RUBY_DEBUG_LOG("wait for barrier finish"); - } - - // wait for restart - while (barrier_cnt == vm->ractor.sync.barrier_cnt) { - rb_vm_cond_wait(vm, &vm->ractor.sync.barrier_release_cond); - } - - RUBY_DEBUG_LOG("barrier is released. Acquire vm_lock"); - - if (running) { - rb_vm_ractor_blocking_cnt_dec(vm, cr, __FILE__, __LINE__); - } - - vm->ractor.sync.lock_owner = NULL; -} - -bool -rb_thread_lock_native_thread(void) -{ - return false; -} - void * rb_thread_prevent_fork(void *(*func)(void *), void *data) { return func(data); } -void -rb_thread_malloc_stack_set(rb_thread_t *th, void *stack, size_t stack_size) -{ - // no-op -} - #endif /* THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION */ - -void -rb_thread_sched_winding_begin(rb_vm_t *vm) -{ - // nothing to count: rb_thread_sched_wait_winding below never waits - (void)vm; -} - -void -rb_thread_sched_winding_end(rb_vm_t *vm) -{ - (void)vm; -} - -void -rb_thread_sched_wait_winding(rb_vm_t *vm) -{ - // no coroutine (M:N) threads on this implementation: nothing winds down - // after leaving the living set (see thread_pthread.c) - (void)vm; -} diff --git a/thread_win32.h b/thread_win32.h index 23cd71fcfe08d9..402cdbde37bd95 100644 --- a/thread_win32.h +++ b/thread_win32.h @@ -8,6 +8,9 @@ Copyright (C) 2004-2007 Koichi Sasada + This platform runs the common scheduler; see thread_sched.h for its data + structures and thread_sched.c for the implementation. + **********************************************************************/ /* interface */ @@ -16,23 +19,10 @@ # undef _WIN32 # endif -#define USE_VM_CLOCK 1 - WINBASEAPI BOOL WINAPI TryEnterCriticalSection(IN OUT LPCRITICAL_SECTION lpCriticalSection); -struct rb_native_thread { - HANDLE thread_id; - HANDLE interrupt_event; -}; - -struct rb_thread_sched_item { - void *vm_stack; -}; - -struct rb_thread_sched { - HANDLE lock; -}; +#include "thread_sched.h" typedef DWORD native_tls_key_t; // TLS index diff --git a/vm.c b/vm.c index d2d1f55d5e88d0..95a75af8ca31ea 100644 --- a/vm.c +++ b/vm.c @@ -4903,10 +4903,6 @@ Init_BareVM(void) vm_opt_method_def_table = st_init_numtable(); vm_opt_mid_table = st_init_numtable(); -#ifdef RUBY_THREAD_WIN32_H - rb_native_cond_initialize(&vm->ractor.sync.barrier_complete_cond); - rb_native_cond_initialize(&vm->ractor.sync.barrier_release_cond); -#endif } void diff --git a/vm_core.h b/vm_core.h index 8d3eba50200b7a..2c416859573d63 100644 --- a/vm_core.h +++ b/vm_core.h @@ -590,11 +590,10 @@ struct rb_iseq_constant_body { /* typedef rb_iseq_t is in method.h */ struct rb_iseq_struct { VALUE flags; /* 1 */ - VALUE wrapper; /* 2 */ - struct rb_iseq_constant_body *body; /* 3 */ + struct rb_iseq_constant_body *body; /* 2 */ - union { /* 4, 5 words */ + union { /* 3, 4 words */ struct iseq_compile_data *compile_data; /* used at compile time */ struct { @@ -727,69 +726,14 @@ typedef struct rb_vm_struct { // join at exit rb_nativethread_cond_t terminate_cond; bool terminate_waiting; - -#ifndef RUBY_THREAD_PTHREAD_H - // win32 - bool barrier_waiting; - unsigned int barrier_cnt; - rb_nativethread_cond_t barrier_complete_cond; - rb_nativethread_cond_t barrier_release_cond; -#endif } sync; /* VM-wide locks for the Ractor transfer/inheritance machinery. All of them * are leaf locks: no safepoint inside a critical section. */ rb_nativethread_lock_t generic_fields_lock; /* the shared generic-fields table in variable.c */ -#ifdef RUBY_THREAD_PTHREAD_H - // ractor scheduling - struct { - rb_nativethread_lock_t lock; - struct rb_ractor_struct *lock_owner; - bool locked; - - rb_nativethread_cond_t cond; // GRQ - rb_atomic_t snt_cnt; // count of shared NTs; lock-free (see native_thread_dedicated_inc) - unsigned int dnt_cnt; // count of dedicated NTs; logging only (USE_RUBY_DEBUG_LOG), not atomic - - - unsigned int max_cpu; - struct ccan_list_head grq; // // Global Ready Queue - rb_atomic_t winding_cnt; // native threads between a coroutine epilogue and its reclaim; ruby_vm_destruct waits for 0 - unsigned int grq_cnt; - - // What the barrier walk visits: threads running on dedicated - // nts, and the shared nts (whose running_th fields hold the rest). - struct { - rb_nativethread_lock_t lock; - struct ccan_list_head running_dnts; - struct ccan_list_head snts; - } ntlist; - - // scheds whose readyq holds waiters: the timer ticks their - // running thread (timeslice_scan) and prunes drained entries. - struct { - rb_nativethread_lock_t lock; - struct ccan_list_head scheds; - } timeslice; - - // true if timeslice timer is not enable - bool timeslice_wait_inf; - - // barrier - rb_nativethread_cond_t barrier_complete_cond; - rb_nativethread_cond_t barrier_release_cond; - // bool; nonzero while a stop-the-world section is active. Set - // before the barrier walks the running records; a record moved - // after the walk sees it (thread_sched_setup_running_threads). - rb_atomic_t barrier_is_waiting; - unsigned int barrier_joined_cnt; // threads joined so far; under sched.lock - unsigned int barrier_running_cnt; // runners counted by the barrier's walk; under sched.lock - unsigned int barrier_serial; - struct rb_ractor_struct *barrier_ractor; - unsigned int barrier_lock_rec; - } sched; -#endif + // ractor scheduling; see thread_sched.h + struct rb_ractor_sched sched; } ractor; #ifdef USE_SIGALTSTACK diff --git a/vm_sync.c b/vm_sync.c index 40adafe09290e1..7b6a587a1442f6 100644 --- a/vm_sync.c +++ b/vm_sync.c @@ -65,21 +65,13 @@ rb_vm_locked_p(void) static bool vm_need_barrier_waiting(const rb_vm_t *vm) { -#ifdef RUBY_THREAD_PTHREAD_H return vm->ractor.sched.barrier_is_waiting; -#else - return vm->ractor.sync.barrier_waiting; -#endif } static bool vm_need_barrier(bool no_barrier, const rb_ractor_t *cr, const rb_vm_t *vm) { -#ifdef RUBY_THREAD_PTHREAD_H return !no_barrier && cr->threads.sched.running != NULL && vm_need_barrier_waiting(vm); // ractor has running threads. -#else - return !no_barrier && vm_need_barrier_waiting(vm); -#endif } static void @@ -142,13 +134,11 @@ vm_lock_leave(rb_vm_t *vm, bool no_barrier, unsigned int *lev APPEND_LOCATION_AR VM_ASSERT(vm->ractor.sync.lock_rec == *lev); VM_ASSERT(cr == GET_RACTOR()); -#ifdef RUBY_THREAD_PTHREAD_H if (vm->ractor.sched.barrier_ractor == cr && vm->ractor.sched.barrier_lock_rec == vm->ractor.sync.lock_rec) { VM_ASSERT(!no_barrier); rb_ractor_sched_barrier_end(vm, cr); } -#endif if (RUBY_DTRACE_GVL_RELEASE_ENABLED()) { RUBY_DTRACE_GVL_RELEASE(); @@ -258,11 +248,7 @@ rb_vm_cond_timedwait(rb_vm_t *vm, rb_nativethread_cond_t *cond, unsigned long ms static bool vm_barrier_acquired_p(const rb_vm_t *vm, const rb_ractor_t *cr) { -#ifdef RUBY_THREAD_PTHREAD_H return vm->ractor.sched.barrier_ractor == cr; -#else - return false; -#endif } void diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index ca22b04274281b..9e2946ea4030e6 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1289,7 +1289,6 @@ pub union rb_iseq_constant_body__bindgen_ty_1 { #[repr(C)] pub struct rb_iseq_struct { pub flags: VALUE, - pub wrapper: VALUE, pub body: *mut rb_iseq_constant_body, pub aux: rb_iseq_struct__bindgen_ty_1, } diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 842d0770ef855c..0b3665c3182fa5 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -1766,7 +1766,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, m@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :m, v20 (0x1038), num_args=1 - PatchPoint MethodRedefined(NilClass@0x1060, nil?@0x1068, cme:0x1070) + PatchPoint MethodRedefined(NilClass@0x1058, nil?@0x1060, cme:0x1068) v52:Fixnum[0] = Const Value(0) CheckInterrupts v84:Fixnum[0] = Const Value(0) @@ -1801,7 +1801,7 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(C@0x1008) PatchPoint MethodRedefined(C@0x1008, fun_new_map@0x1010, cme:0x1018) v25:ArraySubclass[class_exact:C] = GuardType v10, ArraySubclass[class_exact:C] recompile - v26:BasicObject = SendDirect v25, 0x1040, :fun_new_map (0x1068) + v26:BasicObject = SendDirect v25, 0x1040, :fun_new_map (0x1060) PatchPoint NoEPEscape(test) CheckInterrupts Return v26 @@ -1991,7 +1991,7 @@ mod hir_opt_tests { PushInlineFrame :foo, v18 (0x1038), num_args=0 v26:Fixnum[1] = Const Value(1) v34:Fixnum[2] = Const Value(2) - PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) + PatchPoint MethodRedefined(Integer@0x1058, +@0x1060, cme:0x1068) v61:Fixnum[3] = Const Value(3) CheckInterrupts PopInlineFrame @@ -2022,7 +2022,7 @@ mod hir_opt_tests { v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v20 (0x1038), num_args=1 v28:Fixnum[2] = Const Value(2) - PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) + PatchPoint MethodRedefined(Integer@0x1058, +@0x1060, cme:0x1068) v54:Fixnum[5] = Const Value(5) CheckInterrupts PopInlineFrame @@ -2053,7 +2053,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v22 (0x1038), num_args=2 - PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) + PatchPoint MethodRedefined(Integer@0x1058, +@0x1060, cme:0x1068) v47:Fixnum[7] = Const Value(7) CheckInterrupts PopInlineFrame @@ -2097,7 +2097,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v28:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v28 (0x1038), num_args=0 - PatchPoint StableConstantNames(0x1060, DEBUG) + PatchPoint StableConstantNames(0x1058, DEBUG) v58:NilClass = Const Value(nil) CheckInterrupts PopInlineFrame @@ -2137,7 +2137,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v28:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v28 (0x1038), num_args=0 - PatchPoint StableConstantNames(0x1060, CALL_BLOCK) + PatchPoint StableConstantNames(0x1058, CALL_BLOCK) v61:NilClass = Const Value(nil) CheckInterrupts PopInlineFrame @@ -4143,16 +4143,16 @@ mod hir_opt_tests { v52:NilClass = Const Value(nil) PushInlineFrame :foo, v22 (0x1038), num_args=2 v34:CPtr = GetEP 0 - v35:CUInt64 = LoadField v34, :VM_ENV_DATA_INDEX_FLAGS@0x1060 + v35:CUInt64 = LoadField v34, :VM_ENV_DATA_INDEX_FLAGS@0x1058 v36:CBool = IsBlockParamModified v35 CondBranch v36, bb6(), bb7() bb6(): - v38:BasicObject = LoadField v34, :block@0x1061 + v38:BasicObject = LoadField v34, :block@0x1059 Jump bb8(v38, v38) bb7(): - v40:CInt64 = LoadField v34, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 + v40:CInt64 = LoadField v34, :VM_ENV_DATA_INDEX_SPECVAL@0x105a v41:CInt64 = GuardAnyBitSet v40, CUInt64(1) recompile - v42:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) + v42:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1060)) Jump bb8(v42, v52) bb8(v32:BasicObject, v33:BasicObject): v47:BasicObject = Send v32, :call, v11, v13 # SendFallbackReason: Send: unsupported optimized method type BlockCall @@ -4190,10 +4190,10 @@ mod hir_opt_tests { v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v18 (0x1038), num_args=0 v25:CPtr = GetEP 0 - v26:CInt64 = LoadField v25, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 + v26:CInt64 = LoadField v25, :VM_ENV_DATA_INDEX_SPECVAL@0x1058 v27:CInt64[-4] = Const CInt64(-4) v28:CInt64 = IntAnd v26, v27 - v29:BasicObject = InvokeBlockIseqDirect (0x1068), v28 + v29:BasicObject = InvokeBlockIseqDirect (0x1060), v28 CheckInterrupts PopInlineFrame Return v29 @@ -4229,11 +4229,11 @@ mod hir_opt_tests { v29:Fixnum[1] = Const Value(1) v31:Fixnum[2] = Const Value(2) v33:CPtr = GetEP 0 - v34:CInt64 = LoadField v33, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 + v34:CInt64 = LoadField v33, :VM_ENV_DATA_INDEX_SPECVAL@0x1058 v35:CInt64[-4] = Const CInt64(-4) v36:CInt64 = IntAnd v34, v35 - v37:BasicObject = InvokeBlockIseqDirect (0x1068), v36, v29, v31 - PatchPoint MethodRedefined(Integer@0x1090, +@0x1098, cme:0x10a0) + v37:BasicObject = InvokeBlockIseqDirect (0x1060), v36, v29, v31 + PatchPoint MethodRedefined(Integer@0x1080, +@0x1088, cme:0x1090) v52:Fixnum = GuardType v37, Fixnum v53:Fixnum = FixnumAdd v11, v52 CheckInterrupts @@ -4322,10 +4322,10 @@ mod hir_opt_tests { v37:Fixnum[7] = Const Value(7) v39:Fixnum[8] = Const Value(8) v41:CPtr = GetEP 0 - v42:CInt64 = LoadField v41, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 + v42:CInt64 = LoadField v41, :VM_ENV_DATA_INDEX_SPECVAL@0x1058 v43:CInt64[-4] = Const CInt64(-4) v44:CInt64 = IntAnd v42, v43 - v45:BasicObject = InvokeBlockIseqDirect (0x1068), v44, v25, v27, v29, v31, v33, v35, v37, v39 + v45:BasicObject = InvokeBlockIseqDirect (0x1060), v44, v25, v27, v29, v31, v33, v35, v37, v39 CheckInterrupts PopInlineFrame Return v45 @@ -4685,7 +4685,7 @@ mod hir_opt_tests { v44:BasicObject = CCallWithFrame v43, :Kernel#lambda@0x1040, block=0x1048 v22:CPtr = GetEP 0 v23:BasicObject = LoadField v22, :a@0x1000 - PatchPoint MethodRedefined(Object@0x1008, foo@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Object@0x1008, foo@0x1068, cme:0x1070) v34:CPtr = GetEP 0 v35:BasicObject = LoadField v34, :a@0x1000 CheckInterrupts @@ -4719,8 +4719,8 @@ mod hir_opt_tests { v24:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile v25:ArrayExact = NewArray v11, v13, v15 PushInlineFrame :foo, v24 (0x1038), num_args=1 - PatchPoint NoSingletonClass(Array@0x1060) - PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + PatchPoint NoSingletonClass(Array@0x1058) + PatchPoint MethodRedefined(Array@0x1058, length@0x1060, cme:0x1068) v49:CInt64 = ArrayLength v25 v50:Fixnum = BoxFixnum v49 CheckInterrupts @@ -4759,8 +4759,8 @@ mod hir_opt_tests { v32:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile v33:ArrayExact = NewArray v11, v13, v15, v17, v19, v21, v23 PushInlineFrame :foo, v32 (0x1038), num_args=1 - PatchPoint NoSingletonClass(Array@0x1060) - PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + PatchPoint NoSingletonClass(Array@0x1058) + PatchPoint MethodRedefined(Array@0x1058, length@0x1060, cme:0x1068) v57:CInt64 = ArrayLength v33 v58:Fixnum = BoxFixnum v57 CheckInterrupts @@ -4795,15 +4795,15 @@ mod hir_opt_tests { v24:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile v25:ArrayExact = NewArray v11, v13, v15 PushInlineFrame :foo, v24 (0x1038), num_args=1 - PatchPoint NoSingletonClass(Array@0x1060) - PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + PatchPoint NoSingletonClass(Array@0x1058) + PatchPoint MethodRedefined(Array@0x1058, length@0x1060, cme:0x1068) v55:CInt64 = ArrayLength v25 v56:Fixnum = BoxFixnum v55 v38:CPtr = GetEP 0 - v39:CInt64 = LoadField v38, :VM_ENV_DATA_INDEX_SPECVAL@0x1098 + v39:CInt64 = LoadField v38, :VM_ENV_DATA_INDEX_SPECVAL@0x1090 v40:CInt64[-4] = Const CInt64(-4) v41:CInt64 = IntAnd v39, v40 - v42:BasicObject = InvokeBlockIseqDirect (0x10a0), v41, v56 + v42:BasicObject = InvokeBlockIseqDirect (0x1098), v41, v56 CheckInterrupts PopInlineFrame Return v42 @@ -4838,20 +4838,20 @@ mod hir_opt_tests { v57:NilClass = Const Value(nil) PushInlineFrame :foo, v24 (0x1038), num_args=1 v37:CPtr = GetEP 0 - v38:CUInt64 = LoadField v37, :VM_ENV_DATA_INDEX_FLAGS@0x1060 + v38:CUInt64 = LoadField v37, :VM_ENV_DATA_INDEX_FLAGS@0x1058 v39:CBool = IsBlockParamModified v38 CondBranch v39, bb6(), bb7() bb6(): - v41:BasicObject = LoadField v37, :block@0x1061 + v41:BasicObject = LoadField v37, :block@0x1059 Jump bb8(v41, v41) bb7(): - v43:CInt64 = LoadField v37, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 + v43:CInt64 = LoadField v37, :VM_ENV_DATA_INDEX_SPECVAL@0x105a v44:CInt64 = GuardAnyBitSet v43, CUInt64(1) recompile - v45:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) + v45:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1060)) Jump bb8(v45, v57) bb8(v35:BasicObject, v36:BasicObject): - PatchPoint NoSingletonClass(Array@0x1070) - PatchPoint MethodRedefined(Array@0x1070, length@0x1078, cme:0x1080) + PatchPoint NoSingletonClass(Array@0x1068) + PatchPoint MethodRedefined(Array@0x1068, length@0x1070, cme:0x1078) v66:CInt64 = ArrayLength v25 v67:Fixnum = BoxFixnum v66 v52:BasicObject = Send v35, :call, v67 # SendFallbackReason: Send: unsupported optimized method type BlockCall @@ -4888,11 +4888,11 @@ mod hir_opt_tests { v26:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile v27:ArrayExact = NewArray v13, v15 PushInlineFrame :foo, v26 (0x1038), num_args=3 - PatchPoint NoSingletonClass(Array@0x1060) - PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + PatchPoint NoSingletonClass(Array@0x1058) + PatchPoint MethodRedefined(Array@0x1058, length@0x1060, cme:0x1068) v61:CInt64 = ArrayLength v27 v62:Fixnum = BoxFixnum v61 - PatchPoint MethodRedefined(Integer@0x1098, +@0x10a0, cme:0x10a8) + PatchPoint MethodRedefined(Integer@0x1090, +@0x1098, cme:0x10a0) v66:Fixnum = FixnumAdd v62, v11 v70:Fixnum = FixnumAdd v66, v17 CheckInterrupts @@ -4928,11 +4928,11 @@ mod hir_opt_tests { v25:ArrayExact = NewArray v11, v13 v47:Fixnum[0] = Const Value(0) PushInlineFrame :foo, v24 (0x1038), num_args=2 - PatchPoint NoSingletonClass(Array@0x1060) - PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + PatchPoint NoSingletonClass(Array@0x1058) + PatchPoint MethodRedefined(Array@0x1058, length@0x1060, cme:0x1068) v56:CInt64 = ArrayLength v25 v57:Fixnum = BoxFixnum v56 - PatchPoint MethodRedefined(Integer@0x1098, +@0x10a0, cme:0x10a8) + PatchPoint MethodRedefined(Integer@0x1090, +@0x1098, cme:0x10a0) v61:Fixnum = FixnumAdd v57, v15 CheckInterrupts PopInlineFrame @@ -4967,11 +4967,11 @@ mod hir_opt_tests { v24:Fixnum[40] = Const Value(40) v46:Fixnum[0] = Const Value(0) PushInlineFrame :foo, v22 (0x1038), num_args=2 - PatchPoint NoSingletonClass(Array@0x1060) - PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + PatchPoint NoSingletonClass(Array@0x1058) + PatchPoint MethodRedefined(Array@0x1058, length@0x1060, cme:0x1068) v55:CInt64 = ArrayLength v23 v56:Fixnum = BoxFixnum v55 - PatchPoint MethodRedefined(Integer@0x1098, +@0x10a0, cme:0x10a8) + PatchPoint MethodRedefined(Integer@0x1090, +@0x1098, cme:0x10a0) v60:Fixnum = FixnumAdd v56, v24 CheckInterrupts PopInlineFrame @@ -5069,7 +5069,7 @@ mod hir_opt_tests { v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v22 (0x1038), num_args=2 v31:Fixnum[80] = Const Value(80) - PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) + PatchPoint MethodRedefined(Integer@0x1058, +@0x1060, cme:0x1068) v67:Fixnum[110] = Const Value(110) CheckInterrupts PopInlineFrame @@ -5366,9 +5366,9 @@ mod hir_opt_tests { v21:StaticSymbol[:k] = Const Value(VALUE(0x1038)) v22:HashExact = NewHash v21: v11 PushInlineFrame :foo, v20 (0x1040), num_args=1 - PatchPoint NoSingletonClass(Hash@0x1068) - PatchPoint MethodRedefined(Hash@0x1068, class@0x1070, cme:0x1078) - v44:ClassSubclass[Hash@0x1068] = Const Value(VALUE(0x1068)) + PatchPoint NoSingletonClass(Hash@0x1060) + PatchPoint MethodRedefined(Hash@0x1060, class@0x1068, cme:0x1070) + v44:ClassSubclass[Hash@0x1060] = Const Value(VALUE(0x1060)) CheckInterrupts PopInlineFrame Return v44 @@ -5871,7 +5871,7 @@ mod hir_opt_tests { v38:Fixnum[0] = Const Value(0) PushInlineFrame :foo, v18 (0x1038), num_args=1 v30:Fixnum[1] = Const Value(1) - PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) + PatchPoint MethodRedefined(Integer@0x1058, +@0x1060, cme:0x1068) v47:Fixnum[2] = Const Value(2) CheckInterrupts PopInlineFrame @@ -6169,12 +6169,12 @@ mod hir_opt_tests { PatchPoint MethodRedefined(C@0x1008, initialize@0x1038, cme:0x1040) PushInlineFrame :initialize, v45 (0x1068), num_args=1 PatchPoint SingleRactorMode - v63:CShape = LoadField v45, :shape_id@0x1090 - v64:CShape[0x1091] = GuardBitEquals v63, CShape(0x1091) recompile - StoreField v45, :@x@0x1092, v15 + v63:CShape = LoadField v45, :shape_id@0x1088 + v64:CShape[0x1089] = GuardBitEquals v63, CShape(0x1089) recompile + StoreField v45, :@x@0x108a, v15 WriteBarrier v45, v15 - v67:CShape[0x1093] = Const CShape(0x1093) - StoreField v45, :shape_id@0x1090, v67 + v67:CShape[0x108b] = Const CShape(0x108b) + StoreField v45, :shape_id@0x1088, v67 CheckInterrupts PopInlineFrame Return v45 @@ -6275,12 +6275,12 @@ mod hir_opt_tests { v41:ObjectSubclass[class_exact:Factory] = GuardType v12, ObjectSubclass[class_exact:Factory] recompile PushInlineFrame :new, v41 (0x1048), num_args=0 v48:NilClass = Const Value(nil) - PatchPoint StableConstantNames(0x1070, Object) - v51:ClassSubclass[Object@0x1078] = Const Value(VALUE(0x1078)) - PatchPoint MethodRedefined(Object@0x1078, new@0x1018, cme:0x1080) - v84:ObjectExact = ObjectAllocClass Object:VALUE(0x1078) - PatchPoint NoSingletonClass(Object@0x1078) - PatchPoint MethodRedefined(Object@0x1078, initialize@0x10a8, cme:0x10b0) + PatchPoint StableConstantNames(0x1068, Object) + v51:ClassSubclass[Object@0x1070] = Const Value(VALUE(0x1070)) + PatchPoint MethodRedefined(Object@0x1070, new@0x1018, cme:0x1078) + v84:ObjectExact = ObjectAllocClass Object:VALUE(0x1070) + PatchPoint NoSingletonClass(Object@0x1070) + PatchPoint MethodRedefined(Object@0x1070, initialize@0x10a0, cme:0x10a8) CheckInterrupts PopInlineFrame Return v84 @@ -6346,11 +6346,11 @@ mod hir_opt_tests { PushInlineFrame :initialize, v42 (0x1068), num_args=1 v62:TrueClass = Const Value(true) v80:CPtr = GetEP 0 - v81:CUInt64 = LoadField v80, :VM_ENV_DATA_INDEX_FLAGS@0x1090 + v81:CUInt64 = LoadField v80, :VM_ENV_DATA_INDEX_FLAGS@0x1088 v82:CBool = IsBlockParamModified v81 CondBranch v82, bb11(), bb12() bb11(): - v84:BasicObject = LoadField v80, :block@0x1091 + v84:BasicObject = LoadField v80, :block@0x1089 Jump bb13(v84) bb12(): v86:BasicObject = GetBlockParam :block, l0, EP@4 @@ -6768,7 +6768,7 @@ mod hir_opt_tests { PopInlineFrame Return v76 bb4(): - v50:StaticSymbol[:skip] = Const Value(VALUE(0x1068)) + v50:StaticSymbol[:skip] = Const Value(VALUE(0x1060)) CheckInterrupts Return v50 "); @@ -11081,7 +11081,7 @@ mod hir_opt_tests { v11:ArrayExact = ArrayDup v10 PatchPoint NoSingletonClass(Array@0x1008) PatchPoint MethodRedefined(Array@0x1008, map@0x1010, cme:0x1018) - v22:BasicObject = SendDirect v11, 0x1040, :map (0x1068) + v22:BasicObject = SendDirect v11, 0x1040, :map (0x1060) CheckInterrupts Return v22 "); @@ -11279,10 +11279,10 @@ mod hir_opt_tests { PushInlineFrame :foo, v18 (0x1038), num_args=0 v25:Fixnum[1] = Const Value(1) v27:CPtr = GetEP 0 - v28:CInt64 = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 + v28:CInt64 = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1058 v29:CInt64[-4] = Const CInt64(-4) v30:CInt64 = IntAnd v28, v29 - v31:BasicObject = InvokeBlockIseqDirect (0x1068), v30, v25 + v31:BasicObject = InvokeBlockIseqDirect (0x1060), v30, v25 CheckInterrupts PopInlineFrame Return v31 @@ -11317,14 +11317,14 @@ mod hir_opt_tests { v71:NilClass = Const Value(nil) PushInlineFrame :foo, v18 (0x1038), num_args=0 v28:CPtr = GetEP 0 - v29:CUInt64 = LoadField v28, :VM_ENV_DATA_INDEX_FLAGS@0x1060 + v29:CUInt64 = LoadField v28, :VM_ENV_DATA_INDEX_FLAGS@0x1058 v30:CBool = IsBlockParamModified v29 CondBranch v30, bb7(), bb8() bb7(): - v32:BasicObject = LoadField v28, :blk@0x1061 + v32:BasicObject = LoadField v28, :blk@0x1059 Jump bb9(v32, v32) bb8(): - v34:CInt64 = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 + v34:CInt64 = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x105a v35:CInt64[0] = GuardBitEquals v34, CInt64(0) recompile v36:NilClass = Const Value(nil) Jump bb9(v36, v71) @@ -11333,16 +11333,16 @@ mod hir_opt_tests { CondBranch v39, bb10(), bb6() bb10(): v46:CPtr = GetEP 0 - v47:CUInt64 = LoadField v46, :VM_ENV_DATA_INDEX_FLAGS@0x1060 + v47:CUInt64 = LoadField v46, :VM_ENV_DATA_INDEX_FLAGS@0x1058 v48:CBool = IsBlockParamModified v47 CondBranch v48, bb11(), bb12() bb11(): - v50:BasicObject = LoadField v46, :blk@0x1061 + v50:BasicObject = LoadField v46, :blk@0x1059 Jump bb13(v50, v50) bb12(): - v52:CInt64 = LoadField v46, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 + v52:CInt64 = LoadField v46, :VM_ENV_DATA_INDEX_SPECVAL@0x105a v53:CInt64 = GuardAnyBitSet v52, CUInt64(1) recompile - v54:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) + v54:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1060)) Jump bb13(v54, v27) bb13(v44:BasicObject, v45:BasicObject): v57:BasicObject = Send v44, :call # SendFallbackReason: Send: no profile data available @@ -17337,7 +17337,7 @@ mod hir_opt_tests { v21:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1048 v22:FalseClass = GuardBitEquals v21, Value(false) PushInlineFrame :foo, v6 (0x1050), num_args=0 - v29:StringExact[VALUE(0x1078)] = Const Value(VALUE(0x1078)) + v29:StringExact[VALUE(0x1070)] = Const Value(VALUE(0x1070)) v30:StringExact = StringCopy v29 CheckInterrupts PopInlineFrame @@ -17412,13 +17412,13 @@ mod hir_opt_tests { v32:FalseClass = GuardBitEquals v31, Value(false) PushInlineFrame :foo, v9 (0x1058), num_args=1 v45:Fixnum[2] = Const Value(2) - PatchPoint MethodRedefined(Integer@0x1080, *@0x1088, cme:0x1090) + PatchPoint MethodRedefined(Integer@0x1078, *@0x1080, cme:0x1088) v59:Fixnum = GuardType v10, Fixnum recompile v60:Fixnum = FixnumMult v59, v45 CheckInterrupts PopInlineFrame v18:Fixnum[1] = Const Value(1) - PatchPoint MethodRedefined(Integer@0x1080, +@0x10b8, cme:0x10c0) + PatchPoint MethodRedefined(Integer@0x1078, +@0x10b0, cme:0x10b8) v37:Fixnum = FixnumAdd v60, v18 Return v37 "); @@ -17818,7 +17818,7 @@ mod hir_opt_tests { SetLocal :other_block, l0, EP@3, v43 v30:CPtr = GetEP 0 v31:BasicObject = LoadField v30, :other_block@0x1002 - v33:BasicObject = InvokeSuper v42, 0x1070, v31 # SendFallbackReason: super: complex argument passing to `super` call + v33:BasicObject = InvokeSuper v42, 0x1068, v31 # SendFallbackReason: super: complex argument passing to `super` call CheckInterrupts Return v33 "); @@ -18571,14 +18571,14 @@ mod hir_opt_tests { SetLocal :sep, l0, EP@5, v131 Jump bb8(v90, v131, v92, v93) bb8(v98:BasicObject, v99:BasicObject, v100:BasicObject, v101:BasicObject): - PatchPoint StableConstantNames(0x1078, CONST) - v106:HashExact[VALUE(0x1080)] = Const Value(VALUE(0x1080)) + PatchPoint StableConstantNames(0x1070, CONST) + v106:HashExact[VALUE(0x1078)] = Const Value(VALUE(0x1078)) SetLocal :kwsplat, l0, EP@3, v106 v111:CPtr = GetEP 0 v112:BasicObject = LoadField v111, :list@0x1001 v114:CPtr = GetEP 0 v115:BasicObject = LoadField v114, :iter_method@0x1005 - v117:BasicObject = Send v112, 0x1088, :__send__, v115 # SendFallbackReason: Send: unsupported method type Optimized + v117:BasicObject = Send v112, 0x1080, :__send__, v115 # SendFallbackReason: Send: unsupported method type Optimized v118:CPtr = GetEP 0 v119:BasicObject = LoadField v118, :list@0x1001 v120:BasicObject = LoadField v118, :sep@0x1002 @@ -18667,7 +18667,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v18 (0x1038), num_args=0 - v27:StringExact[VALUE(0x1060)] = Const Value(VALUE(0x1060)) + v27:StringExact[VALUE(0x1058)] = Const Value(VALUE(0x1058)) CheckInterrupts PopInlineFrame Return v27 @@ -18741,10 +18741,10 @@ mod hir_opt_tests { v92:CInt64 = UnboxFixnum v65 v93:BasicObject = ArrayAref v11, v92 v95:CPtr = GetEP 0 - v96:CInt64 = LoadField v95, :VM_ENV_DATA_INDEX_SPECVAL@0x1068 + v96:CInt64 = LoadField v95, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 v97:CInt64[-4] = Const CInt64(-4) v98:CInt64 = IntAnd v96, v97 - v99:BasicObject = InvokeBlockIseqDirect (0x1070), v98, v93 + v99:BasicObject = InvokeBlockIseqDirect (0x1068), v98, v93 v103:Fixnum[1] = Const Value(1) v104:Fixnum = FixnumAdd v65, v103 PatchPoint NoEPEscape(each) @@ -18981,13 +18981,13 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, greet_recompile@0x1010, cme:0x1018) v42:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :greet_recompile, v42 (0x1040), num_args=1 - PatchPoint MethodRedefined(Integer@0x1068, to_s@0x1070, cme:0x1078) - v63:StringExact = CCallVariadic v22, :Integer#to_s@0x10a0 + PatchPoint MethodRedefined(Integer@0x1060, to_s@0x1068, cme:0x1070) + v63:StringExact = CCallVariadic v22, :Integer#to_s@0x1098 CheckInterrupts PopInlineFrame Return v63 bb4(): - v34:StringExact[VALUE(0x10a8)] = Const Value(VALUE(0x10a8)) + v34:StringExact[VALUE(0x10a0)] = Const Value(VALUE(0x10a0)) v35:StringExact = StringCopy v34 CheckInterrupts Return v35 @@ -20316,7 +20316,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, double@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :double, v23 (0x1040), num_args=1 - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v45:Fixnum = GuardType v10, Fixnum recompile v47:Fixnum = FixnumAdd v45, v45 CheckInterrupts @@ -20364,7 +20364,7 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :clamp_nonneg, v23 (0x1040), num_args=1 v32:Fixnum[0] = Const Value(0) - PatchPoint MethodRedefined(Integer@0x1068, <@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, <@0x1068, cme:0x1070) v62:Fixnum = GuardType v10, Fixnum recompile v63:BoolExact = FixnumLt v62, v32 v37:CBool = Test v63 @@ -20481,7 +20481,7 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :add_one, v23 (0x1040), num_args=1 v32:Fixnum[1] = Const Value(1) - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v46:Fixnum = GuardType v10, Fixnum recompile v47:Fixnum = FixnumAdd v46, v32 CheckInterrupts @@ -20542,8 +20542,8 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, outer@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :outer, v23 (0x1040), num_args=1 - PatchPoint MethodRedefined(Object@0x1008, inner@0x1068, cme:0x1070) - v44:BasicObject = SendDirect v23, 0x0, :inner (0x1098), v10 + PatchPoint MethodRedefined(Object@0x1008, inner@0x1060, cme:0x1068) + v44:BasicObject = SendDirect v23, 0x0, :inner (0x1090), v10 CheckInterrupts PopInlineFrame Return v44 @@ -20646,7 +20646,7 @@ mod hir_opt_tests { PushInlineFrame :add_opts, v23 (0x1040), num_args=1 v32:Fixnum[10] = Const Value(10) v41:Fixnum[100] = Const Value(100) - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v71:Fixnum = GuardType v10, Fixnum recompile v72:Fixnum = FixnumAdd v71, v32 v76:Fixnum = FixnumAdd v72, v41 @@ -20702,7 +20702,7 @@ mod hir_opt_tests { v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :add_opts, v25 (0x1040), num_args=2 v34:Fixnum[100] = Const Value(100) - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v63:Fixnum = GuardType v10, Fixnum recompile v64:Fixnum = FixnumAdd v63, v16 v68:Fixnum = FixnumAdd v64, v34 @@ -20739,7 +20739,7 @@ mod hir_opt_tests { v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile v40:NilClass = Const Value(nil) PushInlineFrame :callee, v18 (0x1038), num_args=0 - v26:StaticSymbol[:default] = Const Value(VALUE(0x1060)) + v26:StaticSymbol[:default] = Const Value(VALUE(0x1058)) CheckInterrupts PopInlineFrame Return v26 @@ -20826,7 +20826,7 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :maybe_rescue, v23 (0x1040), num_args=1 v32:Fixnum[1] = Const Value(1) - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v47:Fixnum = GuardType v10, Fixnum recompile v48:Fixnum = FixnumAdd v47, v32 CheckInterrupts @@ -20923,22 +20923,22 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Child@0x1008, greet@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact:Child] = GuardType v10, ObjectSubclass[class_exact:Child] recompile PushInlineFrame :greet, v23 (0x1040), num_args=0 - PatchPoint MethodRedefined(Parent@0x1068, greet@0x1010, cme:0x1070) + PatchPoint MethodRedefined(Parent@0x1060, greet@0x1010, cme:0x1068) v47:CPtr = GetEP 0 - v48:RubyValue = LoadField v47, :VM_ENV_DATA_INDEX_ME_CREF@0x1098 + v48:RubyValue = LoadField v47, :VM_ENV_DATA_INDEX_ME_CREF@0x1090 v49:CallableMethodEntry[VALUE(0x1018)] = GuardBitEquals v48, Value(VALUE(0x1018)) - v50:RubyValue = LoadField v47, :VM_ENV_DATA_INDEX_SPECVAL@0x1099 + v50:RubyValue = LoadField v47, :VM_ENV_DATA_INDEX_SPECVAL@0x1091 v51:FalseClass = GuardBitEquals v50, Value(false) - PushInlineFrame :greet, v23 (0x10a0), num_args=0 - v63:StringExact[VALUE(0x10c8)] = Const Value(VALUE(0x10c8)) + PushInlineFrame :greet, v23 (0x1098), num_args=0 + v63:StringExact[VALUE(0x10b8)] = Const Value(VALUE(0x10b8)) v64:StringExact = StringCopy v63 CheckInterrupts PopInlineFrame - v33:StringExact[VALUE(0x10d0)] = Const Value(VALUE(0x10d0)) + v33:StringExact[VALUE(0x10c0)] = Const Value(VALUE(0x10c0)) v34:StringExact = StringCopy v33 - PatchPoint NoSingletonClass(String@0x10d8) - PatchPoint MethodRedefined(String@0x10d8, +@0x10e0, cme:0x10e8) - v57:BasicObject = CCallWithFrame v64, :String#+@0x1110, v34 + PatchPoint NoSingletonClass(String@0x10c8) + PatchPoint MethodRedefined(String@0x10c8, +@0x10d0, cme:0x10d8) + v57:BasicObject = CCallWithFrame v64, :String#+@0x1100, v34 CheckInterrupts PopInlineFrame Return v57 @@ -20990,7 +20990,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_opts@0x1010, cme:0x1018) v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :add_opts, v27 (0x1040), num_args=3 - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v55:Fixnum = GuardType v10, Fixnum recompile v56:Fixnum = FixnumAdd v55, v16 v60:Fixnum = FixnumAdd v56, v18 @@ -21046,7 +21046,7 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :add_opt_post, v23 (0x1040), num_args=1 v31:Fixnum[10] = Const Value(10) - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v56:Fixnum = GuardType v10, Fixnum v57:Fixnum = FixnumAdd v31, v56 CheckInterrupts @@ -21102,7 +21102,7 @@ mod hir_opt_tests { v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :add_lead_opt_post, v25 (0x1040), num_args=2 v34:Fixnum[10] = Const Value(10) - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v63:Fixnum = GuardType v10, Fixnum recompile v64:Fixnum = FixnumAdd v63, v34 v68:Fixnum = FixnumAdd v64, v16 @@ -21155,7 +21155,7 @@ mod hir_opt_tests { v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile v43:Fixnum[0] = Const Value(0) PushInlineFrame :add_kw, v25 (0x1040), num_args=2 - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v50:Fixnum = GuardType v10, Fixnum recompile v51:Fixnum = FixnumAdd v50, v16 CheckInterrupts @@ -21207,7 +21207,7 @@ mod hir_opt_tests { v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile v43:Fixnum[0] = Const Value(0) PushInlineFrame :add_optkw, v25 (0x1040), num_args=2 - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v50:Fixnum = GuardType v10, Fixnum recompile v51:Fixnum = FixnumAdd v50, v16 CheckInterrupts @@ -21259,7 +21259,7 @@ mod hir_opt_tests { v24:Fixnum[10] = Const Value(10) v43:Fixnum[0] = Const Value(0) PushInlineFrame :add_optkw, v23 (0x1040), num_args=2 - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v50:Fixnum = GuardType v10, Fixnum recompile v51:Fixnum = FixnumAdd v50, v24 CheckInterrupts @@ -21314,11 +21314,11 @@ mod hir_opt_tests { v61:Fixnum[0] = Const Value(0) PushInlineFrame :add_kws, v27 (0x1040), num_args=3 v40:Fixnum[100] = Const Value(100) - PatchPoint MethodRedefined(Integer@0x1068, *@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, *@0x1068, cme:0x1070) v68:Fixnum = GuardType v10, Fixnum recompile v69:Fixnum = FixnumMult v68, v40 v82:Fixnum[20] = Const Value(20) - PatchPoint MethodRedefined(Integer@0x1068, +@0x10a0, cme:0x10a8) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1098, cme:0x10a0) v77:Fixnum = FixnumAdd v69, v82 v81:Fixnum = FixnumAdd v77, v16 CheckInterrupts @@ -21376,12 +21376,12 @@ mod hir_opt_tests { CondBranch v36, bb6(v24), bb7() bb7(): v42:Fixnum[2] = Const Value(2) - PatchPoint MethodRedefined(Integer@0x1068, *@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, *@0x1068, cme:0x1070) v70:Fixnum = GuardType v10, Fixnum recompile v71:Fixnum = FixnumMult v70, v42 Jump bb6(v71) bb6(v50:NilClass|Fixnum): - PatchPoint MethodRedefined(Integer@0x1068, +@0x10a0, cme:0x10a8) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1098, cme:0x10a0) v74:Fixnum = GuardType v10, Fixnum recompile v75:Fixnum = GuardType v50, Fixnum v76:Fixnum = FixnumAdd v74, v75 @@ -21437,7 +21437,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_lead_opt_post@0x1010, cme:0x1018) v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :add_lead_opt_post, v27 (0x1040), num_args=3 - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v55:Fixnum = GuardType v10, Fixnum recompile v56:Fixnum = FixnumAdd v55, v16 v60:Fixnum = FixnumAdd v56, v18 @@ -21493,10 +21493,10 @@ mod hir_opt_tests { v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :with_yield, v25 (0x1040), num_args=1 v34:CPtr = GetEP 0 - v35:CInt64 = LoadField v34, :VM_ENV_DATA_INDEX_SPECVAL@0x1068 + v35:CInt64 = LoadField v34, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 v36:CInt64[-4] = Const CInt64(-4) v37:CInt64 = IntAnd v35, v36 - v38:BasicObject = InvokeBlockIseqDirect (0x1070), v37, v10 + v38:BasicObject = InvokeBlockIseqDirect (0x1068), v37, v10 CheckInterrupts PopInlineFrame PatchPoint NoEPEscape(test) @@ -21550,16 +21550,16 @@ mod hir_opt_tests { v53:NilClass = Const Value(nil) PushInlineFrame :with_block_param, v25 (0x1040), num_args=1 v36:CPtr = GetEP 0 - v37:CUInt64 = LoadField v36, :VM_ENV_DATA_INDEX_FLAGS@0x1068 + v37:CUInt64 = LoadField v36, :VM_ENV_DATA_INDEX_FLAGS@0x1060 v38:CBool = IsBlockParamModified v37 CondBranch v38, bb6(), bb7() bb6(): - v40:BasicObject = LoadField v36, :block@0x1069 + v40:BasicObject = LoadField v36, :block@0x1061 Jump bb8(v40, v40) bb7(): - v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x106a + v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 v43:CInt64 = GuardAnyBitSet v42, CUInt64(1) recompile - v44:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1070)) + v44:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) Jump bb8(v44, v53) bb8(v34:BasicObject, v35:BasicObject): v48:BasicObject = Send v34, :call, v10 # SendFallbackReason: Send: unsupported optimized method type BlockCall @@ -21614,16 +21614,16 @@ mod hir_opt_tests { v54:NilClass = Const Value(nil) PushInlineFrame :callee, v25 (0x1040), num_args=1 v38:CPtr = GetEP 0 - v39:CUInt64 = LoadField v38, :VM_ENV_DATA_INDEX_FLAGS@0x1068 + v39:CUInt64 = LoadField v38, :VM_ENV_DATA_INDEX_FLAGS@0x1060 v40:CBool = IsBlockParamModified v39 CondBranch v40, bb6(), bb7() bb6(): - v42:BasicObject = LoadField v38, :block@0x1069 + v42:BasicObject = LoadField v38, :block@0x1061 Jump bb8(v42, v42) bb7(): - v44:CInt64 = LoadField v38, :VM_ENV_DATA_INDEX_SPECVAL@0x106a + v44:CInt64 = LoadField v38, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 v45:CInt64 = GuardAnyBitSet v44, CUInt64(1) recompile - v46:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1070)) + v46:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) Jump bb8(v46, v54) bb8(v36:BasicObject, v37:BasicObject): v49:BasicObject = Send v25, &block, :inner, v10, v36 # SendFallbackReason: Send: block argument is not nil @@ -21690,22 +21690,22 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Point@0x1008, initialize@0x1038, cme:0x1040) PushInlineFrame :initialize, v85 (0x1068), num_args=2 PatchPoint SingleRactorMode - v117:CShape = LoadField v85, :shape_id@0x1090 - v118:CShape[0x1091] = GuardBitEquals v117, CShape(0x1091) recompile - StoreField v85, :@x@0x1092, v15 + v117:CShape = LoadField v85, :shape_id@0x1088 + v118:CShape[0x1089] = GuardBitEquals v117, CShape(0x1089) recompile + StoreField v85, :@x@0x108a, v15 WriteBarrier v85, v15 - v121:CShape[0x1093] = Const CShape(0x1093) - StoreField v85, :shape_id@0x1090, v121 + v121:CShape[0x108b] = Const CShape(0x108b) + StoreField v85, :shape_id@0x1088, v121 PatchPoint NoEPEscape(initialize) PatchPoint SingleRactorMode - StoreField v85, :@y@0x1094, v17 + StoreField v85, :@y@0x108c, v17 WriteBarrier v85, v17 - v136:CShape[0x1095] = Const CShape(0x1095) - StoreField v85, :shape_id@0x1090, v136 + v136:CShape[0x108d] = Const CShape(0x108d) + StoreField v85, :shape_id@0x1088, v136 CheckInterrupts PopInlineFrame v42:NilClass = Const Value(nil) - PatchPoint StableConstantNames(0x1098, Point) + PatchPoint StableConstantNames(0x1090, Point) v45:ClassSubclass[Point@0x1008] = Const Value(VALUE(0x1008)) v47:Fixnum[1] = Const Value(1) v49:Fixnum[2] = Const Value(2) @@ -21715,30 +21715,30 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Point@0x1008, initialize@0x1038, cme:0x1040) PushInlineFrame :initialize, v95 (0x1068), num_args=2 PatchPoint SingleRactorMode - v157:CShape = LoadField v95, :shape_id@0x1090 - v158:CShape[0x1091] = GuardBitEquals v157, CShape(0x1091) recompile - StoreField v95, :@x@0x1092, v47 + v157:CShape = LoadField v95, :shape_id@0x1088 + v158:CShape[0x1089] = GuardBitEquals v157, CShape(0x1089) recompile + StoreField v95, :@x@0x108a, v47 WriteBarrier v95, v47 - v161:CShape[0x1093] = Const CShape(0x1093) - StoreField v95, :shape_id@0x1090, v161 + v161:CShape[0x108b] = Const CShape(0x108b) + StoreField v95, :shape_id@0x1088, v161 PatchPoint NoEPEscape(initialize) PatchPoint SingleRactorMode - StoreField v95, :@y@0x1094, v49 + StoreField v95, :@y@0x108c, v49 WriteBarrier v95, v49 - v176:CShape[0x1095] = Const CShape(0x1095) - StoreField v95, :shape_id@0x1090, v176 + v176:CShape[0x108d] = Const CShape(0x108d) + StoreField v95, :shape_id@0x1088, v176 CheckInterrupts PopInlineFrame PatchPoint NoSingletonClass(Point@0x1008) - PatchPoint MethodRedefined(Point@0x1008, ==@0x10a0, cme:0x10a8) - PushInlineFrame :==, v85 (0x10d0), num_args=1 + PatchPoint MethodRedefined(Point@0x1008, ==@0x1098, cme:0x10a0) + PushInlineFrame :==, v85 (0x10c8), num_args=1 PatchPoint SingleRactorMode - v195:CShape = LoadField v85, :shape_id@0x1090 - v196:CShape[0x1095] = GuardBitEquals v195, CShape(0x1095) recompile - v197:BasicObject = LoadField v85, :@x@0x1092 + v195:CShape = LoadField v85, :shape_id@0x1088 + v196:CShape[0x108d] = GuardBitEquals v195, CShape(0x108d) recompile + v197:BasicObject = LoadField v85, :@x@0x108a PatchPoint NoEPEscape(==) - PatchPoint MethodRedefined(Point@0x1008, x@0x10f8, cme:0x1100) - PatchPoint MethodRedefined(Integer@0x1128, ==@0x10a0, cme:0x1130) + PatchPoint MethodRedefined(Point@0x1008, x@0x10e8, cme:0x10f0) + PatchPoint MethodRedefined(Integer@0x1118, ==@0x1098, cme:0x1120) v253:Fixnum = GuardType v197, Fixnum recompile v255:BoolExact = FixnumEq v253, v47 v208:CBool = Test v255 @@ -21746,16 +21746,16 @@ mod hir_opt_tests { CondBranch v208, bb19(), bb18(v209) bb19(): PatchPoint SingleRactorMode - v216:CShape = LoadField v85, :shape_id@0x1090 - v217:CShape[0x1095] = GuardBitEquals v216, CShape(0x1095) recompile - v218:BasicObject = LoadField v85, :@y@0x1094 + v216:CShape = LoadField v85, :shape_id@0x1088 + v217:CShape[0x108d] = GuardBitEquals v216, CShape(0x108d) recompile + v218:BasicObject = LoadField v85, :@y@0x108c PatchPoint NoEPEscape(==) PatchPoint NoSingletonClass(Point@0x1008) - PatchPoint MethodRedefined(Point@0x1008, y@0x1158, cme:0x1160) - v260:CShape = LoadField v95, :shape_id@0x1090 - v261:CShape[0x1095] = GuardBitEquals v260, CShape(0x1095) recompile - v262:BasicObject = LoadField v95, :@y@0x1094 - PatchPoint MethodRedefined(Integer@0x1128, ==@0x10a0, cme:0x1130) + PatchPoint MethodRedefined(Point@0x1008, y@0x1148, cme:0x1150) + v260:CShape = LoadField v95, :shape_id@0x1088 + v261:CShape[0x108d] = GuardBitEquals v260, CShape(0x108d) recompile + v262:BasicObject = LoadField v95, :@y@0x108c + PatchPoint MethodRedefined(Integer@0x1118, ==@0x1098, cme:0x1120) v265:Fixnum = GuardType v218, Fixnum recompile v266:Fixnum = GuardType v262, Fixnum v267:BoolExact = FixnumEq v265, v266 diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs index 6c80b892d08d4f..bbd0dfadb2e140 100644 --- a/zjit/src/hir/tests.rs +++ b/zjit/src/hir/tests.rs @@ -212,9 +212,9 @@ mod snapshot_tests { v44:Fixnum[0] = Const Value(0) v27:Any = Snapshot FrameState { pc: 0x1008, stack: [], locals: [] } PushInlineFrame :foo, v24 (0x1048), num_args=3 - v38:Any = Snapshot FrameState { pc: 0x1070, stack: [v13, v15, v11], locals: [a=v13, b=v15, c=v11, ID(0)=v44], caller: v27 } + v38:Any = Snapshot FrameState { pc: 0x1068, stack: [v13, v15, v11], locals: [a=v13, b=v15, c=v11, ID(0)=v44], caller: v27 } v39:ArrayExact = NewArray v13, v15, v11 - v40:Any = Snapshot FrameState { pc: 0x1078, stack: [v39], locals: [a=v13, b=v15, c=v11, ID(0)=v44], caller: v27 } + v40:Any = Snapshot FrameState { pc: 0x1070, stack: [v39], locals: [a=v13, b=v15, c=v11, ID(0)=v44], caller: v27 } CheckInterrupts PopInlineFrame Return v39 @@ -252,9 +252,9 @@ mod snapshot_tests { v39:Fixnum[0] = Const Value(0) v24:Any = Snapshot FrameState { pc: 0x1008, stack: [], locals: [] } PushInlineFrame :foo, v22 (0x1048), num_args=2 - v33:Any = Snapshot FrameState { pc: 0x1070, stack: [v11, v13], locals: [a=v11, b=v13, ID(0)=v39], caller: v24 } + v33:Any = Snapshot FrameState { pc: 0x1068, stack: [v11, v13], locals: [a=v11, b=v13, ID(0)=v39], caller: v24 } v34:ArrayExact = NewArray v11, v13 - v35:Any = Snapshot FrameState { pc: 0x1078, stack: [v34], locals: [a=v11, b=v13, ID(0)=v39], caller: v24 } + v35:Any = Snapshot FrameState { pc: 0x1070, stack: [v34], locals: [a=v11, b=v13, ID(0)=v39], caller: v24 } CheckInterrupts PopInlineFrame Return v34 @@ -299,9 +299,9 @@ mod snapshot_tests { v64:Fixnum[0] = Const Value(0) v37:Any = Snapshot FrameState { pc: 0x1008, stack: [], locals: [] } PushInlineFrame :foo, v34 (0x1048), num_args=8 - v58:Any = Snapshot FrameState { pc: 0x1070, stack: [v19, v21, v17, v15, v11, v13, v23, v25], locals: [five=v11, six=v13, a=v19, b=v21, c=v17, d=v15, e=v23, f=v25, ID(0)=v64], caller: v37 } + v58:Any = Snapshot FrameState { pc: 0x1068, stack: [v19, v21, v17, v15, v11, v13, v23, v25], locals: [five=v11, six=v13, a=v19, b=v21, c=v17, d=v15, e=v23, f=v25, ID(0)=v64], caller: v37 } v59:ArrayExact = NewArray v19, v21, v17, v15, v11, v13, v23, v25 - v60:Any = Snapshot FrameState { pc: 0x1078, stack: [v59], locals: [five=v11, six=v13, a=v19, b=v21, c=v17, d=v15, e=v23, f=v25, ID(0)=v64], caller: v37 } + v60:Any = Snapshot FrameState { pc: 0x1070, stack: [v59], locals: [five=v11, six=v13, a=v19, b=v21, c=v17, d=v15, e=v23, f=v25, ID(0)=v64], caller: v37 } CheckInterrupts PopInlineFrame Return v59 @@ -2141,7 +2141,7 @@ pub(crate) mod hir_build_tests { v25:BasicObject = Send v10, 0x1000, :foo # SendFallbackReason: Uncategorized(send) PatchPoint NoEPEscape(test) v28:CPtr = LoadSP - v29:BasicObject = LoadField v28, :a@0x1028 + v29:BasicObject = LoadField v28, :a@0x1020 PatchPoint NoEPEscape(test) v38:BasicObject = Send v29, :+, v20 # SendFallbackReason: Uncategorized(opt_plus) CheckInterrupts @@ -2217,8 +2217,8 @@ pub(crate) mod hir_build_tests { PatchPoint NoEPEscape(test) v18:CPtr = LoadSP v19:BasicObject = LoadField v18, :block@0x1000 - PatchPoint StableConstantNames(0x1030, ::RubyVM::ZJIT) - v24:ModuleSubclass[RubyVM::ZJIT@0x1038] = Const Value(VALUE(0x1038)) + PatchPoint StableConstantNames(0x1028, ::RubyVM::ZJIT) + v24:ModuleSubclass[RubyVM::ZJIT@0x1030] = Const Value(VALUE(0x1030)) SideExit DirectiveInduced "); } @@ -2257,8 +2257,8 @@ pub(crate) mod hir_build_tests { v17:Fixnum[1] = Const Value(1) v22:BasicObject = Send v11, 0x1008, :consume # SendFallbackReason: Uncategorized(send) PatchPoint NoEPEscape(test) - PatchPoint StableConstantNames(0x1030, ::RubyVM::ZJIT) - v29:ModuleSubclass[RubyVM::ZJIT@0x1038] = Const Value(VALUE(0x1038)) + PatchPoint StableConstantNames(0x1028, ::RubyVM::ZJIT) + v29:ModuleSubclass[RubyVM::ZJIT@0x1030] = Const Value(VALUE(0x1030)) SideExit DirectiveInduced "); } @@ -2295,16 +2295,16 @@ pub(crate) mod hir_build_tests { v15:BasicObject = Send v9, 0x1008, :consume # SendFallbackReason: Uncategorized(send) PatchPoint NoEPEscape(test) v24:CPtr = GetEP 0 - v25:CUInt64 = LoadField v24, :VM_ENV_DATA_INDEX_FLAGS@0x1030 + v25:CUInt64 = LoadField v24, :VM_ENV_DATA_INDEX_FLAGS@0x1028 v26:CBool = IsBlockParamModified v25 CondBranch v26, bb4(), bb5() bb4(): - v28:BasicObject = LoadField v24, :&@0x1031 + v28:BasicObject = LoadField v24, :&@0x1029 Jump bb6(v28, v28) bb5(): - v30:CInt64 = LoadField v24, :VM_ENV_DATA_INDEX_SPECVAL@0x1032 + v30:CInt64 = LoadField v24, :VM_ENV_DATA_INDEX_SPECVAL@0x102a v31:CInt64 = GuardAnyBitSet v30, CUInt64(1) recompile - v32:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1038)) + v32:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1030)) Jump bb6(v32, v10) bb6(v22:BasicObject, v23:BasicObject): v35:BasicObject = Send v9, &block, :consume, v22 # SendFallbackReason: Uncategorized(send) @@ -2350,7 +2350,7 @@ pub(crate) mod hir_build_tests { v25:BasicObject = Send v10, 0x1000, :foo # SendFallbackReason: Uncategorized(send) PatchPoint NoEPEscape(test) v28:CPtr = LoadSP - v29:BasicObject = LoadField v28, :a@0x1028 + v29:BasicObject = LoadField v28, :a@0x1020 PatchPoint NoEPEscape(test) v38:BasicObject = Send v29, :+, v20 # SendFallbackReason: Uncategorized(opt_plus) CheckInterrupts