From 16de693774e5444c560d4772ac3e5eb3a8267aa6 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 27 Aug 2026 14:59:55 +0900 Subject: [PATCH 1/7] Remove wrapper field in iseq We keep a singleton of iseq wrappers (RubyVM::InstructionSequence). However, this is rarely used and causes 8 bytes to be wasted on all iseq objects. Since we now have the 32-byte slot size in the default GC, we can save that memory. --- iseq.c | 25 +-- test/ruby/test_iseq.rb | 15 -- vm_core.h | 5 +- zjit/src/cruby_bindings.inc.rs | 1 - zjit/src/hir/opt_tests.rs | 310 ++++++++++++++++----------------- zjit/src/hir/tests.rs | 32 ++-- 6 files changed, 178 insertions(+), 210 deletions(-) diff --git a/iseq.c b/iseq.c index a2e71a1d451183..4cffe90f4d56a9 100644 --- a/iseq.c +++ b/iseq.c @@ -407,8 +407,6 @@ rb_iseq_mark_and_move(rb_iseq_t *iseq, bool reference_updating) { RUBY_MARK_ENTER("iseq"); - rb_gc_mark_and_move(&iseq->wrapper); - if (ISEQ_BODY(iseq)) { struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); @@ -1721,24 +1719,11 @@ static const rb_data_type_t iseqw_data_type = { static VALUE iseqw_new(const rb_iseq_t *iseq) { - if (iseq->wrapper) { - if (*(const rb_iseq_t **)rb_check_typeddata(iseq->wrapper, &iseqw_data_type) != iseq) { - rb_raise(rb_eTypeError, "wrong iseq wrapper: %" PRIsVALUE " for %p", - iseq->wrapper, (void *)iseq); - } - return iseq->wrapper; - } - else { - rb_iseq_t **ptr; - VALUE obj = TypedData_Make_Struct(rb_cISeq, rb_iseq_t *, &iseqw_data_type, ptr); - RB_OBJ_WRITE(obj, ptr, iseq); - - /* cache a wrapper object */ - RB_OBJ_SET_FROZEN_SHAREABLE((VALUE)obj); - RB_OBJ_WRITE((VALUE)iseq, &iseq->wrapper, obj); - - return obj; - } + rb_iseq_t **ptr; + VALUE obj = TypedData_Make_Struct(rb_cISeq, rb_iseq_t *, &iseqw_data_type, ptr); + RB_OBJ_WRITE(obj, ptr, iseq); + RB_OBJ_SET_FROZEN_SHAREABLE((VALUE)obj); + return obj; } VALUE diff --git a/test/ruby/test_iseq.rb b/test/ruby/test_iseq.rb index 9d946c8e8efd69..c01bb3dc90bfa1 100644 --- a/test/ruby/test_iseq.rb +++ b/test/ruby/test_iseq.rb @@ -837,21 +837,6 @@ def test_iseq_of } end - def test_iseq_of_twice_for_same_code - [ - proc{}, - method(:test_iseq_of_twice_for_same_code), - RubyVM::InstructionSequence.compile("p 1"), - begin; raise "error"; rescue => error; error.backtrace_locations[0]; end - ].each{|src| - iseq1 = RubyVM::InstructionSequence.of(src) - iseq2 = RubyVM::InstructionSequence.of(src) - - # ISeq objects should be same for same src - assert_equal iseq1.object_id, iseq2.object_id - } - end - def test_iseq_builtin_to_a invokebuiltin = eval(EnvUtil.invoke_ruby(['-e', <<~EOS], '', true).first) insns = RubyVM::InstructionSequence.of([].method(:pack)).to_a.last diff --git a/vm_core.h b/vm_core.h index 8d3eba50200b7a..0e2a22305ee572 100644 --- a/vm_core.h +++ b/vm_core.h @@ -590,11 +590,10 @@ struct rb_iseq_constant_body { /* typedef rb_iseq_t is in method.h */ struct rb_iseq_struct { VALUE flags; /* 1 */ - VALUE wrapper; /* 2 */ - struct rb_iseq_constant_body *body; /* 3 */ + struct rb_iseq_constant_body *body; /* 2 */ - union { /* 4, 5 words */ + union { /* 3, 4 words */ struct iseq_compile_data *compile_data; /* used at compile time */ struct { diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index ca22b04274281b..9e2946ea4030e6 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1289,7 +1289,6 @@ pub union rb_iseq_constant_body__bindgen_ty_1 { #[repr(C)] pub struct rb_iseq_struct { pub flags: VALUE, - pub wrapper: VALUE, pub body: *mut rb_iseq_constant_body, pub aux: rb_iseq_struct__bindgen_ty_1, } diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 842d0770ef855c..0b3665c3182fa5 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -1766,7 +1766,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, m@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :m, v20 (0x1038), num_args=1 - PatchPoint MethodRedefined(NilClass@0x1060, nil?@0x1068, cme:0x1070) + PatchPoint MethodRedefined(NilClass@0x1058, nil?@0x1060, cme:0x1068) v52:Fixnum[0] = Const Value(0) CheckInterrupts v84:Fixnum[0] = Const Value(0) @@ -1801,7 +1801,7 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(C@0x1008) PatchPoint MethodRedefined(C@0x1008, fun_new_map@0x1010, cme:0x1018) v25:ArraySubclass[class_exact:C] = GuardType v10, ArraySubclass[class_exact:C] recompile - v26:BasicObject = SendDirect v25, 0x1040, :fun_new_map (0x1068) + v26:BasicObject = SendDirect v25, 0x1040, :fun_new_map (0x1060) PatchPoint NoEPEscape(test) CheckInterrupts Return v26 @@ -1991,7 +1991,7 @@ mod hir_opt_tests { PushInlineFrame :foo, v18 (0x1038), num_args=0 v26:Fixnum[1] = Const Value(1) v34:Fixnum[2] = Const Value(2) - PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) + PatchPoint MethodRedefined(Integer@0x1058, +@0x1060, cme:0x1068) v61:Fixnum[3] = Const Value(3) CheckInterrupts PopInlineFrame @@ -2022,7 +2022,7 @@ mod hir_opt_tests { v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v20 (0x1038), num_args=1 v28:Fixnum[2] = Const Value(2) - PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) + PatchPoint MethodRedefined(Integer@0x1058, +@0x1060, cme:0x1068) v54:Fixnum[5] = Const Value(5) CheckInterrupts PopInlineFrame @@ -2053,7 +2053,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v22 (0x1038), num_args=2 - PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) + PatchPoint MethodRedefined(Integer@0x1058, +@0x1060, cme:0x1068) v47:Fixnum[7] = Const Value(7) CheckInterrupts PopInlineFrame @@ -2097,7 +2097,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v28:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v28 (0x1038), num_args=0 - PatchPoint StableConstantNames(0x1060, DEBUG) + PatchPoint StableConstantNames(0x1058, DEBUG) v58:NilClass = Const Value(nil) CheckInterrupts PopInlineFrame @@ -2137,7 +2137,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v28:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v28 (0x1038), num_args=0 - PatchPoint StableConstantNames(0x1060, CALL_BLOCK) + PatchPoint StableConstantNames(0x1058, CALL_BLOCK) v61:NilClass = Const Value(nil) CheckInterrupts PopInlineFrame @@ -4143,16 +4143,16 @@ mod hir_opt_tests { v52:NilClass = Const Value(nil) PushInlineFrame :foo, v22 (0x1038), num_args=2 v34:CPtr = GetEP 0 - v35:CUInt64 = LoadField v34, :VM_ENV_DATA_INDEX_FLAGS@0x1060 + v35:CUInt64 = LoadField v34, :VM_ENV_DATA_INDEX_FLAGS@0x1058 v36:CBool = IsBlockParamModified v35 CondBranch v36, bb6(), bb7() bb6(): - v38:BasicObject = LoadField v34, :block@0x1061 + v38:BasicObject = LoadField v34, :block@0x1059 Jump bb8(v38, v38) bb7(): - v40:CInt64 = LoadField v34, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 + v40:CInt64 = LoadField v34, :VM_ENV_DATA_INDEX_SPECVAL@0x105a v41:CInt64 = GuardAnyBitSet v40, CUInt64(1) recompile - v42:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) + v42:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1060)) Jump bb8(v42, v52) bb8(v32:BasicObject, v33:BasicObject): v47:BasicObject = Send v32, :call, v11, v13 # SendFallbackReason: Send: unsupported optimized method type BlockCall @@ -4190,10 +4190,10 @@ mod hir_opt_tests { v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v18 (0x1038), num_args=0 v25:CPtr = GetEP 0 - v26:CInt64 = LoadField v25, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 + v26:CInt64 = LoadField v25, :VM_ENV_DATA_INDEX_SPECVAL@0x1058 v27:CInt64[-4] = Const CInt64(-4) v28:CInt64 = IntAnd v26, v27 - v29:BasicObject = InvokeBlockIseqDirect (0x1068), v28 + v29:BasicObject = InvokeBlockIseqDirect (0x1060), v28 CheckInterrupts PopInlineFrame Return v29 @@ -4229,11 +4229,11 @@ mod hir_opt_tests { v29:Fixnum[1] = Const Value(1) v31:Fixnum[2] = Const Value(2) v33:CPtr = GetEP 0 - v34:CInt64 = LoadField v33, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 + v34:CInt64 = LoadField v33, :VM_ENV_DATA_INDEX_SPECVAL@0x1058 v35:CInt64[-4] = Const CInt64(-4) v36:CInt64 = IntAnd v34, v35 - v37:BasicObject = InvokeBlockIseqDirect (0x1068), v36, v29, v31 - PatchPoint MethodRedefined(Integer@0x1090, +@0x1098, cme:0x10a0) + v37:BasicObject = InvokeBlockIseqDirect (0x1060), v36, v29, v31 + PatchPoint MethodRedefined(Integer@0x1080, +@0x1088, cme:0x1090) v52:Fixnum = GuardType v37, Fixnum v53:Fixnum = FixnumAdd v11, v52 CheckInterrupts @@ -4322,10 +4322,10 @@ mod hir_opt_tests { v37:Fixnum[7] = Const Value(7) v39:Fixnum[8] = Const Value(8) v41:CPtr = GetEP 0 - v42:CInt64 = LoadField v41, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 + v42:CInt64 = LoadField v41, :VM_ENV_DATA_INDEX_SPECVAL@0x1058 v43:CInt64[-4] = Const CInt64(-4) v44:CInt64 = IntAnd v42, v43 - v45:BasicObject = InvokeBlockIseqDirect (0x1068), v44, v25, v27, v29, v31, v33, v35, v37, v39 + v45:BasicObject = InvokeBlockIseqDirect (0x1060), v44, v25, v27, v29, v31, v33, v35, v37, v39 CheckInterrupts PopInlineFrame Return v45 @@ -4685,7 +4685,7 @@ mod hir_opt_tests { v44:BasicObject = CCallWithFrame v43, :Kernel#lambda@0x1040, block=0x1048 v22:CPtr = GetEP 0 v23:BasicObject = LoadField v22, :a@0x1000 - PatchPoint MethodRedefined(Object@0x1008, foo@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Object@0x1008, foo@0x1068, cme:0x1070) v34:CPtr = GetEP 0 v35:BasicObject = LoadField v34, :a@0x1000 CheckInterrupts @@ -4719,8 +4719,8 @@ mod hir_opt_tests { v24:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile v25:ArrayExact = NewArray v11, v13, v15 PushInlineFrame :foo, v24 (0x1038), num_args=1 - PatchPoint NoSingletonClass(Array@0x1060) - PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + PatchPoint NoSingletonClass(Array@0x1058) + PatchPoint MethodRedefined(Array@0x1058, length@0x1060, cme:0x1068) v49:CInt64 = ArrayLength v25 v50:Fixnum = BoxFixnum v49 CheckInterrupts @@ -4759,8 +4759,8 @@ mod hir_opt_tests { v32:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile v33:ArrayExact = NewArray v11, v13, v15, v17, v19, v21, v23 PushInlineFrame :foo, v32 (0x1038), num_args=1 - PatchPoint NoSingletonClass(Array@0x1060) - PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + PatchPoint NoSingletonClass(Array@0x1058) + PatchPoint MethodRedefined(Array@0x1058, length@0x1060, cme:0x1068) v57:CInt64 = ArrayLength v33 v58:Fixnum = BoxFixnum v57 CheckInterrupts @@ -4795,15 +4795,15 @@ mod hir_opt_tests { v24:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile v25:ArrayExact = NewArray v11, v13, v15 PushInlineFrame :foo, v24 (0x1038), num_args=1 - PatchPoint NoSingletonClass(Array@0x1060) - PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + PatchPoint NoSingletonClass(Array@0x1058) + PatchPoint MethodRedefined(Array@0x1058, length@0x1060, cme:0x1068) v55:CInt64 = ArrayLength v25 v56:Fixnum = BoxFixnum v55 v38:CPtr = GetEP 0 - v39:CInt64 = LoadField v38, :VM_ENV_DATA_INDEX_SPECVAL@0x1098 + v39:CInt64 = LoadField v38, :VM_ENV_DATA_INDEX_SPECVAL@0x1090 v40:CInt64[-4] = Const CInt64(-4) v41:CInt64 = IntAnd v39, v40 - v42:BasicObject = InvokeBlockIseqDirect (0x10a0), v41, v56 + v42:BasicObject = InvokeBlockIseqDirect (0x1098), v41, v56 CheckInterrupts PopInlineFrame Return v42 @@ -4838,20 +4838,20 @@ mod hir_opt_tests { v57:NilClass = Const Value(nil) PushInlineFrame :foo, v24 (0x1038), num_args=1 v37:CPtr = GetEP 0 - v38:CUInt64 = LoadField v37, :VM_ENV_DATA_INDEX_FLAGS@0x1060 + v38:CUInt64 = LoadField v37, :VM_ENV_DATA_INDEX_FLAGS@0x1058 v39:CBool = IsBlockParamModified v38 CondBranch v39, bb6(), bb7() bb6(): - v41:BasicObject = LoadField v37, :block@0x1061 + v41:BasicObject = LoadField v37, :block@0x1059 Jump bb8(v41, v41) bb7(): - v43:CInt64 = LoadField v37, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 + v43:CInt64 = LoadField v37, :VM_ENV_DATA_INDEX_SPECVAL@0x105a v44:CInt64 = GuardAnyBitSet v43, CUInt64(1) recompile - v45:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) + v45:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1060)) Jump bb8(v45, v57) bb8(v35:BasicObject, v36:BasicObject): - PatchPoint NoSingletonClass(Array@0x1070) - PatchPoint MethodRedefined(Array@0x1070, length@0x1078, cme:0x1080) + PatchPoint NoSingletonClass(Array@0x1068) + PatchPoint MethodRedefined(Array@0x1068, length@0x1070, cme:0x1078) v66:CInt64 = ArrayLength v25 v67:Fixnum = BoxFixnum v66 v52:BasicObject = Send v35, :call, v67 # SendFallbackReason: Send: unsupported optimized method type BlockCall @@ -4888,11 +4888,11 @@ mod hir_opt_tests { v26:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile v27:ArrayExact = NewArray v13, v15 PushInlineFrame :foo, v26 (0x1038), num_args=3 - PatchPoint NoSingletonClass(Array@0x1060) - PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + PatchPoint NoSingletonClass(Array@0x1058) + PatchPoint MethodRedefined(Array@0x1058, length@0x1060, cme:0x1068) v61:CInt64 = ArrayLength v27 v62:Fixnum = BoxFixnum v61 - PatchPoint MethodRedefined(Integer@0x1098, +@0x10a0, cme:0x10a8) + PatchPoint MethodRedefined(Integer@0x1090, +@0x1098, cme:0x10a0) v66:Fixnum = FixnumAdd v62, v11 v70:Fixnum = FixnumAdd v66, v17 CheckInterrupts @@ -4928,11 +4928,11 @@ mod hir_opt_tests { v25:ArrayExact = NewArray v11, v13 v47:Fixnum[0] = Const Value(0) PushInlineFrame :foo, v24 (0x1038), num_args=2 - PatchPoint NoSingletonClass(Array@0x1060) - PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + PatchPoint NoSingletonClass(Array@0x1058) + PatchPoint MethodRedefined(Array@0x1058, length@0x1060, cme:0x1068) v56:CInt64 = ArrayLength v25 v57:Fixnum = BoxFixnum v56 - PatchPoint MethodRedefined(Integer@0x1098, +@0x10a0, cme:0x10a8) + PatchPoint MethodRedefined(Integer@0x1090, +@0x1098, cme:0x10a0) v61:Fixnum = FixnumAdd v57, v15 CheckInterrupts PopInlineFrame @@ -4967,11 +4967,11 @@ mod hir_opt_tests { v24:Fixnum[40] = Const Value(40) v46:Fixnum[0] = Const Value(0) PushInlineFrame :foo, v22 (0x1038), num_args=2 - PatchPoint NoSingletonClass(Array@0x1060) - PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + PatchPoint NoSingletonClass(Array@0x1058) + PatchPoint MethodRedefined(Array@0x1058, length@0x1060, cme:0x1068) v55:CInt64 = ArrayLength v23 v56:Fixnum = BoxFixnum v55 - PatchPoint MethodRedefined(Integer@0x1098, +@0x10a0, cme:0x10a8) + PatchPoint MethodRedefined(Integer@0x1090, +@0x1098, cme:0x10a0) v60:Fixnum = FixnumAdd v56, v24 CheckInterrupts PopInlineFrame @@ -5069,7 +5069,7 @@ mod hir_opt_tests { v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v22 (0x1038), num_args=2 v31:Fixnum[80] = Const Value(80) - PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) + PatchPoint MethodRedefined(Integer@0x1058, +@0x1060, cme:0x1068) v67:Fixnum[110] = Const Value(110) CheckInterrupts PopInlineFrame @@ -5366,9 +5366,9 @@ mod hir_opt_tests { v21:StaticSymbol[:k] = Const Value(VALUE(0x1038)) v22:HashExact = NewHash v21: v11 PushInlineFrame :foo, v20 (0x1040), num_args=1 - PatchPoint NoSingletonClass(Hash@0x1068) - PatchPoint MethodRedefined(Hash@0x1068, class@0x1070, cme:0x1078) - v44:ClassSubclass[Hash@0x1068] = Const Value(VALUE(0x1068)) + PatchPoint NoSingletonClass(Hash@0x1060) + PatchPoint MethodRedefined(Hash@0x1060, class@0x1068, cme:0x1070) + v44:ClassSubclass[Hash@0x1060] = Const Value(VALUE(0x1060)) CheckInterrupts PopInlineFrame Return v44 @@ -5871,7 +5871,7 @@ mod hir_opt_tests { v38:Fixnum[0] = Const Value(0) PushInlineFrame :foo, v18 (0x1038), num_args=1 v30:Fixnum[1] = Const Value(1) - PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) + PatchPoint MethodRedefined(Integer@0x1058, +@0x1060, cme:0x1068) v47:Fixnum[2] = Const Value(2) CheckInterrupts PopInlineFrame @@ -6169,12 +6169,12 @@ mod hir_opt_tests { PatchPoint MethodRedefined(C@0x1008, initialize@0x1038, cme:0x1040) PushInlineFrame :initialize, v45 (0x1068), num_args=1 PatchPoint SingleRactorMode - v63:CShape = LoadField v45, :shape_id@0x1090 - v64:CShape[0x1091] = GuardBitEquals v63, CShape(0x1091) recompile - StoreField v45, :@x@0x1092, v15 + v63:CShape = LoadField v45, :shape_id@0x1088 + v64:CShape[0x1089] = GuardBitEquals v63, CShape(0x1089) recompile + StoreField v45, :@x@0x108a, v15 WriteBarrier v45, v15 - v67:CShape[0x1093] = Const CShape(0x1093) - StoreField v45, :shape_id@0x1090, v67 + v67:CShape[0x108b] = Const CShape(0x108b) + StoreField v45, :shape_id@0x1088, v67 CheckInterrupts PopInlineFrame Return v45 @@ -6275,12 +6275,12 @@ mod hir_opt_tests { v41:ObjectSubclass[class_exact:Factory] = GuardType v12, ObjectSubclass[class_exact:Factory] recompile PushInlineFrame :new, v41 (0x1048), num_args=0 v48:NilClass = Const Value(nil) - PatchPoint StableConstantNames(0x1070, Object) - v51:ClassSubclass[Object@0x1078] = Const Value(VALUE(0x1078)) - PatchPoint MethodRedefined(Object@0x1078, new@0x1018, cme:0x1080) - v84:ObjectExact = ObjectAllocClass Object:VALUE(0x1078) - PatchPoint NoSingletonClass(Object@0x1078) - PatchPoint MethodRedefined(Object@0x1078, initialize@0x10a8, cme:0x10b0) + PatchPoint StableConstantNames(0x1068, Object) + v51:ClassSubclass[Object@0x1070] = Const Value(VALUE(0x1070)) + PatchPoint MethodRedefined(Object@0x1070, new@0x1018, cme:0x1078) + v84:ObjectExact = ObjectAllocClass Object:VALUE(0x1070) + PatchPoint NoSingletonClass(Object@0x1070) + PatchPoint MethodRedefined(Object@0x1070, initialize@0x10a0, cme:0x10a8) CheckInterrupts PopInlineFrame Return v84 @@ -6346,11 +6346,11 @@ mod hir_opt_tests { PushInlineFrame :initialize, v42 (0x1068), num_args=1 v62:TrueClass = Const Value(true) v80:CPtr = GetEP 0 - v81:CUInt64 = LoadField v80, :VM_ENV_DATA_INDEX_FLAGS@0x1090 + v81:CUInt64 = LoadField v80, :VM_ENV_DATA_INDEX_FLAGS@0x1088 v82:CBool = IsBlockParamModified v81 CondBranch v82, bb11(), bb12() bb11(): - v84:BasicObject = LoadField v80, :block@0x1091 + v84:BasicObject = LoadField v80, :block@0x1089 Jump bb13(v84) bb12(): v86:BasicObject = GetBlockParam :block, l0, EP@4 @@ -6768,7 +6768,7 @@ mod hir_opt_tests { PopInlineFrame Return v76 bb4(): - v50:StaticSymbol[:skip] = Const Value(VALUE(0x1068)) + v50:StaticSymbol[:skip] = Const Value(VALUE(0x1060)) CheckInterrupts Return v50 "); @@ -11081,7 +11081,7 @@ mod hir_opt_tests { v11:ArrayExact = ArrayDup v10 PatchPoint NoSingletonClass(Array@0x1008) PatchPoint MethodRedefined(Array@0x1008, map@0x1010, cme:0x1018) - v22:BasicObject = SendDirect v11, 0x1040, :map (0x1068) + v22:BasicObject = SendDirect v11, 0x1040, :map (0x1060) CheckInterrupts Return v22 "); @@ -11279,10 +11279,10 @@ mod hir_opt_tests { PushInlineFrame :foo, v18 (0x1038), num_args=0 v25:Fixnum[1] = Const Value(1) v27:CPtr = GetEP 0 - v28:CInt64 = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 + v28:CInt64 = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1058 v29:CInt64[-4] = Const CInt64(-4) v30:CInt64 = IntAnd v28, v29 - v31:BasicObject = InvokeBlockIseqDirect (0x1068), v30, v25 + v31:BasicObject = InvokeBlockIseqDirect (0x1060), v30, v25 CheckInterrupts PopInlineFrame Return v31 @@ -11317,14 +11317,14 @@ mod hir_opt_tests { v71:NilClass = Const Value(nil) PushInlineFrame :foo, v18 (0x1038), num_args=0 v28:CPtr = GetEP 0 - v29:CUInt64 = LoadField v28, :VM_ENV_DATA_INDEX_FLAGS@0x1060 + v29:CUInt64 = LoadField v28, :VM_ENV_DATA_INDEX_FLAGS@0x1058 v30:CBool = IsBlockParamModified v29 CondBranch v30, bb7(), bb8() bb7(): - v32:BasicObject = LoadField v28, :blk@0x1061 + v32:BasicObject = LoadField v28, :blk@0x1059 Jump bb9(v32, v32) bb8(): - v34:CInt64 = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 + v34:CInt64 = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x105a v35:CInt64[0] = GuardBitEquals v34, CInt64(0) recompile v36:NilClass = Const Value(nil) Jump bb9(v36, v71) @@ -11333,16 +11333,16 @@ mod hir_opt_tests { CondBranch v39, bb10(), bb6() bb10(): v46:CPtr = GetEP 0 - v47:CUInt64 = LoadField v46, :VM_ENV_DATA_INDEX_FLAGS@0x1060 + v47:CUInt64 = LoadField v46, :VM_ENV_DATA_INDEX_FLAGS@0x1058 v48:CBool = IsBlockParamModified v47 CondBranch v48, bb11(), bb12() bb11(): - v50:BasicObject = LoadField v46, :blk@0x1061 + v50:BasicObject = LoadField v46, :blk@0x1059 Jump bb13(v50, v50) bb12(): - v52:CInt64 = LoadField v46, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 + v52:CInt64 = LoadField v46, :VM_ENV_DATA_INDEX_SPECVAL@0x105a v53:CInt64 = GuardAnyBitSet v52, CUInt64(1) recompile - v54:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) + v54:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1060)) Jump bb13(v54, v27) bb13(v44:BasicObject, v45:BasicObject): v57:BasicObject = Send v44, :call # SendFallbackReason: Send: no profile data available @@ -17337,7 +17337,7 @@ mod hir_opt_tests { v21:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1048 v22:FalseClass = GuardBitEquals v21, Value(false) PushInlineFrame :foo, v6 (0x1050), num_args=0 - v29:StringExact[VALUE(0x1078)] = Const Value(VALUE(0x1078)) + v29:StringExact[VALUE(0x1070)] = Const Value(VALUE(0x1070)) v30:StringExact = StringCopy v29 CheckInterrupts PopInlineFrame @@ -17412,13 +17412,13 @@ mod hir_opt_tests { v32:FalseClass = GuardBitEquals v31, Value(false) PushInlineFrame :foo, v9 (0x1058), num_args=1 v45:Fixnum[2] = Const Value(2) - PatchPoint MethodRedefined(Integer@0x1080, *@0x1088, cme:0x1090) + PatchPoint MethodRedefined(Integer@0x1078, *@0x1080, cme:0x1088) v59:Fixnum = GuardType v10, Fixnum recompile v60:Fixnum = FixnumMult v59, v45 CheckInterrupts PopInlineFrame v18:Fixnum[1] = Const Value(1) - PatchPoint MethodRedefined(Integer@0x1080, +@0x10b8, cme:0x10c0) + PatchPoint MethodRedefined(Integer@0x1078, +@0x10b0, cme:0x10b8) v37:Fixnum = FixnumAdd v60, v18 Return v37 "); @@ -17818,7 +17818,7 @@ mod hir_opt_tests { SetLocal :other_block, l0, EP@3, v43 v30:CPtr = GetEP 0 v31:BasicObject = LoadField v30, :other_block@0x1002 - v33:BasicObject = InvokeSuper v42, 0x1070, v31 # SendFallbackReason: super: complex argument passing to `super` call + v33:BasicObject = InvokeSuper v42, 0x1068, v31 # SendFallbackReason: super: complex argument passing to `super` call CheckInterrupts Return v33 "); @@ -18571,14 +18571,14 @@ mod hir_opt_tests { SetLocal :sep, l0, EP@5, v131 Jump bb8(v90, v131, v92, v93) bb8(v98:BasicObject, v99:BasicObject, v100:BasicObject, v101:BasicObject): - PatchPoint StableConstantNames(0x1078, CONST) - v106:HashExact[VALUE(0x1080)] = Const Value(VALUE(0x1080)) + PatchPoint StableConstantNames(0x1070, CONST) + v106:HashExact[VALUE(0x1078)] = Const Value(VALUE(0x1078)) SetLocal :kwsplat, l0, EP@3, v106 v111:CPtr = GetEP 0 v112:BasicObject = LoadField v111, :list@0x1001 v114:CPtr = GetEP 0 v115:BasicObject = LoadField v114, :iter_method@0x1005 - v117:BasicObject = Send v112, 0x1088, :__send__, v115 # SendFallbackReason: Send: unsupported method type Optimized + v117:BasicObject = Send v112, 0x1080, :__send__, v115 # SendFallbackReason: Send: unsupported method type Optimized v118:CPtr = GetEP 0 v119:BasicObject = LoadField v118, :list@0x1001 v120:BasicObject = LoadField v118, :sep@0x1002 @@ -18667,7 +18667,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame :foo, v18 (0x1038), num_args=0 - v27:StringExact[VALUE(0x1060)] = Const Value(VALUE(0x1060)) + v27:StringExact[VALUE(0x1058)] = Const Value(VALUE(0x1058)) CheckInterrupts PopInlineFrame Return v27 @@ -18741,10 +18741,10 @@ mod hir_opt_tests { v92:CInt64 = UnboxFixnum v65 v93:BasicObject = ArrayAref v11, v92 v95:CPtr = GetEP 0 - v96:CInt64 = LoadField v95, :VM_ENV_DATA_INDEX_SPECVAL@0x1068 + v96:CInt64 = LoadField v95, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 v97:CInt64[-4] = Const CInt64(-4) v98:CInt64 = IntAnd v96, v97 - v99:BasicObject = InvokeBlockIseqDirect (0x1070), v98, v93 + v99:BasicObject = InvokeBlockIseqDirect (0x1068), v98, v93 v103:Fixnum[1] = Const Value(1) v104:Fixnum = FixnumAdd v65, v103 PatchPoint NoEPEscape(each) @@ -18981,13 +18981,13 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, greet_recompile@0x1010, cme:0x1018) v42:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :greet_recompile, v42 (0x1040), num_args=1 - PatchPoint MethodRedefined(Integer@0x1068, to_s@0x1070, cme:0x1078) - v63:StringExact = CCallVariadic v22, :Integer#to_s@0x10a0 + PatchPoint MethodRedefined(Integer@0x1060, to_s@0x1068, cme:0x1070) + v63:StringExact = CCallVariadic v22, :Integer#to_s@0x1098 CheckInterrupts PopInlineFrame Return v63 bb4(): - v34:StringExact[VALUE(0x10a8)] = Const Value(VALUE(0x10a8)) + v34:StringExact[VALUE(0x10a0)] = Const Value(VALUE(0x10a0)) v35:StringExact = StringCopy v34 CheckInterrupts Return v35 @@ -20316,7 +20316,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, double@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :double, v23 (0x1040), num_args=1 - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v45:Fixnum = GuardType v10, Fixnum recompile v47:Fixnum = FixnumAdd v45, v45 CheckInterrupts @@ -20364,7 +20364,7 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :clamp_nonneg, v23 (0x1040), num_args=1 v32:Fixnum[0] = Const Value(0) - PatchPoint MethodRedefined(Integer@0x1068, <@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, <@0x1068, cme:0x1070) v62:Fixnum = GuardType v10, Fixnum recompile v63:BoolExact = FixnumLt v62, v32 v37:CBool = Test v63 @@ -20481,7 +20481,7 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :add_one, v23 (0x1040), num_args=1 v32:Fixnum[1] = Const Value(1) - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v46:Fixnum = GuardType v10, Fixnum recompile v47:Fixnum = FixnumAdd v46, v32 CheckInterrupts @@ -20542,8 +20542,8 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, outer@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :outer, v23 (0x1040), num_args=1 - PatchPoint MethodRedefined(Object@0x1008, inner@0x1068, cme:0x1070) - v44:BasicObject = SendDirect v23, 0x0, :inner (0x1098), v10 + PatchPoint MethodRedefined(Object@0x1008, inner@0x1060, cme:0x1068) + v44:BasicObject = SendDirect v23, 0x0, :inner (0x1090), v10 CheckInterrupts PopInlineFrame Return v44 @@ -20646,7 +20646,7 @@ mod hir_opt_tests { PushInlineFrame :add_opts, v23 (0x1040), num_args=1 v32:Fixnum[10] = Const Value(10) v41:Fixnum[100] = Const Value(100) - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v71:Fixnum = GuardType v10, Fixnum recompile v72:Fixnum = FixnumAdd v71, v32 v76:Fixnum = FixnumAdd v72, v41 @@ -20702,7 +20702,7 @@ mod hir_opt_tests { v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :add_opts, v25 (0x1040), num_args=2 v34:Fixnum[100] = Const Value(100) - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v63:Fixnum = GuardType v10, Fixnum recompile v64:Fixnum = FixnumAdd v63, v16 v68:Fixnum = FixnumAdd v64, v34 @@ -20739,7 +20739,7 @@ mod hir_opt_tests { v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile v40:NilClass = Const Value(nil) PushInlineFrame :callee, v18 (0x1038), num_args=0 - v26:StaticSymbol[:default] = Const Value(VALUE(0x1060)) + v26:StaticSymbol[:default] = Const Value(VALUE(0x1058)) CheckInterrupts PopInlineFrame Return v26 @@ -20826,7 +20826,7 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :maybe_rescue, v23 (0x1040), num_args=1 v32:Fixnum[1] = Const Value(1) - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v47:Fixnum = GuardType v10, Fixnum recompile v48:Fixnum = FixnumAdd v47, v32 CheckInterrupts @@ -20923,22 +20923,22 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Child@0x1008, greet@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact:Child] = GuardType v10, ObjectSubclass[class_exact:Child] recompile PushInlineFrame :greet, v23 (0x1040), num_args=0 - PatchPoint MethodRedefined(Parent@0x1068, greet@0x1010, cme:0x1070) + PatchPoint MethodRedefined(Parent@0x1060, greet@0x1010, cme:0x1068) v47:CPtr = GetEP 0 - v48:RubyValue = LoadField v47, :VM_ENV_DATA_INDEX_ME_CREF@0x1098 + v48:RubyValue = LoadField v47, :VM_ENV_DATA_INDEX_ME_CREF@0x1090 v49:CallableMethodEntry[VALUE(0x1018)] = GuardBitEquals v48, Value(VALUE(0x1018)) - v50:RubyValue = LoadField v47, :VM_ENV_DATA_INDEX_SPECVAL@0x1099 + v50:RubyValue = LoadField v47, :VM_ENV_DATA_INDEX_SPECVAL@0x1091 v51:FalseClass = GuardBitEquals v50, Value(false) - PushInlineFrame :greet, v23 (0x10a0), num_args=0 - v63:StringExact[VALUE(0x10c8)] = Const Value(VALUE(0x10c8)) + PushInlineFrame :greet, v23 (0x1098), num_args=0 + v63:StringExact[VALUE(0x10b8)] = Const Value(VALUE(0x10b8)) v64:StringExact = StringCopy v63 CheckInterrupts PopInlineFrame - v33:StringExact[VALUE(0x10d0)] = Const Value(VALUE(0x10d0)) + v33:StringExact[VALUE(0x10c0)] = Const Value(VALUE(0x10c0)) v34:StringExact = StringCopy v33 - PatchPoint NoSingletonClass(String@0x10d8) - PatchPoint MethodRedefined(String@0x10d8, +@0x10e0, cme:0x10e8) - v57:BasicObject = CCallWithFrame v64, :String#+@0x1110, v34 + PatchPoint NoSingletonClass(String@0x10c8) + PatchPoint MethodRedefined(String@0x10c8, +@0x10d0, cme:0x10d8) + v57:BasicObject = CCallWithFrame v64, :String#+@0x1100, v34 CheckInterrupts PopInlineFrame Return v57 @@ -20990,7 +20990,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_opts@0x1010, cme:0x1018) v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :add_opts, v27 (0x1040), num_args=3 - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v55:Fixnum = GuardType v10, Fixnum recompile v56:Fixnum = FixnumAdd v55, v16 v60:Fixnum = FixnumAdd v56, v18 @@ -21046,7 +21046,7 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :add_opt_post, v23 (0x1040), num_args=1 v31:Fixnum[10] = Const Value(10) - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v56:Fixnum = GuardType v10, Fixnum v57:Fixnum = FixnumAdd v31, v56 CheckInterrupts @@ -21102,7 +21102,7 @@ mod hir_opt_tests { v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :add_lead_opt_post, v25 (0x1040), num_args=2 v34:Fixnum[10] = Const Value(10) - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v63:Fixnum = GuardType v10, Fixnum recompile v64:Fixnum = FixnumAdd v63, v34 v68:Fixnum = FixnumAdd v64, v16 @@ -21155,7 +21155,7 @@ mod hir_opt_tests { v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile v43:Fixnum[0] = Const Value(0) PushInlineFrame :add_kw, v25 (0x1040), num_args=2 - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v50:Fixnum = GuardType v10, Fixnum recompile v51:Fixnum = FixnumAdd v50, v16 CheckInterrupts @@ -21207,7 +21207,7 @@ mod hir_opt_tests { v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile v43:Fixnum[0] = Const Value(0) PushInlineFrame :add_optkw, v25 (0x1040), num_args=2 - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v50:Fixnum = GuardType v10, Fixnum recompile v51:Fixnum = FixnumAdd v50, v16 CheckInterrupts @@ -21259,7 +21259,7 @@ mod hir_opt_tests { v24:Fixnum[10] = Const Value(10) v43:Fixnum[0] = Const Value(0) PushInlineFrame :add_optkw, v23 (0x1040), num_args=2 - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v50:Fixnum = GuardType v10, Fixnum recompile v51:Fixnum = FixnumAdd v50, v24 CheckInterrupts @@ -21314,11 +21314,11 @@ mod hir_opt_tests { v61:Fixnum[0] = Const Value(0) PushInlineFrame :add_kws, v27 (0x1040), num_args=3 v40:Fixnum[100] = Const Value(100) - PatchPoint MethodRedefined(Integer@0x1068, *@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, *@0x1068, cme:0x1070) v68:Fixnum = GuardType v10, Fixnum recompile v69:Fixnum = FixnumMult v68, v40 v82:Fixnum[20] = Const Value(20) - PatchPoint MethodRedefined(Integer@0x1068, +@0x10a0, cme:0x10a8) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1098, cme:0x10a0) v77:Fixnum = FixnumAdd v69, v82 v81:Fixnum = FixnumAdd v77, v16 CheckInterrupts @@ -21376,12 +21376,12 @@ mod hir_opt_tests { CondBranch v36, bb6(v24), bb7() bb7(): v42:Fixnum[2] = Const Value(2) - PatchPoint MethodRedefined(Integer@0x1068, *@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, *@0x1068, cme:0x1070) v70:Fixnum = GuardType v10, Fixnum recompile v71:Fixnum = FixnumMult v70, v42 Jump bb6(v71) bb6(v50:NilClass|Fixnum): - PatchPoint MethodRedefined(Integer@0x1068, +@0x10a0, cme:0x10a8) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1098, cme:0x10a0) v74:Fixnum = GuardType v10, Fixnum recompile v75:Fixnum = GuardType v50, Fixnum v76:Fixnum = FixnumAdd v74, v75 @@ -21437,7 +21437,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_lead_opt_post@0x1010, cme:0x1018) v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :add_lead_opt_post, v27 (0x1040), num_args=3 - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) v55:Fixnum = GuardType v10, Fixnum recompile v56:Fixnum = FixnumAdd v55, v16 v60:Fixnum = FixnumAdd v56, v18 @@ -21493,10 +21493,10 @@ mod hir_opt_tests { v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame :with_yield, v25 (0x1040), num_args=1 v34:CPtr = GetEP 0 - v35:CInt64 = LoadField v34, :VM_ENV_DATA_INDEX_SPECVAL@0x1068 + v35:CInt64 = LoadField v34, :VM_ENV_DATA_INDEX_SPECVAL@0x1060 v36:CInt64[-4] = Const CInt64(-4) v37:CInt64 = IntAnd v35, v36 - v38:BasicObject = InvokeBlockIseqDirect (0x1070), v37, v10 + v38:BasicObject = InvokeBlockIseqDirect (0x1068), v37, v10 CheckInterrupts PopInlineFrame PatchPoint NoEPEscape(test) @@ -21550,16 +21550,16 @@ mod hir_opt_tests { v53:NilClass = Const Value(nil) PushInlineFrame :with_block_param, v25 (0x1040), num_args=1 v36:CPtr = GetEP 0 - v37:CUInt64 = LoadField v36, :VM_ENV_DATA_INDEX_FLAGS@0x1068 + v37:CUInt64 = LoadField v36, :VM_ENV_DATA_INDEX_FLAGS@0x1060 v38:CBool = IsBlockParamModified v37 CondBranch v38, bb6(), bb7() bb6(): - v40:BasicObject = LoadField v36, :block@0x1069 + v40:BasicObject = LoadField v36, :block@0x1061 Jump bb8(v40, v40) bb7(): - v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x106a + v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 v43:CInt64 = GuardAnyBitSet v42, CUInt64(1) recompile - v44:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1070)) + v44:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) Jump bb8(v44, v53) bb8(v34:BasicObject, v35:BasicObject): v48:BasicObject = Send v34, :call, v10 # SendFallbackReason: Send: unsupported optimized method type BlockCall @@ -21614,16 +21614,16 @@ mod hir_opt_tests { v54:NilClass = Const Value(nil) PushInlineFrame :callee, v25 (0x1040), num_args=1 v38:CPtr = GetEP 0 - v39:CUInt64 = LoadField v38, :VM_ENV_DATA_INDEX_FLAGS@0x1068 + v39:CUInt64 = LoadField v38, :VM_ENV_DATA_INDEX_FLAGS@0x1060 v40:CBool = IsBlockParamModified v39 CondBranch v40, bb6(), bb7() bb6(): - v42:BasicObject = LoadField v38, :block@0x1069 + v42:BasicObject = LoadField v38, :block@0x1061 Jump bb8(v42, v42) bb7(): - v44:CInt64 = LoadField v38, :VM_ENV_DATA_INDEX_SPECVAL@0x106a + v44:CInt64 = LoadField v38, :VM_ENV_DATA_INDEX_SPECVAL@0x1062 v45:CInt64 = GuardAnyBitSet v44, CUInt64(1) recompile - v46:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1070)) + v46:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1068)) Jump bb8(v46, v54) bb8(v36:BasicObject, v37:BasicObject): v49:BasicObject = Send v25, &block, :inner, v10, v36 # SendFallbackReason: Send: block argument is not nil @@ -21690,22 +21690,22 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Point@0x1008, initialize@0x1038, cme:0x1040) PushInlineFrame :initialize, v85 (0x1068), num_args=2 PatchPoint SingleRactorMode - v117:CShape = LoadField v85, :shape_id@0x1090 - v118:CShape[0x1091] = GuardBitEquals v117, CShape(0x1091) recompile - StoreField v85, :@x@0x1092, v15 + v117:CShape = LoadField v85, :shape_id@0x1088 + v118:CShape[0x1089] = GuardBitEquals v117, CShape(0x1089) recompile + StoreField v85, :@x@0x108a, v15 WriteBarrier v85, v15 - v121:CShape[0x1093] = Const CShape(0x1093) - StoreField v85, :shape_id@0x1090, v121 + v121:CShape[0x108b] = Const CShape(0x108b) + StoreField v85, :shape_id@0x1088, v121 PatchPoint NoEPEscape(initialize) PatchPoint SingleRactorMode - StoreField v85, :@y@0x1094, v17 + StoreField v85, :@y@0x108c, v17 WriteBarrier v85, v17 - v136:CShape[0x1095] = Const CShape(0x1095) - StoreField v85, :shape_id@0x1090, v136 + v136:CShape[0x108d] = Const CShape(0x108d) + StoreField v85, :shape_id@0x1088, v136 CheckInterrupts PopInlineFrame v42:NilClass = Const Value(nil) - PatchPoint StableConstantNames(0x1098, Point) + PatchPoint StableConstantNames(0x1090, Point) v45:ClassSubclass[Point@0x1008] = Const Value(VALUE(0x1008)) v47:Fixnum[1] = Const Value(1) v49:Fixnum[2] = Const Value(2) @@ -21715,30 +21715,30 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Point@0x1008, initialize@0x1038, cme:0x1040) PushInlineFrame :initialize, v95 (0x1068), num_args=2 PatchPoint SingleRactorMode - v157:CShape = LoadField v95, :shape_id@0x1090 - v158:CShape[0x1091] = GuardBitEquals v157, CShape(0x1091) recompile - StoreField v95, :@x@0x1092, v47 + v157:CShape = LoadField v95, :shape_id@0x1088 + v158:CShape[0x1089] = GuardBitEquals v157, CShape(0x1089) recompile + StoreField v95, :@x@0x108a, v47 WriteBarrier v95, v47 - v161:CShape[0x1093] = Const CShape(0x1093) - StoreField v95, :shape_id@0x1090, v161 + v161:CShape[0x108b] = Const CShape(0x108b) + StoreField v95, :shape_id@0x1088, v161 PatchPoint NoEPEscape(initialize) PatchPoint SingleRactorMode - StoreField v95, :@y@0x1094, v49 + StoreField v95, :@y@0x108c, v49 WriteBarrier v95, v49 - v176:CShape[0x1095] = Const CShape(0x1095) - StoreField v95, :shape_id@0x1090, v176 + v176:CShape[0x108d] = Const CShape(0x108d) + StoreField v95, :shape_id@0x1088, v176 CheckInterrupts PopInlineFrame PatchPoint NoSingletonClass(Point@0x1008) - PatchPoint MethodRedefined(Point@0x1008, ==@0x10a0, cme:0x10a8) - PushInlineFrame :==, v85 (0x10d0), num_args=1 + PatchPoint MethodRedefined(Point@0x1008, ==@0x1098, cme:0x10a0) + PushInlineFrame :==, v85 (0x10c8), num_args=1 PatchPoint SingleRactorMode - v195:CShape = LoadField v85, :shape_id@0x1090 - v196:CShape[0x1095] = GuardBitEquals v195, CShape(0x1095) recompile - v197:BasicObject = LoadField v85, :@x@0x1092 + v195:CShape = LoadField v85, :shape_id@0x1088 + v196:CShape[0x108d] = GuardBitEquals v195, CShape(0x108d) recompile + v197:BasicObject = LoadField v85, :@x@0x108a PatchPoint NoEPEscape(==) - PatchPoint MethodRedefined(Point@0x1008, x@0x10f8, cme:0x1100) - PatchPoint MethodRedefined(Integer@0x1128, ==@0x10a0, cme:0x1130) + PatchPoint MethodRedefined(Point@0x1008, x@0x10e8, cme:0x10f0) + PatchPoint MethodRedefined(Integer@0x1118, ==@0x1098, cme:0x1120) v253:Fixnum = GuardType v197, Fixnum recompile v255:BoolExact = FixnumEq v253, v47 v208:CBool = Test v255 @@ -21746,16 +21746,16 @@ mod hir_opt_tests { CondBranch v208, bb19(), bb18(v209) bb19(): PatchPoint SingleRactorMode - v216:CShape = LoadField v85, :shape_id@0x1090 - v217:CShape[0x1095] = GuardBitEquals v216, CShape(0x1095) recompile - v218:BasicObject = LoadField v85, :@y@0x1094 + v216:CShape = LoadField v85, :shape_id@0x1088 + v217:CShape[0x108d] = GuardBitEquals v216, CShape(0x108d) recompile + v218:BasicObject = LoadField v85, :@y@0x108c PatchPoint NoEPEscape(==) PatchPoint NoSingletonClass(Point@0x1008) - PatchPoint MethodRedefined(Point@0x1008, y@0x1158, cme:0x1160) - v260:CShape = LoadField v95, :shape_id@0x1090 - v261:CShape[0x1095] = GuardBitEquals v260, CShape(0x1095) recompile - v262:BasicObject = LoadField v95, :@y@0x1094 - PatchPoint MethodRedefined(Integer@0x1128, ==@0x10a0, cme:0x1130) + PatchPoint MethodRedefined(Point@0x1008, y@0x1148, cme:0x1150) + v260:CShape = LoadField v95, :shape_id@0x1088 + v261:CShape[0x108d] = GuardBitEquals v260, CShape(0x108d) recompile + v262:BasicObject = LoadField v95, :@y@0x108c + PatchPoint MethodRedefined(Integer@0x1118, ==@0x1098, cme:0x1120) v265:Fixnum = GuardType v218, Fixnum recompile v266:Fixnum = GuardType v262, Fixnum v267:BoolExact = FixnumEq v265, v266 diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs index 6c80b892d08d4f..bbd0dfadb2e140 100644 --- a/zjit/src/hir/tests.rs +++ b/zjit/src/hir/tests.rs @@ -212,9 +212,9 @@ mod snapshot_tests { v44:Fixnum[0] = Const Value(0) v27:Any = Snapshot FrameState { pc: 0x1008, stack: [], locals: [] } PushInlineFrame :foo, v24 (0x1048), num_args=3 - v38:Any = Snapshot FrameState { pc: 0x1070, stack: [v13, v15, v11], locals: [a=v13, b=v15, c=v11, ID(0)=v44], caller: v27 } + v38:Any = Snapshot FrameState { pc: 0x1068, stack: [v13, v15, v11], locals: [a=v13, b=v15, c=v11, ID(0)=v44], caller: v27 } v39:ArrayExact = NewArray v13, v15, v11 - v40:Any = Snapshot FrameState { pc: 0x1078, stack: [v39], locals: [a=v13, b=v15, c=v11, ID(0)=v44], caller: v27 } + v40:Any = Snapshot FrameState { pc: 0x1070, stack: [v39], locals: [a=v13, b=v15, c=v11, ID(0)=v44], caller: v27 } CheckInterrupts PopInlineFrame Return v39 @@ -252,9 +252,9 @@ mod snapshot_tests { v39:Fixnum[0] = Const Value(0) v24:Any = Snapshot FrameState { pc: 0x1008, stack: [], locals: [] } PushInlineFrame :foo, v22 (0x1048), num_args=2 - v33:Any = Snapshot FrameState { pc: 0x1070, stack: [v11, v13], locals: [a=v11, b=v13, ID(0)=v39], caller: v24 } + v33:Any = Snapshot FrameState { pc: 0x1068, stack: [v11, v13], locals: [a=v11, b=v13, ID(0)=v39], caller: v24 } v34:ArrayExact = NewArray v11, v13 - v35:Any = Snapshot FrameState { pc: 0x1078, stack: [v34], locals: [a=v11, b=v13, ID(0)=v39], caller: v24 } + v35:Any = Snapshot FrameState { pc: 0x1070, stack: [v34], locals: [a=v11, b=v13, ID(0)=v39], caller: v24 } CheckInterrupts PopInlineFrame Return v34 @@ -299,9 +299,9 @@ mod snapshot_tests { v64:Fixnum[0] = Const Value(0) v37:Any = Snapshot FrameState { pc: 0x1008, stack: [], locals: [] } PushInlineFrame :foo, v34 (0x1048), num_args=8 - v58:Any = Snapshot FrameState { pc: 0x1070, stack: [v19, v21, v17, v15, v11, v13, v23, v25], locals: [five=v11, six=v13, a=v19, b=v21, c=v17, d=v15, e=v23, f=v25, ID(0)=v64], caller: v37 } + v58:Any = Snapshot FrameState { pc: 0x1068, stack: [v19, v21, v17, v15, v11, v13, v23, v25], locals: [five=v11, six=v13, a=v19, b=v21, c=v17, d=v15, e=v23, f=v25, ID(0)=v64], caller: v37 } v59:ArrayExact = NewArray v19, v21, v17, v15, v11, v13, v23, v25 - v60:Any = Snapshot FrameState { pc: 0x1078, stack: [v59], locals: [five=v11, six=v13, a=v19, b=v21, c=v17, d=v15, e=v23, f=v25, ID(0)=v64], caller: v37 } + v60:Any = Snapshot FrameState { pc: 0x1070, stack: [v59], locals: [five=v11, six=v13, a=v19, b=v21, c=v17, d=v15, e=v23, f=v25, ID(0)=v64], caller: v37 } CheckInterrupts PopInlineFrame Return v59 @@ -2141,7 +2141,7 @@ pub(crate) mod hir_build_tests { v25:BasicObject = Send v10, 0x1000, :foo # SendFallbackReason: Uncategorized(send) PatchPoint NoEPEscape(test) v28:CPtr = LoadSP - v29:BasicObject = LoadField v28, :a@0x1028 + v29:BasicObject = LoadField v28, :a@0x1020 PatchPoint NoEPEscape(test) v38:BasicObject = Send v29, :+, v20 # SendFallbackReason: Uncategorized(opt_plus) CheckInterrupts @@ -2217,8 +2217,8 @@ pub(crate) mod hir_build_tests { PatchPoint NoEPEscape(test) v18:CPtr = LoadSP v19:BasicObject = LoadField v18, :block@0x1000 - PatchPoint StableConstantNames(0x1030, ::RubyVM::ZJIT) - v24:ModuleSubclass[RubyVM::ZJIT@0x1038] = Const Value(VALUE(0x1038)) + PatchPoint StableConstantNames(0x1028, ::RubyVM::ZJIT) + v24:ModuleSubclass[RubyVM::ZJIT@0x1030] = Const Value(VALUE(0x1030)) SideExit DirectiveInduced "); } @@ -2257,8 +2257,8 @@ pub(crate) mod hir_build_tests { v17:Fixnum[1] = Const Value(1) v22:BasicObject = Send v11, 0x1008, :consume # SendFallbackReason: Uncategorized(send) PatchPoint NoEPEscape(test) - PatchPoint StableConstantNames(0x1030, ::RubyVM::ZJIT) - v29:ModuleSubclass[RubyVM::ZJIT@0x1038] = Const Value(VALUE(0x1038)) + PatchPoint StableConstantNames(0x1028, ::RubyVM::ZJIT) + v29:ModuleSubclass[RubyVM::ZJIT@0x1030] = Const Value(VALUE(0x1030)) SideExit DirectiveInduced "); } @@ -2295,16 +2295,16 @@ pub(crate) mod hir_build_tests { v15:BasicObject = Send v9, 0x1008, :consume # SendFallbackReason: Uncategorized(send) PatchPoint NoEPEscape(test) v24:CPtr = GetEP 0 - v25:CUInt64 = LoadField v24, :VM_ENV_DATA_INDEX_FLAGS@0x1030 + v25:CUInt64 = LoadField v24, :VM_ENV_DATA_INDEX_FLAGS@0x1028 v26:CBool = IsBlockParamModified v25 CondBranch v26, bb4(), bb5() bb4(): - v28:BasicObject = LoadField v24, :&@0x1031 + v28:BasicObject = LoadField v24, :&@0x1029 Jump bb6(v28, v28) bb5(): - v30:CInt64 = LoadField v24, :VM_ENV_DATA_INDEX_SPECVAL@0x1032 + v30:CInt64 = LoadField v24, :VM_ENV_DATA_INDEX_SPECVAL@0x102a v31:CInt64 = GuardAnyBitSet v30, CUInt64(1) recompile - v32:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1038)) + v32:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1030)) Jump bb6(v32, v10) bb6(v22:BasicObject, v23:BasicObject): v35:BasicObject = Send v9, &block, :consume, v22 # SendFallbackReason: Uncategorized(send) @@ -2350,7 +2350,7 @@ pub(crate) mod hir_build_tests { v25:BasicObject = Send v10, 0x1000, :foo # SendFallbackReason: Uncategorized(send) PatchPoint NoEPEscape(test) v28:CPtr = LoadSP - v29:BasicObject = LoadField v28, :a@0x1028 + v29:BasicObject = LoadField v28, :a@0x1020 PatchPoint NoEPEscape(test) v38:BasicObject = Send v29, :+, v20 # SendFallbackReason: Uncategorized(opt_plus) CheckInterrupts From b95de2b9a4de6a7a9d95a876b6520fa1c89681f7 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 28 Aug 2026 22:32:42 +0900 Subject: [PATCH 2/7] [ruby/date] Preserve input encoding when parsing time zones Pass the input encoding through `date__strptime` so non-ASCII trailing data is treated as a parse failure instead of raising an encoding error. Fixes https://github.com/ruby/date/pull/179. https://github.com/ruby/date/commit/edf7cf016c --- ext/date/date_core.c | 25 ++++++------------------- ext/date/date_strptime.c | 20 ++++++++++---------- test/date/test_date_strptime.rb | 7 +++++++ 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/ext/date/date_core.c b/ext/date/date_core.c index 72d697c8ea233e..e9c169f452cdc3 100644 --- a/ext/date/date_core.c +++ b/ext/date/date_core.c @@ -4515,7 +4515,8 @@ d_new_by_frags(VALUE klass, VALUE hash, VALUE sg) } VALUE date__strptime(const char *str, size_t slen, - const char *fmt, size_t flen, VALUE hash); + const char *fmt, size_t flen, + VALUE hash, rb_encoding *enc); static VALUE date_s__strptime_internal(int argc, VALUE *argv, VALUE klass, @@ -4524,6 +4525,7 @@ date_s__strptime_internal(int argc, VALUE *argv, VALUE klass, VALUE vstr, vfmt, hash; const char *str, *fmt; size_t slen, flen; + rb_encoding *enc; rb_scan_args(argc, argv, "11", &vstr, &vfmt); @@ -4537,33 +4539,18 @@ date_s__strptime_internal(int argc, VALUE *argv, VALUE klass, if (argc < 2) { fmt = default_fmt; flen = strlen(default_fmt); + enc = rb_enc_get(vstr); } else { if (!rb_enc_str_asciicompat_p(vfmt)) rb_raise(rb_eArgError, "format should have ASCII compatible encoding"); + enc = rb_enc_check(vstr, vfmt); fmt = RSTRING_PTR(vfmt); flen = RSTRING_LEN(vfmt); } hash = rb_hash_new(); - if (NIL_P(date__strptime(str, slen, fmt, flen, hash))) - return Qnil; - - { - VALUE zone = ref_hash("zone"); - VALUE left = ref_hash("leftover"); - - if (!NIL_P(zone)) { - rb_enc_copy(zone, vstr); - set_hash("zone", zone); - } - if (!NIL_P(left)) { - rb_enc_copy(left, vstr); - set_hash("leftover", left); - } - } - - return hash; + return date__strptime(str, slen, fmt, flen, hash, enc); } /* diff --git a/ext/date/date_strptime.c b/ext/date/date_strptime.c index 1dde5fa3ec93b5..0ad03a0da9c693 100644 --- a/ext/date/date_strptime.c +++ b/ext/date/date_strptime.c @@ -41,7 +41,6 @@ static const int ABBREVIATED_MONTH_NAME_LENGTH = 3; #define f_match(r,s) rb_funcall(r, rb_intern("match"), 1, s) #define f_aref(o,i) rb_funcall(o, rb_intern("[]"), 1, i) -#define f_end(o,i) rb_funcall(o, rb_intern("end"), 1, i) #define issign(c) ((c) == '-' || (c) == '+') @@ -144,7 +143,7 @@ valid_range_p(VALUE v, int a, int b) do { \ size_t l; \ l = date__strptime_internal(&str[si], slen - si, \ - fmt, sizeof fmt - 1, hash); \ + fmt, sizeof fmt - 1, hash, enc); \ if (fail_p()) \ return 0; \ si += l; \ @@ -160,7 +159,8 @@ head_match_p(size_t len, const char *name, const char *str, size_t slen, size_t static size_t date__strptime_internal(const char *str, size_t slen, - const char *fmt, size_t flen, VALUE hash) + const char *fmt, size_t flen, + VALUE hash, rb_encoding *enc) { size_t si, fi; int c; @@ -597,15 +597,14 @@ date__strptime_internal(const char *str, size_t slen, b = rb_backref_get(); rb_match_busy(b); - m = f_match(pat, rb_usascii_str_new(&str[si], slen - si)); + m = f_match(pat, rb_enc_str_new(&str[si], slen - si, enc)); if (!NIL_P(m)) { - VALUE s, l, o; + VALUE s, o; s = rb_reg_nth_match(1, m); - l = f_end(m, INT2FIX(0)); o = date_zone_to_diff(s); - si += NUM2LONG(l); + si += RSTRING_LEN(s); set_hash("zone", s); set_hash("offset", o); rb_backref_set(b); @@ -654,12 +653,13 @@ date__strptime_internal(const char *str, size_t slen, VALUE date__strptime(const char *str, size_t slen, - const char *fmt, size_t flen, VALUE hash) + const char *fmt, size_t flen, + VALUE hash, rb_encoding *enc) { size_t si; VALUE cent, merid; - si = date__strptime_internal(str, slen, fmt, flen, hash); + si = date__strptime_internal(str, slen, fmt, flen, hash, enc); if (fail_p()) return Qnil; @@ -667,7 +667,7 @@ date__strptime(const char *str, size_t slen, if (slen > si) { VALUE s; - s = rb_usascii_str_new(&str[si], slen - si); + s = rb_enc_str_new(&str[si], slen - si, enc); set_hash("leftover", s); } diff --git a/test/date/test_date_strptime.rb b/test/date/test_date_strptime.rb index 6aa7db292de424..f68f749e054269 100644 --- a/test/date/test_date_strptime.rb +++ b/test/date/test_date_strptime.rb @@ -533,4 +533,11 @@ def to_str assert_equal(6, d[:mon]) assert_equal(1, d[:mday]) end + + def test_nonascii_string + nonalpha = "\u{2600 fe0f}" + s = "2011-10-05T22:26:12#{nonalpha}" + + assert_nil(DateTime._strptime(s)) + end end From 2084db2eab3fc22c2c9b85f485841e0e6a7bb56b Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 28 Aug 2026 23:57:13 +0900 Subject: [PATCH 3/7] [ruby/date] Validate the commercial week range before conversion Reject week numbers outside -53..-1 and 1..53 before conversion so integer overflow cannot make invalid input appear valid. Fixes https://github.com/ruby/date/pull/180. https://github.com/ruby/date/commit/e9c07010c7 --- ext/date/date_core.c | 4 ++++ test/date/test_date_new.rb | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/ext/date/date_core.c b/ext/date/date_core.c index e9c169f452cdc3..a7a00e1751602d 100644 --- a/ext/date/date_core.c +++ b/ext/date/date_core.c @@ -992,12 +992,16 @@ c_valid_commercial_p(int y, int w, int d, double sg, if (w < 0) { int rjd2; + if (w < -53) return 0; c_commercial_to_jd(y + 1, 1, 1, sg, &rjd2, &ns2); c_jd_to_commercial(rjd2 + w * 7, sg, &ry2, &rw2, &rd2); if (ry2 != y) return 0; w = rw2; } + else { + if (w < 1 || 53 < w) return 0; + } c_commercial_to_jd(y, w, d, sg, rjd, ns); c_jd_to_commercial(*rjd, sg, &ry2, rw, rd); if (y != ry2 || w != *rw || d != *rd) diff --git a/test/date/test_date_new.rb b/test/date/test_date_new.rb index eddeeff820f59e..8f71566b12a2ac 100644 --- a/test/date/test_date_new.rb +++ b/test/date/test_date_new.rb @@ -256,6 +256,11 @@ def test_commercial__ex end end + def test_commercial_p + assert_equal(false, Date.valid_commercial?(2024, -53, 1)) + assert_equal(false, Date.valid_commercial?(2024, -1227133565, 1)) + end + def test_weeknum d = Date.weeknum dt = DateTime.weeknum From c84cb4d4f343aaf4be581be2461a26791c315e9f Mon Sep 17 00:00:00 2001 From: sjh9714 Date: Wed, 19 Aug 2026 10:07:40 +0900 Subject: [PATCH 4/7] [ruby/date] Range-check the integral Rational offset Every branch of offset_to_sec range-checks the resulting number of seconds except one path through the Rational branch: when the day fraction is an integral Rational, n is assigned inside the if arm and reaches *rof without passing the guard that sits in the else arm. DateTime.new(2024, 1, 1, 0, 0, 0, Rational(2, 1)) therefore produced a 48-hour offset, while the equivalent Integer 2 is rejected and falls back to +00:00. Rational(49710, 1) is 4_294_944_000 seconds, over INT_MAX, so the (int) narrowing turned a large positive offset into a negative one. Move the check below the if/else so it covers both arms. That also bounds n before the narrowing. Rational(1, 1) is exactly DAY_IN_SECONDS and the guard is inclusive, so in-range values are unaffected. https://github.com/ruby/date/commit/7524d7c41f --- ext/date/date_core.c | 4 ++-- test/date/test_date_new.rb | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/ext/date/date_core.c b/ext/date/date_core.c index a7a00e1751602d..58f1c0c81ceff8 100644 --- a/ext/date/date_core.c +++ b/ext/date/date_core.c @@ -2625,9 +2625,9 @@ offset_to_sec(VALUE vof, int *rof) if (!FIXNUM_P(vn)) return 0; n = FIX2LONG(vn); - if (n < -DAY_IN_SECONDS || n > DAY_IN_SECONDS) - return 0; } + if (n < -DAY_IN_SECONDS || n > DAY_IN_SECONDS) + return 0; *rof = (int)n; return 1; } diff --git a/test/date/test_date_new.rb b/test/date/test_date_new.rb index 8f71566b12a2ac..74a29f907d42cd 100644 --- a/test/date/test_date_new.rb +++ b/test/date/test_date_new.rb @@ -192,6 +192,21 @@ def test_civil__ex end end + def test_civil__offset + d = DateTime.civil(2001,2,3, 0,0,0, Rational(1, 1)) + assert_equal(1.to_r, d.offset) + d = DateTime.civil(2001,2,3, 0,0,0, Rational(-1, 1)) + assert_equal(-1.to_r, d.offset) + + # An out-of-range offset is ignored, as it is for the equivalent Integer. + d = DateTime.civil(2001,2,3, 0,0,0, 2) + assert_equal(0, d.offset) + d = DateTime.civil(2001,2,3, 0,0,0, Rational(2, 1)) + assert_equal(0, d.offset) + d = DateTime.civil(2001,2,3, 0,0,0, Rational(49710, 1)) + assert_equal(0, d.offset) + end + def test_civil__reform d = Date.jd(Date::ENGLAND, Date::ENGLAND) dt = DateTime.jd(Date::ENGLAND, 0,0,0,0, Date::ENGLAND) From 8971984922055465ef81b7274f5125a13ef36f48 Mon Sep 17 00:00:00 2001 From: sjh9714 <163989462+sjh9714@users.noreply.github.com> Date: Fri, 28 Aug 2026 21:06:49 +0900 Subject: [PATCH 5/7] [ruby/date] Assert warnings for invalid offsets https://github.com/ruby/date/commit/569565d8c0 --- test/date/test_date_new.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/date/test_date_new.rb b/test/date/test_date_new.rb index 74a29f907d42cd..b839c5283b4053 100644 --- a/test/date/test_date_new.rb +++ b/test/date/test_date_new.rb @@ -199,11 +199,17 @@ def test_civil__offset assert_equal(-1.to_r, d.offset) # An out-of-range offset is ignored, as it is for the equivalent Integer. - d = DateTime.civil(2001,2,3, 0,0,0, 2) + assert_warning(/invalid offset/) do + d = DateTime.civil(2001,2,3, 0,0,0, 2) + end assert_equal(0, d.offset) - d = DateTime.civil(2001,2,3, 0,0,0, Rational(2, 1)) + assert_warning(/invalid offset/) do + d = DateTime.civil(2001,2,3, 0,0,0, Rational(2, 1)) + end assert_equal(0, d.offset) - d = DateTime.civil(2001,2,3, 0,0,0, Rational(49710, 1)) + assert_warning(/invalid offset/) do + d = DateTime.civil(2001,2,3, 0,0,0, Rational(49710, 1)) + end assert_equal(0, d.offset) end From 46e53809f5a7d05fed8b92ac6bf6a9ca8336a78f Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Mon, 30 Mar 2026 18:57:26 +0100 Subject: [PATCH 6/7] [ruby/date] Implement compaction and embedded structs Currently dates are allocated in a 40B slot plus a 32B (simple) or 48B (complex) external struct. Once embedded they entirely fit into a 80B slot. https://github.com/ruby/date/commit/e8a1442c60 --- ext/date/date_core.c | 64 +++++++++++++++++++++++++++++++++----------- ext/date/extconf.rb | 3 +++ 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/ext/date/date_core.c b/ext/date/date_core.c index 58f1c0c81ceff8..0d69633ab48fd1 100644 --- a/ext/date/date_core.c +++ b/ext/date/date_core.c @@ -3199,34 +3199,66 @@ date_s_gregorian_leap_p(VALUE klass, VALUE y) return f_boolcast(c_gregorian_leap_p(ry)); } +#ifndef HAVE_RB_GC_MARK_MOVABLE +#define rb_gc_mark_movable rb_gc_mark +#else static void -d_lite_gc_mark(void *ptr) +d_lite_gc_compact(void *ptr) { union DateData *dat = ptr; if (simple_dat_p(dat)) - rb_gc_mark(dat->s.nth); + dat->s.nth = rb_gc_location(dat->s.nth); else { - rb_gc_mark(dat->c.nth); - rb_gc_mark(dat->c.sf); + dat->c.nth = rb_gc_location(dat->c.nth); + dat->c.sf = rb_gc_location(dat->c.sf); } } +#endif -static size_t -d_lite_memsize(const void *ptr) +static void +d_lite_gc_mark(void *ptr) { - const union DateData *dat = ptr; - return complex_dat_p(dat) ? sizeof(struct ComplexDateData) : sizeof(struct SimpleDateData); + union DateData *dat = ptr; + if (simple_dat_p(dat)) + rb_gc_mark_movable(dat->s.nth); + else { + rb_gc_mark_movable(dat->c.nth); + rb_gc_mark_movable(dat->c.sf); + } } #ifndef HAVE_RB_EXT_RACTOR_SAFE # define RUBY_TYPED_FROZEN_SHAREABLE 0 #endif +#if defined(RUBY_TYPED_EMBEDDABLE) || defined(HAVE_CONST_RUBY_TYPED_EMBEDDABLE) +# define HAVE_RUBY_TYPED_EMBEDDABLE 1 +#else +# define RUBY_TYPED_EMBEDDABLE 0 +#endif + +static size_t +d_lite_memsize(const void *ptr) +{ +#ifdef HAVE_RUBY_TYPED_EMBEDDABLE + return 0; +#else + return sizeof(union DateData); +#endif +} + static const rb_data_type_t d_lite_type = { "Date", - {d_lite_gc_mark, RUBY_TYPED_DEFAULT_FREE, d_lite_memsize,}, + { + d_lite_gc_mark, + RUBY_TYPED_DEFAULT_FREE, + d_lite_memsize, +#ifdef HAVE_RB_GC_MARK_MOVABLE + d_lite_gc_compact, +#endif + }, 0, 0, - RUBY_TYPED_FREE_IMMEDIATELY|RUBY_TYPED_WB_PROTECTED|RUBY_TYPED_FROZEN_SHAREABLE, + RUBY_TYPED_FREE_IMMEDIATELY|RUBY_TYPED_WB_PROTECTED|RUBY_TYPED_EMBEDDABLE|RUBY_TYPED_FROZEN_SHAREABLE, }; inline static VALUE @@ -3237,10 +3269,11 @@ d_simple_new_internal(VALUE klass, unsigned flags) { struct SimpleDateData *dat; + union DateData *u_dat; VALUE obj; - obj = TypedData_Make_Struct(klass, struct SimpleDateData, - &d_lite_type, dat); + obj = TypedData_Make_Struct(klass, union DateData, &d_lite_type, u_dat); + dat = &u_dat->s; set_to_simple(obj, dat, nth, jd, sg, y, m, d, flags); assert(have_jd_p(dat) || have_civil_p(dat)); @@ -3258,10 +3291,11 @@ d_complex_new_internal(VALUE klass, unsigned flags) { struct ComplexDateData *dat; + union DateData *u_dat; VALUE obj; - obj = TypedData_Make_Struct(klass, struct ComplexDateData, - &d_lite_type, dat); + obj = TypedData_Make_Struct(klass, union DateData, &d_lite_type, u_dat); + dat = &u_dat->c; set_to_complex(obj, dat, nth, jd, df, sf, of, sg, y, m, d, h, min, s, flags); @@ -7795,8 +7829,6 @@ d_lite_marshal_load(VALUE self, VALUE a) if (simple_dat_p(dat)) { if (df || !f_zero_p(sf) || of) { /* loading a fractional date; promote to complex */ - dat = ruby_xrealloc(dat, sizeof(struct ComplexDateData)); - RTYPEDDATA(self)->data = dat; goto complex_data; } set_to_simple(self, &dat->s, nth, jd, sg, 0, 0, 0, HAVE_JD); diff --git a/ext/date/extconf.rb b/ext/date/extconf.rb index 3ab534f833edfd..963f96f0ac3201 100644 --- a/ext/date/extconf.rb +++ b/ext/date/extconf.rb @@ -13,4 +13,7 @@ have_var("altzone", "time.h", opt) end +have_func("rb_gc_mark_movable", "ruby.h") # RUBY_VERSION >= 2.7 +have_const("RUBY_TYPED_EMBEDDABLE", "ruby.h") # RUBY_VERSION >= 3.3 + create_makefile('date_core') From 90e729c9bcd7d5b909cc1546cd40a0e2afbb906c Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Fri, 28 Aug 2026 02:50:37 +0900 Subject: [PATCH 7/7] Share one thread/ractor scheduler between POSIX and Windows Windows carried a second, much weaker scheduler: a single Win32 mutex as the GVL, vm->clock for preemption, its own VM barrier over vm->ractor.sync.barrier_*, its own Ractor wait/wakeup on a per-Ractor condvar, no ready queue and no time slice. Keeping two made the rest of the interpreter branch on which one was compiled in, and left Ractors on Windows without real scheduling. Move the scheduler out of thread_pthread.c into thread_sched.c and let both platforms use it. thread.c now includes thread_sched.c, which declares the scheduler <-> platform contract, includes THREAD_IMPL_SRC, and then builds the scheduler on the primitives that file provides. thread_pthread.c keeps only those primitives: synchronization, thread local storage, native thread creation and stacks, the communication pipe, the timer thread backend, the fork lock and the event hooks. The data structures the scheduler works on move to a new thread_sched.h, included from both platform headers, so struct rb_thread_sched, struct rb_native_thread and struct rb_thread_sched_item are now shared. thread_pthread_mn.c becomes thread_sched_mn.c: it is the M:N scheduler, not a pthread detail. Most of it -- the coroutine threads, the native thread stack pool, the timer wheel and the fd -> waiters map -- is platform independent; only the readiness backend is not, and that section is now marked as the seam where an epoll/kqueue split (or an IOCP backend, which is what Windows would need for M:N) would go. What Windows had to add is small: native_thread_interrupt(), native_thread_default_max_cpu(), timer_thread_polling(), timer_thread_wakeup_force(), RB_NATIVE_COND_MONOTONIC_P() and RB_NATIVE_MUTEX_TRYLOCK_DETECTS_SELF() -- the last because a CRITICAL_SECTION is recursive, so trylock cannot back a "somebody holds this" assertion. It sets USE_MN_THREADS to 0 and supplies the stubs thread_sched_mn.c provides elsewhere, so every thread there is dedicated, as it already was. With one scheduler the platform branches go away: all 16 uses of RUBY_THREAD_PTHREAD_H / RUBY_THREAD_WIN32_H are removed from vm_core.h, ractor.c, ractor_core.h, ractor_sync.c, vm_sync.c, vm.c and thread.c. ractor_check_blocking() went with them -- it only ever fed the win32 scheduler's vm->ractor.blocking_cnt -- and so did USE_VM_CLOCK, now that the common time slice drives preemption on Windows too. Verified on both platforms. Linux: 11 rounds of make test-all (36236 tests) and btest (2066 tests) with no failures, plus 474 runs of a scheduler and Ractor stress soak. Windows: no test fails that does not also fail on the same tree without this change, btest passes, and two 40-minute soaks of 438 and 460 runs are clean. Ractors now run in parallel there -- 3.6x on four Ractors, where before they could not. Co-Authored-By: Claude Opus 5 --- ractor.c | 56 - ractor_core.h | 4 - ractor_sync.c | 70 - thread.c | 17 +- thread_none.c | 23 +- thread_none.h | 14 +- thread_pthread.c | 2525 +--------------------- thread_pthread.h | 181 +- thread_sched.c | 2501 +++++++++++++++++++++ thread_sched.h | 264 +++ thread_pthread_mn.c => thread_sched_mn.c | 28 +- thread_win32.c | 1078 +++++---- thread_win32.h | 18 +- vm.c | 4 - vm_core.h | 59 +- vm_sync.c | 14 - 16 files changed, 3432 insertions(+), 3424 deletions(-) create mode 100644 thread_sched.c create mode 100644 thread_sched.h rename thread_pthread_mn.c => thread_sched_mn.c (97%) diff --git a/ractor.c b/ractor.c index 201e363d2bb99f..5311c4b96ba861 100644 --- a/ractor.c +++ b/ractor.c @@ -418,9 +418,7 @@ free_targeted_hooks(st_table *hooks_tbl) st_foreach(hooks_tbl, free_targeted_hook_lists, 0); } -#ifdef RUBY_THREAD_PTHREAD_H void rb_thread_sched_destroy(struct rb_thread_sched *); -#endif static void ractor_free(void *ptr) @@ -429,13 +427,8 @@ ractor_free(void *ptr) RUBY_DEBUG_LOG("free r:%d", rb_ractor_id(r)); free_targeted_hooks(&r->pub.targeted_hooks); -#ifdef RUBY_THREAD_PTHREAD_H rb_thread_sched_destroy(&r->threads.sched); -#endif rb_native_mutex_destroy(&r->sync.lock); -#ifdef RUBY_THREAD_WIN32_H - rb_native_cond_destroy(&r->sync.wakeup_cond); -#endif ractor_local_storage_free(r); rb_hook_list_free(&r->pub.hooks); rb_st_free_embedded_table(&r->pub.targeted_hooks); @@ -1024,39 +1017,6 @@ rb_vm_ractor_blocking_cnt_dec(rb_vm_t *vm, rb_ractor_t *cr, const char *file, in ractor_status_set(cr, ractor_running); } -static void -ractor_check_blocking(rb_ractor_t *cr, unsigned int remained_thread_cnt, const char *file, int line) -{ - VM_ASSERT(cr == GET_RACTOR()); - -#ifdef RUBY_THREAD_PTHREAD_H - // vm->ractor.blocking_cnt is only consumed by the win32 scheduler; the - // pthread one must not pay a VM lock per blocking region for it. The - // running<->blocking status flips stop with it (all callers), matching - // rb_ractor_blocking_threads_dec skipping the reverse transition. - return; -#endif - - RUBY_DEBUG_LOG2(file, line, - "cr->threads.cnt:%u cr->threads.blocking_cnt:%u vm->ractor.cnt:%u vm->ractor.blocking_cnt:%u", - cr->threads.cnt, cr->threads.blocking_cnt, - GET_VM()->ractor.cnt, GET_VM()->ractor.blocking_cnt); - - VM_ASSERT(cr->threads.cnt >= cr->threads.blocking_cnt + 1); - - if (remained_thread_cnt > 0 && - // will be block - cr->threads.cnt == cr->threads.blocking_cnt + 1) { - // change ractor status: running -> blocking - rb_vm_t *vm = GET_VM(); - - RB_VM_LOCKING() { - rb_vm_ractor_blocking_cnt_inc(vm, cr, file, line); - } - } -} - - /* Remove a child that never started (send_parameters failed during creation). The * creator calls this (rb_ractor_living_threads_remove assumes the current Ractor); * leaving the set and disowning the objspace share one VM-lock section, no window. */ @@ -1100,8 +1060,6 @@ rb_ractor_living_threads_remove(rb_ractor_t *cr, rb_thread_t *th) { VM_ASSERT(cr == GET_RACTOR()); RUBY_DEBUG_LOG("r->threads.cnt:%d--", cr->threads.cnt); - ractor_check_blocking(cr, cr->threads.cnt - 1, __FILE__, __LINE__); - if (cr->threads.cnt == 1) { vm_remove_ractor(th->vm, cr); @@ -1124,7 +1082,6 @@ rb_ractor_blocking_threads_inc(rb_ractor_t *cr, const char *file, int line) VM_ASSERT(cr->threads.cnt > 0); VM_ASSERT(cr == GET_RACTOR()); - ractor_check_blocking(cr, cr->threads.cnt, __FILE__, __LINE__); cr->threads.blocking_cnt++; } @@ -1137,17 +1094,6 @@ rb_ractor_blocking_threads_dec(rb_ractor_t *cr, const char *file, int line) VM_ASSERT(cr == GET_RACTOR()); -#ifndef RUBY_THREAD_PTHREAD_H - // see rb_ractor_blocking_threads_inc - if (cr->threads.cnt == cr->threads.blocking_cnt) { - rb_vm_t *vm = GET_VM(); - - RB_VM_LOCKING() { - rb_vm_ractor_blocking_cnt_dec(vm, cr, __FILE__, __LINE__); - } - } -#endif - cr->threads.blocking_cnt--; } @@ -1236,7 +1182,6 @@ rb_ractor_terminate_all(void) rb_vm_ractor_blocking_cnt_inc(vm, cr, __FILE__, __LINE__); rb_del_running_thread(rb_ec_thread_ptr(cr->threads.running_ec)); rb_vm_cond_timedwait(vm, &vm->ractor.sync.terminate_cond, 1000 /* ms */); -#ifdef RUBY_THREAD_PTHREAD_H while (vm->ractor.sched.barrier_is_waiting) { // A barrier is waiting. Threads relinquish the VM lock before joining the barrier and // since we just acquired the VM lock back, we're blocking other threads from joining it. @@ -1246,7 +1191,6 @@ rb_ractor_terminate_all(void) unsigned int lev; RB_VM_LOCK_ENTER_LEV_NB(&lev); } -#endif rb_add_running_thread(rb_ec_thread_ptr(cr->threads.running_ec)); rb_vm_ractor_blocking_cnt_dec(vm, cr, __FILE__, __LINE__); diff --git a/ractor_core.h b/ractor_core.h index d918648c1f6276..6a545251473dc5 100644 --- a/ractor_core.h +++ b/ractor_core.h @@ -24,10 +24,6 @@ struct rb_ractor_sync { VALUE locked_by; #endif -#ifndef RUBY_THREAD_PTHREAD_H - rb_nativethread_cond_t wakeup_cond; -#endif - // incoming messages struct ractor_queue *recv_queue; diff --git a/ractor_sync.c b/ractor_sync.c index 9a63a5f84757c1..c6e4d8a6d31df1 100644 --- a/ractor_sync.c +++ b/ractor_sync.c @@ -945,9 +945,6 @@ ractor_sync_init(rb_ractor_t *r) // no receive is rebuilding a payload yet -#ifndef RUBY_THREAD_PTHREAD_H - rb_native_cond_initialize(&r->sync.wakeup_cond); -#endif } /* Create the default port. Call only after the Ractor joined vm->ractor.set, so the @@ -1304,73 +1301,6 @@ basket_type_name(enum ractor_basket_type type) #endif // USE_RUBY_DEBUG_LOG -#ifdef RUBY_THREAD_PTHREAD_H - -// - -#else // win32 - -static void -ractor_cond_wait(rb_ractor_t *r, const rb_hrtime_t *end) -{ -#if RACTOR_CHECK_MODE > 0 - VALUE locked_by = r->sync.locked_by; - r->sync.locked_by = Qnil; -#endif - if (end) { - rb_hrtime_t now = rb_hrtime_now(); - rb_hrtime_t rel = *end > now ? *end - now : 0; - // the condvar takes msec: never round a live timeout down to 0 - unsigned long msec = (unsigned long)(rel / RB_HRTIME_PER_MSEC); - rb_native_cond_timedwait(&r->sync.wakeup_cond, &r->sync.lock, msec > 0 ? msec : 1); - } - else { - rb_native_cond_wait(&r->sync.wakeup_cond, &r->sync.lock); - } - -#if RACTOR_CHECK_MODE > 0 - r->sync.locked_by = locked_by; -#endif -} - -static void * -ractor_wait_no_gvl(void *ptr) -{ - struct ractor_waiter *waiter = (struct ractor_waiter *)ptr; - rb_ractor_t *cr = waiter->th->ractor; - - RACTOR_LOCK_SELF(cr); - { - if (waiter->wakeup_status == wakeup_none) { - ractor_cond_wait(cr, waiter->end); - } - } - RACTOR_UNLOCK_SELF(cr); - return NULL; -} - -static void -rb_ractor_sched_wait(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf, void *ptr) -{ - struct ractor_waiter *waiter = (struct ractor_waiter *)ptr; - - RACTOR_UNLOCK(cr); - { - rb_nogvl(ractor_wait_no_gvl, waiter, - ubf, waiter, - RB_NOGVL_UBF_ASYNC_SAFE | RB_NOGVL_INTR_FAIL); - } - RACTOR_LOCK(cr); -} - -static void -rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *th) -{ - // ractor lock is acquired - rb_native_cond_broadcast(&r->sync.wakeup_cond); -} -#endif - static bool ractor_wakeup_all(rb_ractor_t *r, enum ractor_wakeup_status wakeup_status) { diff --git a/thread.c b/thread.c index ea404dd66dd97f..4736a8cd67ae75 100644 --- a/thread.c +++ b/thread.c @@ -278,12 +278,19 @@ MAYBE_UNUSED(NOINLINE(static int thread_start_func_2(rb_thread_t *th, VALUE *sta MAYBE_UNUSED(static bool th_has_dedicated_nt(const rb_thread_t *th)); MAYBE_UNUSED(static int waitfd_to_waiting_flag(int wfd_event)); -#include THREAD_IMPL_SRC +#ifdef RB_THREAD_SCHED_NONE +// The no-thread model is not a set of primitives under the common scheduler: +// it replaces the scheduler with stubs, so it stands alone. +# include THREAD_IMPL_SRC +#else +// The scheduler pulls in the platform implementation (THREAD_IMPL_SRC) itself: +// the platform primitives come first, the scheduler is built on top of them. +# include "thread_sched.c" +#endif /* * TODO: somebody with win32 knowledge should be able to get rid of - * timer-thread by busy-waiting on signals. And it should be possible - * to make the GVL in thread_pthread.c be platform-independent. + * timer-thread by busy-waiting on signals. */ #ifndef BUSY_WAIT_SIGNALS # define BUSY_WAIT_SIGNALS (0) @@ -841,7 +848,7 @@ thread_start_func_2(rb_thread_t *th, VALUE *stack_start) // Run the coroutine thread's epilogue here, while th is still valid; // co_start then only makes the final transfer (see - // coroutine_thread_terminated in thread_pthread_mn.c). + // coroutine_thread_terminated in thread_sched_mn.c). coroutine_thread_terminated(th); rb_ractor_postmortem_free(&pf); return 0; @@ -6112,9 +6119,7 @@ rb_check_deadlock(rb_ractor_t *r) { if (GET_THREAD()->vm->thread_ignore_deadlock) return; -#ifdef RUBY_THREAD_PTHREAD_H if (r->threads.sched.readyq_cnt > 0) return; -#endif int sleeper_num = rb_ractor_sleeper_thread_num(r); int ltnum = rb_ractor_living_thread_num(r); diff --git a/thread_none.c b/thread_none.c index 1a3ff8bcd2cd39..7de59b6f9af6f2 100644 --- a/thread_none.c +++ b/thread_none.c @@ -42,12 +42,10 @@ rb_thread_sched_init(struct rb_thread_sched *sched, bool atfork) { } -#if 0 -static void +void rb_thread_sched_destroy(struct rb_thread_sched *sched) { } -#endif // Do nothing for mutex guard void @@ -311,6 +309,25 @@ rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr) // do nothing } +void +rb_ractor_sched_barrier_end(rb_vm_t *vm, rb_ractor_t *cr) +{ + // do nothing +} + +void +rb_ractor_sched_wait(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf, void *ptr) +{ + // nothing can wake this: with no threads there is nobody to send + rb_bug("unreachable"); +} + +void +rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *th) +{ + // do nothing +} + bool diff --git a/thread_none.h b/thread_none.h index ac47e52bdaeba6..7cc045ea1dfbca 100644 --- a/thread_none.h +++ b/thread_none.h @@ -8,12 +8,16 @@ // based implementation in vm.c #define RB_THREAD_LOCAL_SPECIFIER -struct rb_native_thread { - void *thread_id; // NULL -}; +// This model brings its own scheduler stubs (thread_none.c) instead of the +// common one; thread.c keys off this. +#define RB_THREAD_SCHED_NONE 1 -struct rb_thread_sched_item {}; -struct rb_thread_sched {}; +// The scheduler's types are shared with the threaded platforms: the code that +// reads them (vm_sync.c, ractor.c, thread.c) is compiled for every thread +// model, so it needs the real fields even here, where nothing ever runs +// concurrently and they all stay zero. thread_none.c only ever passes the +// structs around, never looks inside them. +#include "thread_sched.h" RUBY_EXTERN struct rb_execution_context_struct *ruby_current_ec; NOINLINE(struct rb_execution_context_struct *rb_current_ec_noinline(void)); // for assertions diff --git a/thread_pthread.c b/thread_pthread.c index f31cad114a303f..275a4f9b97a4de 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -62,6 +62,11 @@ static pthread_condattr_t *condattr_monotonic = &condattr_mono; static const void *const condattr_monotonic = NULL; #endif +// Whether native_cond_timedwait() takes an rb_hrtime_t deadline as is. It +// does when the condvar counts in the same clock rb_hrtime_now() reads; +// otherwise the caller has to restate the deadline in the condvar's clock. +#define RB_NATIVE_COND_HRTIME_DEADLINE_P() (condattr_monotonic != NULL) + /* A retiring shared native thread frees its own context while the threads it * parked are still suspended with that context as their target. */ #define COROUTINE_TARGET_MAY_BE_FREED 1 @@ -324,21 +329,6 @@ static rb_serial_t current_fork_gen = 1; /* We can't use GET_VM()->fork_gen */ # define USE_UBF_LIST 1 #endif -static void threadptr_trap_interrupt(rb_thread_t *); - -static void native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt); -static void native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt); -static void native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th); - -static void ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r); -static void timer_thread_wakeup(void); -static void timer_thread_wakeup_locked(rb_vm_t *vm); -static void timer_thread_wakeup_force(void); -static void timer_thread_wake_fence(struct rb_thread_struct *th); -static bool ractor_sched_timeout_arm(rb_thread_t *th, const rb_hrtime_t *rel); -static bool ractor_sched_timeout_disarm(rb_thread_t *th); -static void thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th); -static void ractor_sched_cancel_enq(rb_vm_t *vm, struct rb_thread_sched *sched); #if USE_MN_THREADS static void nt_machine_stack_atfork(void); @@ -357,1788 +347,102 @@ struct rb_thread_context { static bool thread_sched_reclaim(struct coroutine_context *dead_co); #endif -static void coroutine_transfer0(struct coroutine_context *transfer_from, - struct coroutine_context *transfer_to, bool to_dead); - -#define thread_sched_dump(s) thread_sched_dump_(__FILE__, __LINE__, s) - -static bool -th_has_dedicated_nt(const rb_thread_t *th) -{ - // TODO: th->has_dedicated_nt - return th->nt->dedicated > 0; -} - -RBIMPL_ATTR_MAYBE_UNUSED() -static void -thread_sched_dump_(const char *file, int line, struct rb_thread_sched *sched) -{ - fprintf(stderr, "@%s:%d running:%d\n", file, line, sched->running ? (int)sched->running->serial : -1); - rb_thread_t *th; - int i = 0; - ccan_list_for_each(&sched->readyq, th, sched.node.readyq) { - i++; if (i>10) rb_bug("too many"); - fprintf(stderr, " ready:%d (%sNT:%d)\n", th->serial, - th->nt ? (th->nt->dedicated ? "D" : "S") : "x", - th->nt ? (int)th->nt->serial : -1); - } -} - -#define ractor_sched_dump(s) ractor_sched_dump_(__FILE__, __LINE__, s) - -RBIMPL_ATTR_MAYBE_UNUSED() -static void -ractor_sched_dump_(const char *file, int line, rb_vm_t *vm) -{ - rb_ractor_t *r; - fprintf(stderr, "ractor_sched_dump %s:%d\n", file, line); - int i = 0; - ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) { - i++; - if (i>10) rb_bug("!!"); - fprintf(stderr, " %d ready:%d\n", i, rb_ractor_id(r)); - } -} - -#define thread_sched_lock(a, b) thread_sched_lock_(a, b, __FILE__, __LINE__) -#define thread_sched_unlock(a, b) thread_sched_unlock_(a, b, __FILE__, __LINE__) - -static void -thread_sched_set_locked(struct rb_thread_sched *sched, rb_thread_t *th) +#ifdef RB_THREAD_T_HAS_NATIVE_ID +static int +get_native_thread_id(void) { -#if VM_CHECK_MODE > 0 - VM_ASSERT(sched->lock_owner == NULL); - - sched->lock_owner = th; +#ifdef __linux__ + return (int)syscall(SYS_gettid); +#elif defined(__FreeBSD__) + return pthread_getthreadid_np(); #endif } - -static void -thread_sched_set_unlocked(struct rb_thread_sched *sched, rb_thread_t *th) -{ -#if VM_CHECK_MODE > 0 - VM_ASSERT(sched->lock_owner == th); - - sched->lock_owner = NULL; #endif -} -static void -thread_sched_lock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line) -{ - rb_native_mutex_lock(&sched->lock_); -#if VM_CHECK_MODE - RUBY_DEBUG_LOG2(file, line, "r:%d th:%u", th ? (int)rb_ractor_id(th->ractor) : -1, rb_th_serial(th)); +#ifdef RB_THREAD_LOCAL_SPECIFIER +static RB_THREAD_LOCAL_SPECIFIER rb_thread_t *ruby_native_thread; #else - RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th)); +static pthread_key_t ruby_native_thread_key; #endif - thread_sched_set_locked(sched, th); -} - static void -thread_sched_unlock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line) +null_func(int i) { - RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th)); - - thread_sched_set_unlocked(sched, th); - - rb_native_mutex_unlock(&sched->lock_); + /* null */ + // This function can be called from signal handler + // RUBY_DEBUG_LOG("i:%d", i); } -#if USE_MN_THREADS -// Like thread_sched_unlock(), but never dereferences th (the debug log above -// reads th->serial). For the MN termination epilogue, which unlocks after th -// may already be collectable. Keep in sync with thread_sched_unlock_. -static void -thread_sched_unlock_no_log(struct rb_thread_sched *sched, rb_thread_t *th) +rb_thread_t * +ruby_thread_from_native(void) { - thread_sched_set_unlocked(sched, th); // pointer compare only - - rb_native_mutex_unlock(&sched->lock_); -} +#ifdef RB_THREAD_LOCAL_SPECIFIER + return ruby_native_thread; +#else + return pthread_getspecific(ruby_native_thread_key); #endif +} -static void -ASSERT_thread_sched_locked(struct rb_thread_sched *sched, rb_thread_t *th) +int +ruby_thread_set_native(rb_thread_t *th) { - VM_ASSERT(rb_native_mutex_trylock(&sched->lock_) == EBUSY); - -#if VM_CHECK_MODE if (th) { - VM_ASSERT(sched->lock_owner == th); - } - else { - VM_ASSERT(sched->lock_owner != NULL); - } +#ifdef USE_UBF_LIST + ccan_list_node_init(&th->sched.node.ubf); #endif -} - -#define ractor_sched_lock(a, b) ractor_sched_lock_(a, b, __FILE__, __LINE__) -#define ractor_sched_unlock(a, b) ractor_sched_unlock_(a, b, __FILE__, __LINE__) - -RBIMPL_ATTR_MAYBE_UNUSED() -static unsigned int -rb_ractor_serial(const rb_ractor_t *r) -{ - if (r) { - return rb_ractor_id(r); - } - else { - return 0; } -} - -static void -ractor_sched_set_locked(rb_vm_t *vm, rb_ractor_t *cr) -{ -#if VM_CHECK_MODE > 0 - VM_ASSERT(vm->ractor.sched.lock_owner == NULL); - VM_ASSERT(vm->ractor.sched.locked == false); - - vm->ractor.sched.lock_owner = cr; - vm->ractor.sched.locked = true; -#endif -} - -static void -ractor_sched_set_unlocked(rb_vm_t *vm, rb_ractor_t *cr) -{ -#if VM_CHECK_MODE > 0 - VM_ASSERT(vm->ractor.sched.locked); - VM_ASSERT(vm->ractor.sched.lock_owner == cr); - - vm->ractor.sched.locked = false; - vm->ractor.sched.lock_owner = NULL; -#endif -} - -static void -ractor_sched_lock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line) -{ - rb_native_mutex_lock(&vm->ractor.sched.lock); + // setup TLS -#if VM_CHECK_MODE - RUBY_DEBUG_LOG2(file, line, "cr:%u prev_owner:%u", rb_ractor_serial(cr), rb_ractor_serial(vm->ractor.sched.lock_owner)); + if (th && th->ec) { + rb_ractor_set_current_ec(th->ractor, th->ec); + } +#ifdef RB_THREAD_LOCAL_SPECIFIER + ruby_native_thread = th; + return 1; #else - RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr)); + return pthread_setspecific(ruby_native_thread_key, th) == 0; #endif - - ractor_sched_set_locked(vm, cr); -} - -static void -ractor_sched_unlock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line) -{ - RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr)); - - ractor_sched_set_unlocked(vm, cr); - rb_native_mutex_unlock(&vm->ractor.sched.lock); -} - -static void -ASSERT_ractor_sched_locked(rb_vm_t *vm, rb_ractor_t *cr) -{ - VM_ASSERT(rb_native_mutex_trylock(&vm->ractor.sched.lock) == EBUSY); - VM_ASSERT(vm->ractor.sched.locked); - VM_ASSERT(cr == NULL || vm->ractor.sched.lock_owner == cr); -} - -static void ractor_sched_barrier_join_signal_locked(rb_vm_t *vm); - -/* ntlist registration: a thread that executes Ruby code is always registered, - * in its snt's nt->running_th or on running_dnts via its dedicated nt. The - * only unregistered execution is scheduler glue (parking, resuming), which - * touches no Ruby heap, and the barrier wait below. */ -static void -ntlist_add_running(rb_vm_t *vm, rb_thread_t *th) -{ - struct rb_native_thread *nt = th->nt; - - // a dedicated nt is not on the snts list the scans walk: running_dnts instead - if (nt != NULL && nt->dedicated == 0) { - rb_native_mutex_lock(&nt->running_th_lock); - { - VM_ASSERT(nt->running_th == NULL); - nt->running_th = th; - } - rb_native_mutex_unlock(&nt->running_th_lock); - } - else { - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); - { - // an snt gone dedicated (rb_thread_lock_native_thread) has no - // creation-time running_thread: the registration supplies it - nt->running_thread = th; - ccan_list_add(&vm->ractor.sched.ntlist.running_dnts, &nt->running_dnts_node); - } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); - } -} - -// Returns whether the active barrier's walk had counted this registration: -// such a deregistration owes the snapshot count a decrement. Read and -// cleared under the registration's own lock, so it pairs with the walk. -static bool -ntlist_del_running(rb_vm_t *vm, rb_thread_t *th) -{ - struct rb_native_thread *nt = th->nt; - uint32_t serial; - bool counted; - bool in_running_th; - - // The registration itself says where it is: nt->running_th holds th, or - // th's nt hangs on running_dnts. barrier_serial is read inside the - // registration's lock, ordered with the walk that stamped there. - rb_native_mutex_lock(&nt->running_th_lock); - { - in_running_th = (nt->running_th == th); - if (in_running_th) { - nt->running_th = NULL; - serial = vm->ractor.sched.barrier_serial; - counted = (nt->barrier_counted_serial == serial); - nt->barrier_counted_serial = serial - 1; // only once per barrier - } - } - rb_native_mutex_unlock(&nt->running_th_lock); - - if (!in_running_th) { - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); - { - ccan_list_del_init(&nt->running_dnts_node); - serial = vm->ractor.sched.barrier_serial; - counted = (nt->barrier_counted_serial == serial); - nt->barrier_counted_serial = serial - 1; - } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); - } - return counted; -} - -// Stamp a registration into the active barrier's snapshot unless the walk -// already counted it; returns whether it stamped. Called under sched.lock, -// so it is serialized with the walk: the stamp says exactly whether the -// registration came first. -static bool -ntlist_stamp_if_uncounted(rb_vm_t *vm, rb_thread_t *th) -{ - struct rb_native_thread *nt = th->nt; - uint32_t serial = vm->ractor.sched.barrier_serial; // sched.lock is held - bool stamped; - bool in_running_th; - - rb_native_mutex_lock(&nt->running_th_lock); - { - in_running_th = (nt->running_th == th); - if (in_running_th) { - stamped = (nt->barrier_counted_serial != serial); - nt->barrier_counted_serial = serial; - } - } - rb_native_mutex_unlock(&nt->running_th_lock); - - if (!in_running_th) { - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); - { - stamped = (nt->barrier_counted_serial != serial); - nt->barrier_counted_serial = serial; - } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); - } - return stamped; -} - -// Record a thread entering/leaving the running set, with no global lock and -// no count: the records themselves are what the barrier counts. Pairing: -// the barrier sets barrier_is_waiting and then walks the records under their -// locks; we move a record and then read the flag, so one side sees the other. -// List sched for the timer's timeslice ticks. The caller holds sched->lock_ -// with the readyq non-empty, so the timer cannot prune the entry meanwhile. -static void -timeslice_sched_link(rb_vm_t *vm, struct rb_thread_sched *sched) -{ - rb_native_mutex_lock(&vm->ractor.sched.timeslice.lock); - { - if (sched->timeslice_node.next == &sched->timeslice_node) { - ccan_list_add_tail(&vm->ractor.sched.timeslice.scheds, &sched->timeslice_node); - } - } - rb_native_mutex_unlock(&vm->ractor.sched.timeslice.lock); -} - -static void -thread_sched_setup_running_threads(struct rb_thread_sched *sched, rb_ractor_t *cr, rb_vm_t *vm, - rb_thread_t *add_th, rb_thread_t *del_th) -{ - RUBY_DEBUG_LOG("+:%u -:%u", rb_th_serial(add_th), rb_th_serial(del_th)); - - if (del_th) { - bool counted = ntlist_del_running(vm, del_th); - sched->is_running = false; - - // The first load is only a filter; the one under sched.lock decides. - // A missed flag means this deregistration preceded the barrier's walk. - if (UNLIKELY(RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting))) { - ractor_sched_lock(vm, cr); - { - if (RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting)) { - if (counted) { - VM_ASSERT(vm->ractor.sched.barrier_running_cnt > 0); - vm->ractor.sched.barrier_running_cnt--; - } - ractor_sched_barrier_join_signal_locked(vm); - } - } - ractor_sched_unlock(vm, cr); - } - } - - if (add_th) { - ntlist_add_running(vm, add_th); - - if (UNLIKELY(RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting))) { - // A stop-the-world section. In its waiting phase sched.lock is - // takable: join the snapshot count and take the interrupt (this - // thread joins at its next check, like any walked runner). In - // the GC phase the barrier holds sched.lock to its end, so this - // blocks here, as the old global-lock design did. - ractor_sched_lock(vm, cr); - { - if (RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting) && - ntlist_stamp_if_uncounted(vm, add_th)) { - // the walk ran before this registration; count it in - RUBY_DEBUG_LOG("barrier_is_waiting"); - vm->ractor.sched.barrier_running_cnt++; - RUBY_VM_SET_VM_BARRIER_INTERRUPT(add_th->ec); - } - } - ractor_sched_unlock(vm, cr); - } - - sched->is_running = true; - - // taking a turn with waiters already queued needs the timeslice ticks - if (!ccan_list_empty(&sched->readyq)) { - timeslice_sched_link(vm, sched); - ractor_sched_lock(vm, cr); - { - if (vm->ractor.sched.timeslice_wait_inf) { - timer_thread_wakeup_locked(vm); - } - } - ractor_sched_unlock(vm, cr); - } - } -} - -static void -thread_sched_add_running_thread(struct rb_thread_sched *sched, rb_thread_t *th) -{ - ASSERT_thread_sched_locked(sched, th); - VM_ASSERT(sched->running == th); - - rb_vm_t *vm = th->vm; - thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL); -} - -static void -thread_sched_del_running_thread(struct rb_thread_sched *sched, rb_thread_t *th) -{ - ASSERT_thread_sched_locked(sched, th); - - rb_vm_t *vm = th->vm; - thread_sched_setup_running_threads(sched, th->ractor, vm, NULL, th); } -void -rb_add_running_thread(rb_thread_t *th) -{ - struct rb_thread_sched *sched = TH_SCHED(th); +static void native_thread_setup(struct rb_native_thread *nt); +static void native_thread_setup_on_thread(struct rb_native_thread *nt); - thread_sched_lock(sched, th); - { - thread_sched_add_running_thread(sched, th); - } - thread_sched_unlock(sched, th); -} +// Internal cache of page size: +static size_t RB_THREAD_PAGE_SIZE; void -rb_del_running_thread(rb_thread_t *th) -{ - struct rb_thread_sched *sched = TH_SCHED(th); - - thread_sched_lock(sched, th); - { - thread_sched_del_running_thread(sched, th); - } - thread_sched_unlock(sched, th); -} - -// setup current or next running thread -// sched->running should be set only on this function. -// -// if th is NULL, there is no running threads. -static void -thread_sched_set_running(struct rb_thread_sched *sched, rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%u->th:%u", rb_th_serial(sched->running), rb_th_serial(th)); - VM_ASSERT(sched->running != th); - - if (RUBY_DTRACE_RTS_SET_RUNNING_ENABLED()) { - RUBY_DTRACE_RTS_SET_RUNNING(sched, sched->running, th); - } - - sched->running = th; -} - -RBIMPL_ATTR_MAYBE_UNUSED() -static bool -thread_sched_readyq_contain_p(struct rb_thread_sched *sched, rb_thread_t *th) -{ - rb_thread_t *rth; - ccan_list_for_each(&sched->readyq, rth, sched.node.readyq) { - if (rth == th) { - VM_ASSERT(th->sched.node.is_ready); - return true; - } - } - VM_ASSERT(!th->sched.node.is_ready); - return false; -} - -// deque thread from the ready queue. -// if the ready queue is empty, return NULL. -// -// return deque'ed running thread (or NULL). -static rb_thread_t * -thread_sched_deq(struct rb_thread_sched *sched) -{ - ASSERT_thread_sched_locked(sched, NULL); - rb_thread_t *next_th; - - VM_ASSERT(sched->running != NULL); - - if (ccan_list_empty(&sched->readyq)) { - next_th = NULL; - } - else { - next_th = ccan_list_pop(&sched->readyq, rb_thread_t, sched.node.readyq); - VM_ASSERT(next_th->sched.node.is_ready); - next_th->sched.node.is_ready = false; - - VM_ASSERT(sched->readyq_cnt > 0); - sched->readyq_cnt--; - ccan_list_node_init(&next_th->sched.node.readyq); - } - - RUBY_DEBUG_LOG("next_th:%u readyq_cnt:%d", rb_th_serial(next_th), sched->readyq_cnt); - - return next_th; -} - -// enqueue ready thread to the ready queue. -static void -thread_sched_enq(struct rb_thread_sched *sched, rb_thread_t *ready_th) +Init_native_thread(rb_thread_t *main_th) { - ASSERT_thread_sched_locked(sched, NULL); - RUBY_DEBUG_LOG("ready_th:%u readyq_cnt:%d", rb_th_serial(ready_th), sched->readyq_cnt); - - VM_ASSERT(sched->running != NULL); - VM_ASSERT(!thread_sched_readyq_contain_p(sched, ready_th)); - - bool timeslice_onset = sched->is_running && ccan_list_empty(&sched->readyq); - - ccan_list_add_tail(&sched->readyq, &ready_th->sched.node.readyq); - ready_th->sched.node.is_ready = true; - sched->readyq_cnt++; - - if (timeslice_onset) { - // The running thread needs the timeslice ticks now. Linked before - // the check under sched.lock: either the timer's scan (same lock) - // sees the sched, or this sees timeslice_wait_inf. - rb_vm_t *vm = ready_th->vm; - timeslice_sched_link(vm, sched); - ractor_sched_lock(vm, NULL); - { - if (vm->ractor.sched.timeslice_wait_inf) { - timer_thread_wakeup_locked(vm); - } - } - ractor_sched_unlock(vm, NULL); - } -} + // Get the system page size for later use in stack allocation and stack overflow checks: + RB_THREAD_PAGE_SIZE = sysconf(_SC_PAGESIZE); -// DNT: kick condvar -// SNT: TODO -static void -thread_sched_wakeup_running_thread(struct rb_thread_sched *sched, rb_thread_t *next_th, bool will_switch) -{ - ASSERT_thread_sched_locked(sched, NULL); - VM_ASSERT(sched->running == next_th); - - if (next_th) { - if (next_th->nt) { - if (th_has_dedicated_nt(next_th)) { - RUBY_DEBUG_LOG("pinning th:%u", next_th->serial); - rb_native_cond_signal(&next_th->nt->readyq); - } - else { - // TODO - RUBY_DEBUG_LOG("th:%u is already running.", next_th->serial); - } - } - else { - if (will_switch) { - RUBY_DEBUG_LOG("th:%u (do nothing)", rb_th_serial(next_th)); - } - else { - RUBY_DEBUG_LOG("th:%u (enq)", rb_th_serial(next_th)); - ractor_sched_enq(next_th->vm, next_th->ractor); - } +#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) + if (condattr_monotonic) { + int r = pthread_condattr_init(condattr_monotonic); + if (r == 0) { + r = pthread_condattr_setclock(condattr_monotonic, CLOCK_MONOTONIC); } + if (r) condattr_monotonic = NULL; } - else { - RUBY_DEBUG_LOG("no waiting threads%s", ""); - } -} - -// waiting -> ready (locked) -static void -thread_sched_to_ready_common(struct rb_thread_sched *sched, rb_thread_t *th, bool wakeup, bool will_switch) -{ - RUBY_DEBUG_LOG("th:%u running:%u redyq_cnt:%d", rb_th_serial(th), rb_th_serial(sched->running), sched->readyq_cnt); - - VM_ASSERT(sched->running != th); - VM_ASSERT(!thread_sched_readyq_contain_p(sched, th)); - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_READY, th); +#endif - if (sched->running == NULL) { - thread_sched_set_running(sched, th); - if (wakeup) thread_sched_wakeup_running_thread(sched, th, will_switch); +#ifndef RB_THREAD_LOCAL_SPECIFIER + if (pthread_key_create(&ruby_native_thread_key, 0) == EAGAIN) { + rb_bug("pthread_key_create failed (ruby_native_thread_key)"); } - else { - thread_sched_enq(sched, th); + if (pthread_key_create(&ruby_current_ec_key, 0) == EAGAIN) { + rb_bug("pthread_key_create failed (ruby_current_ec_key)"); } -} +#endif + ruby_posix_signal(SIGVTALRM, null_func); -// waiting -> ready -// -// `th` had became "waiting" state by `thread_sched_to_waiting` -// and `thread_sched_to_ready` enqueue `th` to the thread ready queue. -RBIMPL_ATTR_MAYBE_UNUSED() -static void -thread_sched_to_ready(struct rb_thread_sched *sched, rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); - - thread_sched_lock(sched, th); - { - thread_sched_to_ready_common(sched, th, true, false); - } - thread_sched_unlock(sched, th); -} - -// wait until sched->running is `th`. `end` is an absolute deadline for a dedicated -static void -thread_sched_wait_running_turn(struct rb_thread_sched *sched, rb_thread_t *th, bool can_direct_transfer, const rb_hrtime_t *end) -{ - RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); - - ASSERT_thread_sched_locked(sched, th); - VM_ASSERT(th == rb_ec_thread_ptr(rb_current_ec_noinline())); - - bool timedout = false; - - if (th != sched->running) { - // TODO: This optimization should also be made to work for MN_THREADS - if (th->has_dedicated_nt && th == sched->runnable_hot_th && (sched->running == NULL || sched->running->has_dedicated_nt)) { - RUBY_DEBUG_LOG("(nt) stealing: hot-th:%u. running:%u", rb_th_serial(th), rb_th_serial(sched->running)); - - // th serves itself on its own nt, displacing the enqueued - // running thread back to the readyq: cancel the entry that was - // posted for it (a later dequeue would find this Ractor served - // and its next enqueue would double-list the node) - ractor_sched_cancel_enq(th->vm, sched); - - // If there is a thread set to run, move it back to the front of the readyq - if (sched->running != NULL) { - rb_thread_t *running = sched->running; - VM_ASSERT(!thread_sched_readyq_contain_p(sched, running)); - running->sched.node.is_ready = true; - ccan_list_add(&sched->readyq, &running->sched.node.readyq); - sched->readyq_cnt++; - } - - // Pull off the ready queue and start running. - if (th->sched.node.is_ready) { - VM_ASSERT(thread_sched_readyq_contain_p(sched, th)); - ccan_list_del_init(&th->sched.node.readyq); - th->sched.node.is_ready = false; - sched->readyq_cnt--; - } - thread_sched_set_running(sched, th); - rb_ractor_thread_switch(th->ractor, th, false); - } - else if (th == sched->runnable_hot_th) { - // The hot thread cannot steal the control (e.g. the running thread - // is an MN thread). It is going to sleep, so it is no longer spinning; - // drop the hint so that other threads don't yield the lock to it. - sched->runnable_hot_th = NULL; - sched->runnable_hot_th_waiting = 0; - } - - // already deleted from running threads - - - // wait for execution right - rb_thread_t *next_th; - while((next_th = sched->running) != th) { - if (th_has_dedicated_nt(th)) { - RUBY_DEBUG_LOG("(nt) sleep th:%u running:%u", rb_th_serial(th), rb_th_serial(sched->running)); - - thread_sched_set_unlocked(sched, th); - { - RUBY_DEBUG_LOG("nt:%d cond:%p", th->nt->serial, &th->nt->readyq); - rb_nativethread_cond_t *cond = &th->nt->readyq; - - // Once someone has queued this thread the deadline is spent: it - // is waiting for a turn, not for the time, and arming a kernel - // timer for every round of that costs more than the wait. - // Once someone has queued this thread the deadline is spent: it - // is waiting for a turn, not for the time, and arming a kernel - // timer for every round of that costs more than the wait. - if (end && !th->sched.node.is_ready) { - rb_hrtime_t abs = *end; - - if (!condattr_monotonic) { - // the condvar counts in another clock: restate it there - rb_hrtime_t now = rb_hrtime_now(); - abs = native_cond_timeout(cond, *end > now ? *end - now : 0); - } - timedout = native_cond_timedwait(cond, &sched->lock_, &abs) == ETIMEDOUT; - } - else { - rb_native_cond_wait(cond, &sched->lock_); - } - } - thread_sched_set_locked(sched, th); - - if (timedout && - sched->running != th && !th->sched.node.is_ready) { - // the deadline passed and nobody woke this thread: get back in - // line for the running turn, then wait for it without a deadline - thread_sched_to_ready_common(sched, th, false, false); - end = NULL; - } - - if (sched->runnable_hot_th != NULL && sched->runnable_hot_th_waiting) { - VM_ASSERT(sched->runnable_hot_th != th); - // Give the hot thread a chance to preempt, if it's actively spinning. - // On multicore, this reduces the rate of core-switching. On single-core it - // should mostly be a nop, since the other thread can't be concurrently spinning. - thread_sched_unlock(sched, th); - thread_sched_lock(sched, th); - } - - RUBY_DEBUG_LOG("(nt) wakeup %s", sched->running == th ? "success" : "failed"); - if (th == sched->running) { - rb_ractor_thread_switch(th->ractor, th, false); - } - } - else { - // search another ready thread - if (can_direct_transfer && - (next_th = sched->running) != NULL && - !next_th->nt // next_th is running or has dedicated nt - ) { - - RUBY_DEBUG_LOG("th:%u->%u (direct)", rb_th_serial(th), rb_th_serial(next_th)); - - thread_sched_set_unlocked(sched, th); - { - rb_ractor_set_current_ec(th->ractor, NULL); - thread_sched_switch(th, next_th); - } - thread_sched_set_locked(sched, th); - } - else { - // search another ready ractor - struct rb_native_thread *nt = th->nt; - native_thread_assign(NULL, th); - - RUBY_DEBUG_LOG("th:%u->%u (ractor scheduling)", rb_th_serial(th), rb_th_serial(next_th)); - - thread_sched_set_unlocked(sched, th); - { - rb_ractor_set_current_ec(th->ractor, NULL); - coroutine_transfer0(th->sched.context, nt->nt_context, false); - } - thread_sched_set_locked(sched, th); - } - - VM_ASSERT(rb_current_ec_noinline() == th->ec); - } - } - - VM_ASSERT(th->nt != NULL); - VM_ASSERT(rb_current_ec_noinline() == th->ec); - VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none); - - // add th to running threads - thread_sched_add_running_thread(sched, th); - } - - // Control transfer to the current thread is now complete. The original thread - // cannot steal control at this point. - sched->runnable_hot_th = NULL; - sched->runnable_hot_th_waiting = 0; - - - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_RESUMED, th); -} - -// waiting -> ready -> running (locked) -static void -thread_sched_to_running_common(struct rb_thread_sched *sched, rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%u dedicated:%d", rb_th_serial(th), th_has_dedicated_nt(th)); - - VM_ASSERT(sched->running != th); - VM_ASSERT(th_has_dedicated_nt(th)); - VM_ASSERT(GET_THREAD() == th); - - native_thread_dedicated_dec(th->vm, th->ractor, th->nt); - - // waiting -> ready - thread_sched_to_ready_common(sched, th, false, false); - - if (sched->running == th) { - thread_sched_add_running_thread(sched, th); - } - - // TODO: check SNT number - thread_sched_wait_running_turn(sched, th, false, NULL); -} - -// waiting -> ready -> running -// -// `th` had been waiting by `thread_sched_to_waiting()` -// and run a dedicated task (like waitpid and so on). -// After the dedicated task, this function is called -// to join a normal thread-scheduling. -static void -thread_sched_to_running(struct rb_thread_sched *sched, rb_thread_t *th) -{ - // We are reading and writing these sched fields without lock cover, but - // there are no correctness issues resulting from stale cache or delayed writeback. - // When it works, this causes the next-scheduled thread to yield the sched lock - // briefly so that we can grab it if we're still spinning (not descheduled yet). - if (sched->runnable_hot_th == th) { - sched->runnable_hot_th_waiting = 1; - } - thread_sched_lock(sched, th); - { - thread_sched_to_running_common(sched, th); - } - thread_sched_unlock(sched, th); -} - -// resume a next thread in the thread ready queue. -// -// deque next running thread from the ready thread queue and -// resume this thread if available. -// -// If the next therad has a dedicated native thraed, simply signal to resume. -// Otherwise, make the ractor ready and other nt will run the ractor and the thread. -static void -thread_sched_wakeup_next_thread(struct rb_thread_sched *sched, rb_thread_t *th, bool will_switch) -{ - ASSERT_thread_sched_locked(sched, th); - - VM_ASSERT(sched->running == th); - VM_ASSERT(sched->running->nt != NULL); - - rb_thread_t *next_th = thread_sched_deq(sched); - - RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th)); - VM_ASSERT(th != next_th); - - thread_sched_set_running(sched, next_th); - VM_ASSERT(next_th == sched->running); - thread_sched_wakeup_running_thread(sched, next_th, will_switch); - - if (th != next_th) { - thread_sched_del_running_thread(sched, th); - } -} - -// running -> dead (locked) -static void -thread_sched_to_dead_common(struct rb_thread_sched *sched, rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated); - - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); - - // A dying coroutine thread (will_switch=true here) does NOT wake the - // next thread now: it is still winding down (co_start's epilogue), and - // the same Ractor must not have two threads executing at once. The - // epilogue enqueues the Ractor after its last rb_ractor_t access. - thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th)); - - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_EXITED, th); -} - -// running -> dead -static void -thread_sched_to_dead(struct rb_thread_sched *sched, rb_thread_t *th) -{ - // wait out any pending wake here, while th's Ractor is still alive - timer_thread_wake_fence(th); - - thread_sched_lock(sched, th); - { - thread_sched_to_dead_common(sched, th); - } - thread_sched_unlock(sched, th); -} - -// running -> waiting (locked) -// -// This thread will run dedicated task (th->nt->dedicated++). -static void -thread_sched_to_waiting_common(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately) -{ - RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated); - - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); - - native_thread_dedicated_inc(th->vm, th->ractor, th->nt); - if (!yield_immediately) { - sched->runnable_hot_th = th; - sched->runnable_hot_th_waiting = 0; - } - thread_sched_wakeup_next_thread(sched, th, false); -} - -// running -> waiting -// -// This thread will run a dedicated task. -static void -thread_sched_to_waiting(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately) -{ - thread_sched_lock(sched, th); - { - thread_sched_to_waiting_common(sched, th, yield_immediately); - } - thread_sched_unlock(sched, th); -} - -// mini utility func -// return true if any there are any interrupts -static bool -ubf_set(rb_thread_t *th, rb_unblock_function_t *func, void *arg, rb_atomic_t *event_serial) -{ - VM_ASSERT(func != NULL); - - retry: - if (RUBY_VM_INTERRUPTED(th->ec)) { - RUBY_DEBUG_LOG("interrupted:0x%x", th->ec->interrupt_flag); - return true; - } - - rb_native_mutex_lock(&th->interrupt_lock); - { - if (!th->ec->raised_flag && RUBY_VM_INTERRUPTED(th->ec)) { - rb_native_mutex_unlock(&th->interrupt_lock); - goto retry; - } - - VM_ASSERT(th->unblock.func == NULL); - th->unblock.func = func; - th->unblock.arg = arg; - if (event_serial) { - rb_atomic_t prev_serial = RUBY_ATOMIC_FETCH_ADD(th->unblock.event_serial, 1); - *event_serial = prev_serial+1; - } - } - rb_native_mutex_unlock(&th->interrupt_lock); - - return false; -} - -static void -ubf_clear(rb_thread_t *th, bool clear_serial) -{ - rb_native_mutex_lock(&th->interrupt_lock); - { - th->unblock.func = NULL; - th->unblock.arg = NULL; - if (clear_serial) { - RUBY_ATOMIC_ADD(th->unblock.event_serial, 1); - } - } - rb_native_mutex_unlock(&th->interrupt_lock); -} - -static void -ubf_waiting(void *ptr) -{ - rb_thread_t *th = (rb_thread_t *)ptr; - struct rb_thread_sched *sched = TH_SCHED(th); - - // only once. it is safe because th->interrupt_lock is already acquired. - th->unblock.func = NULL; - th->unblock.arg = NULL; - - RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); - - thread_sched_lock(sched, th); - { - if (sched->running == th || th->sched.node.is_ready) { - // not sleeping yet, or a deadline already put it back in line - } - else { - thread_sched_to_ready_common(sched, th, true, false); - - // If the turn is taken, th stays parked until the running thread yields. - // For a timed wait, wake it early anyway: it re-parks at once, but its - // wakeup then runs on another core in parallel with the running thread, - // off the handoff path. An untimed wait has no post-wake bookkeeping - // worth pipelining, so it skips the extra futex round. - if (sched->running != th && th->sched.waiting_timed && - th->nt != NULL && th_has_dedicated_nt(th)) { - rb_native_cond_signal(&th->nt->readyq); - } - } - } - thread_sched_unlock(sched, th); -} - -// running -> waiting -// -// This thread will sleep until other thread wakeup the thread. `end` is an -// absolute deadline, NULL to sleep until woken; only a dedicated native thread, -// which parks on its own condvar, can take one. -static void -thread_sched_to_waiting_until_wakeup(struct rb_thread_sched *sched, rb_thread_t *th, const rb_hrtime_t *end) -{ - RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); - - VM_ASSERT(end == NULL || th_has_dedicated_nt(th)); - - RB_VM_SAVE_MACHINE_CONTEXT(th); - - - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); - - thread_sched_lock(sched, th); - { - // NOTE: there's a lock ordering inversion here with the ubf call, but it's benign. - if (ubf_set(th, ubf_waiting, (void *)th, NULL)) { - RUBY_DEBUG_LOG("th:%u interrupted", rb_th_serial(th)); - } - else { - bool can_direct_transfer = !th_has_dedicated_nt(th); - th->sched.waiting_timed = (end != NULL); // never true here for M:N (end is NULL) - // NOTE: th->status is set before and after this sleep outside of this function in `sleep_forever` - thread_sched_wakeup_next_thread(sched, th, can_direct_transfer); - thread_sched_wait_running_turn(sched, th, can_direct_transfer, end); - th->sched.waiting_timed = false; - } - } - thread_sched_unlock(sched, th); - - ubf_clear(th, false); -} - -// run another thread in the ready queue. -// continue to run if there are no ready threads. -static void -thread_sched_yield(struct rb_thread_sched *sched, rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%d sched->readyq_cnt:%d", (int)th->serial, sched->readyq_cnt); - - thread_sched_lock(sched, th); - { - if (!ccan_list_empty(&sched->readyq)) { - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); - thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th)); - bool can_direct_transfer = !th_has_dedicated_nt(th); - thread_sched_to_ready_common(sched, th, false, can_direct_transfer); - thread_sched_wait_running_turn(sched, th, can_direct_transfer, NULL); - th->status = THREAD_RUNNABLE; - } - else { - VM_ASSERT(sched->readyq_cnt == 0); - } - } - thread_sched_unlock(sched, th); -} - -void -rb_thread_sched_init(struct rb_thread_sched *sched, bool atfork) -{ - rb_native_mutex_initialize(&sched->lock_); - -#if VM_CHECK_MODE - sched->lock_owner = NULL; -#endif - - ccan_list_head_init(&sched->readyq); - sched->readyq_cnt = 0; - ccan_list_node_init(&sched->grq_node); // self-linked = not enqueued - ccan_list_node_init(&sched->timeslice_node); - -#if USE_MN_THREADS - if (!atfork) sched->enable_mn_threads = true; // MN is enabled on Ractors -#endif -} - -static void -coroutine_transfer0(struct coroutine_context *transfer_from, struct coroutine_context *transfer_to, bool to_dead) -{ -#ifdef RUBY_ASAN_ENABLED - void **fake_stack = to_dead ? NULL : &transfer_from->fake_stack; - __sanitizer_start_switch_fiber(fake_stack, transfer_to->stack_base, transfer_to->stack_size); -#endif - -#if defined(COROUTINE_SANITIZE_THREAD) - /* Tell TSan we are switching to transfer_to's fiber before the stack - * switch, so its per-thread shadow stack stays bound to the right - * coroutine. */ - __tsan_switch_to_fiber(transfer_to->tsan_fiber, 0); -#endif - - RBIMPL_ATTR_MAYBE_UNUSED() - struct coroutine_context *returning_from = coroutine_transfer(transfer_from, transfer_to); - - /* if to_dead was passed, the caller is promising that this coroutine is finished and it should - * never be resumed! */ - VM_ASSERT(!to_dead); -#ifdef RUBY_ASAN_ENABLED - __sanitizer_finish_switch_fiber(transfer_from->fake_stack, - (const void**)&returning_from->stack_base, &returning_from->stack_size); -#endif -} - -static void -thread_sched_switch0(struct coroutine_context *current_cont, rb_thread_t *next_th, struct rb_native_thread *nt, bool to_dead) -{ - VM_ASSERT(!nt->dedicated); - VM_ASSERT(next_th->nt == NULL); - - RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th)); - - // this direct transfer serves next_th without a dequeue; cancel its - // Ractor's outstanding grq entry (no-op when nothing is enqueued) - ractor_sched_cancel_enq(next_th->vm, TH_SCHED(next_th)); - - ruby_thread_set_native(next_th); - native_thread_assign(nt, next_th); - - coroutine_transfer0(current_cont, next_th->sched.context, to_dead); -} - -static void -thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th) -{ - struct rb_native_thread *nt = cth->nt; - native_thread_assign(NULL, cth); - RUBY_DEBUG_LOG("th:%u->%u on nt:%d", rb_th_serial(cth), rb_th_serial(next_th), nt->serial); - thread_sched_switch0(cth->sched.context, next_th, nt, cth->status == THREAD_KILLED); -} - -#if VM_CHECK_MODE > 0 -RBIMPL_ATTR_MAYBE_UNUSED() -static unsigned int -grq_size(rb_vm_t *vm, rb_ractor_t *cr) -{ - ASSERT_ractor_sched_locked(vm, cr); - - rb_ractor_t *r, *prev_r = NULL; - unsigned int i = 0; - - ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) { - i++; - - VM_ASSERT(r != prev_r); - prev_r = r; - } - return i; -} -#endif - -// A native thread enters/leaves an epilogue that outlives its Ractor: from -// the increment until the decrement, ruby_vm_destruct waits for it below. -// The increment must happen while the VM still counts the thread's Ractor, -// so that the two never look absent at the same time. -void -rb_thread_sched_winding_begin(rb_vm_t *vm) -{ - RUBY_ATOMIC_INC(vm->ractor.sched.winding_cnt); -} - -void -rb_thread_sched_winding_end(rb_vm_t *vm) -{ - VM_ASSERT(RUBY_ATOMIC_LOAD(vm->ractor.sched.winding_cnt) > 0); - RUBY_ATOMIC_DEC(vm->ractor.sched.winding_cnt); -} - -// ruby_vm_destruct: wait until no native thread is between a coroutine -// epilogue and its reclaim -- past that point the reclaim frees through the -// (about to be destroyed) objspace and reads the (about to be unset) VM. -// Runs without the VM lock, which the epilogue needs to progress. -void -rb_thread_sched_wait_winding(rb_vm_t *vm) -{ - while (RUBY_ATOMIC_LOAD(vm->ractor.sched.winding_cnt) > 0) { - native_thread_yield(); - } -} - -// A direct service of a runnable thread (direct transfer or the hot-thread -// steal) bypasses the grq; cancel the Ractor's outstanding entry so that -// "enqueued <=> runnable and unserved" keeps holding. The caller holds the -// per-Ractor sched lock, so no concurrent enqueue can relink the node: a -// self-linked read needs no lock (the common case -- direct switches whose -// transition never enqueued). A linked read can race only with a dequeue, -// hence the recheck under the grq lock. -static void -ractor_sched_cancel_enq(rb_vm_t *vm, struct rb_thread_sched *sched) -{ - if (sched->grq_node.next != &sched->grq_node) { - ractor_sched_lock(vm, NULL); - { - if (sched->grq_node.next != &sched->grq_node) { - ccan_list_del_init(&sched->grq_node); - VM_ASSERT(vm->ractor.sched.grq_cnt > 0); - vm->ractor.sched.grq_cnt--; - } - } - ractor_sched_unlock(vm, NULL); - } -} - -static void -ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r) -{ - struct rb_thread_sched *sched = &r->threads.sched; - rb_ractor_t *cr = NULL; // timer thread can call this function - - VM_ASSERT(sched->running != NULL); - VM_ASSERT(sched->running->nt == NULL); - - ractor_sched_lock(vm, cr); - { - // Precondition: not already enqueued (the grq_node is self-linked). - // This holds because every service of a runnable-but-unserved thread - // either dequeues the entry (the nt scheduling loop) or cancels it - // (direct transfers / the hot-thread steal; see - // ractor_sched_cancel_enq) -- re-adding a linked node would corrupt - // the queue, so check unconditionally (a CHECK-mode-only assert - // would miss it: the race needs timing that CHECK builds perturb). - if (sched->grq_node.next != &sched->grq_node) { - rb_bug("ractor_sched_enq: already enqueued"); - } - ccan_list_add_tail(&vm->ractor.sched.grq, &sched->grq_node); - vm->ractor.sched.grq_cnt++; - VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt); - - RUBY_DEBUG_LOG("r:%u th:%u grq_cnt:%u", rb_ractor_id(r), rb_th_serial(sched->running), vm->ractor.sched.grq_cnt); - - rb_native_cond_signal(&vm->ractor.sched.cond); - - // The signal reaches a parked snt, and a running one revisits the - // queue in ractor_sched_deq before it can wait (same lock as here). - // With every snt dedicated or retired, only the timer thread's - // timeout branch can serve the entry or widen the pool: wake it - // (a no-op unless it sleeps untimed). - if (RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt) == 0) { - timer_thread_wakeup_locked(vm); - } - - // ractor_sched_dump(vm); - } - ractor_sched_unlock(vm, cr); -} - - -#ifndef MINIMUM_SNT -// make at least MINIMUM_SNT snts for debug. -#define MINIMUM_SNT 0 -#endif - -/* A shared thread woken with nothing to run is one whose turn another thread - * took first. After this many in a row it gives itself back: the queue keeps - * running dry, so the pool is wider than the work. 0 retires on the first one - * and is too eager to be useful; a negative value keeps every thread. */ -#ifndef SNT_IDLE_RETIRE -#define SNT_IDLE_RETIRE 3 -#endif - -/* Never give the last shared thread back. With none left an enqueue has nobody - * to signal, and the only code that makes one runs on the timer thread's - * timeout branch, which is reached only once it has seen a backlog. */ -#define SNT_KEEP_MINIMUM (MINIMUM_SNT > 1 ? MINIMUM_SNT : 1) - -static rb_ractor_t * -ractor_sched_deq(rb_vm_t *vm, rb_ractor_t *cr) -{ - rb_ractor_t *r; - int idle_streak = 0; // consecutive pops that found the queue empty - - ractor_sched_lock(vm, cr); - { - RUBY_DEBUG_LOG("empty? %d", ccan_list_empty(&vm->ractor.sched.grq)); - // ractor_sched_dump(vm); - - VM_ASSERT(rb_current_execution_context(false) == NULL); - VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt); - - while ((r = ccan_list_pop(&vm->ractor.sched.grq, rb_ractor_t, threads.sched.grq_node)) == NULL) { - RUBY_DEBUG_LOG("wait grq_cnt:%d", (int)vm->ractor.sched.grq_cnt); - - if (SNT_IDLE_RETIRE >= 0 && ++idle_streak > SNT_IDLE_RETIRE && - (int)RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt) > SNT_KEEP_MINIMUM) { - RUBY_ATOMIC_DEC(vm->ractor.sched.snt_cnt); - RUBY_DEBUG_LOG("retire, snt_cnt:%d", (int)vm->ractor.sched.snt_cnt); - break; // returning NULL ends this nt; see the caller - } - - ractor_sched_set_unlocked(vm, cr); - rb_native_cond_wait(&vm->ractor.sched.cond, &vm->ractor.sched.lock); - ractor_sched_set_locked(vm, cr); - - RUBY_DEBUG_LOG("wakeup grq_cnt:%d", (int)vm->ractor.sched.grq_cnt); - } - - VM_ASSERT(rb_current_execution_context(false) == NULL); - - if (r) { - ccan_list_node_init(&r->threads.sched.grq_node); // back to self-linked - VM_ASSERT(vm->ractor.sched.grq_cnt > 0); - vm->ractor.sched.grq_cnt--; - RUBY_DEBUG_LOG("r:%d grq_cnt:%u", (int)rb_ractor_id(r), vm->ractor.sched.grq_cnt); - } - else { - // the retire branch is the only way out of the loop without a ractor - VM_ASSERT(idle_streak > SNT_IDLE_RETIRE); - } - } - ractor_sched_unlock(vm, cr); - - return r; -} - -void rb_ractor_lock_self(rb_ractor_t *r); -void rb_ractor_unlock_self(rb_ractor_t *r); - -// The current thread for a ractor is put to "sleep" (descheduled in the STOPPED_FOREVER state) waiting for -// a ractor action to wake it up. -void -rb_ractor_sched_wait(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf, void *ubf_arg) -{ - // ractor lock of cr is acquired - - RUBY_DEBUG_LOG("start%s", ""); - - rb_thread_t * volatile th = rb_ec_thread_ptr(ec); - struct rb_thread_sched *sched = TH_SCHED(th); - struct ractor_waiter *waiter = (struct ractor_waiter*)ubf_arg; - - if (ubf_set(th, ubf, ubf_arg, &waiter->event_serial)) { - // interrupted - return; - } - - thread_sched_lock(sched, th); - rb_ractor_unlock_self(cr); - { - // A dedicated native thread takes the deadline on the very condvar a wakeup - // signals. An M:N thread has no condvar of its own, so its deadline goes to - // the timer thread, which then wakes it the way rb_ractor_sched_wakeup() does. - bool dedicated = th_has_dedicated_nt(th); - const rb_hrtime_t *end_p = NULL; - bool armed = false, expired = false; - - if (waiter->end) { - if (dedicated) { - end_p = waiter->end; - } - else { - // the timer wheel takes a relative timeout - rb_hrtime_t now = rb_hrtime_now(); - rb_hrtime_t rel = *waiter->end > now ? *waiter->end - now : 0; - - armed = ractor_sched_timeout_arm(th, &rel); - expired = !armed; - } - } - - if (expired) { - RUBY_DEBUG_LOG("expired before sleep%s", ""); - } - else if (armed && th->sched.waiting_reason.flags == thread_sched_waiting_none) { - // the timer thread already took this thread out of the wheel; bump the - // serial so that it does not try to wake a thread that never slept - th->sched.event_serial++; - } - else { - // setup sleep - bool can_direct_transfer = !dedicated; - RB_VM_SAVE_MACHINE_CONTEXT(th); - th->status = THREAD_STOPPED_FOREVER; - th->sched.waiting_timed = (end_p != NULL); // never true here for M:N (end_p is NULL) - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); - thread_sched_wakeup_next_thread(sched, th, can_direct_transfer); - // sleep - thread_sched_wait_running_turn(sched, th, can_direct_transfer, end_p); - th->sched.waiting_timed = false; - th->status = THREAD_RUNNABLE; - - // whoever woke this thread took the timeout back first - VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none); - } - } - thread_sched_unlock(sched, th); - rb_ractor_lock_self(cr); - - ubf_clear(th, true); - - RUBY_DEBUG_LOG("end%s", ""); -} - -void -rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *r_th) -{ - // ractor lock of r acquired - struct rb_thread_sched *sched = TH_SCHED(r_th); - - RUBY_DEBUG_LOG("r:%u th:%d", (unsigned int)rb_ractor_id(r), r_th->serial); - - thread_sched_lock(sched, r_th); - { - if (r_th->status == THREAD_STOPPED_FOREVER) { - RUBY_ATOMIC_ADD(r_th->unblock.event_serial, 1); - - // r_th must not resume with a wheel entry left behind: take its timeout - // back, as ubf_event_waiting() does. Only r_th arms it, and it is - // parked here, so reading the flags without the timer lock is safe. - if (r_th->sched.waiting_reason.flags != thread_sched_waiting_none) { - ractor_sched_timeout_disarm(r_th); - } - - // a timeout that fired first may have made r_th runnable already: waking - // it twice would put it on the readyq twice - if (sched->running != r_th && !r_th->sched.node.is_ready) { - r_th->sched.event_serial++; // a timeout still armed must not wake it again - thread_sched_to_ready_common(sched, r_th, true, false); - } - } - } - thread_sched_unlock(sched, r_th); -} - -static bool -ractor_sched_barrier_completed_p(rb_vm_t *vm) -{ - // The snapshot barrier_running_cnt is taken by the barrier's walk and - // decremented by counted deregistrations; no rescan is needed here. - RUBY_DEBUG_LOG("run:%u wait:%u", vm->ractor.sched.barrier_running_cnt, vm->ractor.sched.barrier_joined_cnt); - VM_ASSERT(vm->ractor.sched.barrier_running_cnt - 1 >= vm->ractor.sched.barrier_joined_cnt); - - return (vm->ractor.sched.barrier_running_cnt - vm->ractor.sched.barrier_joined_cnt) == 1; -} - -void -rb_ractor_sched_barrier_start(rb_vm_t *vm, rb_ractor_t *cr) -{ - VM_ASSERT(cr == GET_RACTOR()); - VM_ASSERT(vm->ractor.sync.lock_owner == cr); // VM is locked - VM_ASSERT(!vm->ractor.sched.barrier_is_waiting); - VM_ASSERT(vm->ractor.sched.barrier_joined_cnt == 0); - VM_ASSERT(vm->ractor.sched.barrier_ractor == NULL); - VM_ASSERT(vm->ractor.sched.barrier_lock_rec == 0); - - RUBY_DEBUG_LOG("start serial:%u", vm->ractor.sched.barrier_serial); - - unsigned int lock_rec; - - ractor_sched_lock(vm, cr); - { - RUBY_ATOMIC_SET(vm->ractor.sched.barrier_is_waiting, 1); - vm->ractor.sched.barrier_ractor = cr; - vm->ractor.sched.barrier_lock_rec = vm->ractor.sync.lock_rec; - - // release VM lock - lock_rec = vm->ractor.sync.lock_rec; - vm->ractor.sync.lock_rec = 0; - vm->ractor.sync.lock_owner = NULL; - rb_native_mutex_unlock(&vm->ractor.sync.lock); - - // Interrupt all running threads: running_dnts plus each snt's running_th. - // A switch before this scan is visible to it; one after it sees - // barrier_is_waiting (set above) and waits. - // Interrupt and count every registered runner, stamping each nt so a - // deregistration during this barrier knows it was counted. - rb_thread_t *ith; - unsigned int running_cnt = 0; - uint32_t serial = vm->ractor.sched.barrier_serial; - - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); - { - struct rb_native_thread *dnt; - ccan_list_for_each(&vm->ractor.sched.ntlist.running_dnts, dnt, running_dnts_node) { - ith = dnt->running_thread; - dnt->barrier_counted_serial = serial; - running_cnt++; - if (ith->ractor != cr) { - RUBY_DEBUG_LOG("barrier request to th:%u", rb_th_serial(ith)); - RUBY_VM_SET_VM_BARRIER_INTERRUPT(ith->ec); - } - } - - struct rb_native_thread *nt; - ccan_list_for_each(&vm->ractor.sched.ntlist.snts, nt, snts_node) { - rb_native_mutex_lock(&nt->running_th_lock); - { - ith = nt->running_th; - if (ith != NULL) { - nt->barrier_counted_serial = serial; - running_cnt++; - if (ith->ractor != cr) { - RUBY_DEBUG_LOG("barrier request to th:%u", rb_th_serial(ith)); - RUBY_VM_SET_VM_BARRIER_INTERRUPT(ith->ec); - } - } - } - rb_native_mutex_unlock(&nt->running_th_lock); - } - } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); - - vm->ractor.sched.barrier_running_cnt = running_cnt; - - // wait for other ractors - while (!ractor_sched_barrier_completed_p(vm)) { - ractor_sched_set_unlocked(vm, cr); - rb_native_cond_wait(&vm->ractor.sched.barrier_complete_cond, &vm->ractor.sched.lock); - ractor_sched_set_locked(vm, cr); - } - - RUBY_DEBUG_LOG("completed seirial:%u", vm->ractor.sched.barrier_serial); - - // no other ractors are there - vm->ractor.sched.barrier_serial++; - vm->ractor.sched.barrier_joined_cnt = 0; - rb_native_cond_broadcast(&vm->ractor.sched.barrier_release_cond); - - // acquire VM lock - rb_native_mutex_lock(&vm->ractor.sync.lock); - vm->ractor.sync.lock_rec = lock_rec; - vm->ractor.sync.lock_owner = cr; - } - - // do not release ractor_sched_lock and there is no newly added (resumed) thread - // thread_sched_setup_running_threads -} - -// called from vm_lock_leave if the vm_lock used for barrierred -void -rb_ractor_sched_barrier_end(rb_vm_t *vm, rb_ractor_t *cr) -{ - RUBY_DEBUG_LOG("serial:%u", (unsigned int)vm->ractor.sched.barrier_serial - 1); - VM_ASSERT(vm->ractor.sched.barrier_is_waiting); - VM_ASSERT(vm->ractor.sched.barrier_ractor); - VM_ASSERT(vm->ractor.sched.barrier_lock_rec > 0); - - RUBY_ATOMIC_SET(vm->ractor.sched.barrier_is_waiting, 0); - vm->ractor.sched.barrier_ractor = NULL; - vm->ractor.sched.barrier_lock_rec = 0; - ractor_sched_unlock(vm, cr); -} - -static void -ractor_sched_barrier_join_signal_locked(rb_vm_t *vm) -{ - if (ractor_sched_barrier_completed_p(vm)) { - rb_native_cond_signal(&vm->ractor.sched.barrier_complete_cond); - } -} - -static void -ractor_sched_barrier_join_wait_locked(rb_vm_t *vm, rb_thread_t *th) -{ - VM_ASSERT(vm->ractor.sched.barrier_is_waiting); - - unsigned int barrier_serial = vm->ractor.sched.barrier_serial; - - while (vm->ractor.sched.barrier_serial == barrier_serial) { - RUBY_DEBUG_LOG("sleep serial:%u", barrier_serial); - RB_VM_SAVE_MACHINE_CONTEXT(th); - - rb_ractor_t *cr = th->ractor; - ractor_sched_set_unlocked(vm, cr); - rb_native_cond_wait(&vm->ractor.sched.barrier_release_cond, &vm->ractor.sched.lock); - ractor_sched_set_locked(vm, cr); - - RUBY_DEBUG_LOG("wakeup serial:%u", barrier_serial); - } -} - -void -rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr) -{ - VM_ASSERT(cr->threads.sched.running != NULL); // running ractor - VM_ASSERT(cr == GET_RACTOR()); - VM_ASSERT(vm->ractor.sync.lock_owner == NULL); // VM is locked, but owner == NULL - VM_ASSERT(vm->ractor.sched.barrier_is_waiting); // VM needs barrier sync - -#if USE_RUBY_DEBUG_LOG || VM_CHECK_MODE > 0 - unsigned int barrier_serial = vm->ractor.sched.barrier_serial; -#endif - - RUBY_DEBUG_LOG("join"); - - rb_native_mutex_unlock(&vm->ractor.sync.lock); - { - VM_ASSERT(vm->ractor.sched.barrier_is_waiting); // VM needs barrier sync - VM_ASSERT(vm->ractor.sched.barrier_serial == barrier_serial); - - ractor_sched_lock(vm, cr); - { - // running_cnt - /* Every joiner is a member of the running set: a dying thread - * leaves the living set before handing over its scheduler slot. */ - vm->ractor.sched.barrier_joined_cnt++; - RUBY_DEBUG_LOG("waiting_cnt:%u serial:%u", vm->ractor.sched.barrier_joined_cnt, barrier_serial); - - ractor_sched_barrier_join_signal_locked(vm); - ractor_sched_barrier_join_wait_locked(vm, cr->threads.sched.running); - } - ractor_sched_unlock(vm, cr); - } - - rb_native_mutex_lock(&vm->ractor.sync.lock); - // VM locked here -} - -// Called when the ractor holding this sched is freed. A drained sched can -// still be on timeslice.scheds (pruning is lazy); an unlisted node is -// self-linked (fork re-inits them all), making this del a no-op. -void -rb_thread_sched_destroy(struct rb_thread_sched *sched) -{ - rb_vm_t *vm = GET_VM(); - - rb_native_mutex_lock(&vm->ractor.sched.timeslice.lock); - { - ccan_list_del_init(&sched->timeslice_node); - } - rb_native_mutex_unlock(&vm->ractor.sched.timeslice.lock); -} - -#ifdef RB_THREAD_T_HAS_NATIVE_ID -static int -get_native_thread_id(void) -{ -#ifdef __linux__ - return (int)syscall(SYS_gettid); -#elif defined(__FreeBSD__) - return pthread_getthreadid_np(); -#endif -} -#endif - -#if defined(HAVE_WORKING_FORK) -static void rb_internal_thread_event_hooks_rw_lock_atfork(void); - -static void -thread_sched_atfork(struct rb_thread_sched *sched) -{ - current_fork_gen++; - rb_thread_sched_init(sched, true); - rb_thread_t *th = GET_THREAD(); - rb_vm_t *vm = GET_VM(); - - if (th_has_dedicated_nt(th)) { - vm->ractor.sched.snt_cnt = 0; -#if USE_RUBY_DEBUG_LOG - vm->ractor.sched.dnt_cnt = 1; -#endif - } - else { - vm->ractor.sched.snt_cnt = 1; -#if USE_RUBY_DEBUG_LOG - vm->ractor.sched.dnt_cnt = 0; -#endif - } - - rb_native_mutex_initialize(&vm->ractor.sched.lock); -#if VM_CHECK_MODE > 0 - vm->ractor.sched.lock_owner = NULL; - vm->ractor.sched.locked = false; -#endif - - // rb_native_cond_destroy(&vm->ractor.sched.cond); - rb_native_cond_initialize(&vm->ractor.sched.cond); - rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond); - rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond); - - ccan_list_head_init(&vm->ractor.sched.grq); - vm->ractor.sched.grq_cnt = 0; // the list was just emptied; reset the count with it - // A fork during a VM barrier leaves the child with barrier state that can - // never complete (the other ractors are gone); reset it like the rest. - vm->ractor.sched.barrier_is_waiting = 0; // single-threaded child - vm->ractor.sched.barrier_joined_cnt = 0; - vm->ractor.sched.barrier_ractor = NULL; - vm->ractor.sched.barrier_lock_rec = 0; - // Threads that were winding down in the parent do not exist in the child; - // without this reset the child's ruby_vm_destruct would wait for their - // reclaim (which never comes) forever. - vm->ractor.sched.winding_cnt = 0; - rb_native_mutex_initialize(&vm->ractor.sched.ntlist.lock); - ccan_list_head_init(&vm->ractor.sched.ntlist.running_dnts); - ccan_list_head_init(&vm->ractor.sched.ntlist.snts); // those nts are gone - rb_native_mutex_initialize(&vm->ractor.sched.timeslice.lock); - ccan_list_head_init(&vm->ractor.sched.timeslice.scheds); - rb_native_mutex_initialize(&th->nt->running_th_lock); // a scan could hold it at fork - // Fork can copy nodes linked (or torn mid-link); re-init every sched's - // node so rb_thread_sched_destroy's del_init stays a no-op for them. - rb_ractor_t *r; - ccan_list_for_each(&vm->ractor.set, r, vmlr_node) { - ccan_list_node_init(&r->threads.sched.timeslice_node); - } - ccan_list_for_each(&vm->ractor.terminated_set, r, vmlr_node) { - ccan_list_node_init(&r->threads.sched.timeslice_node); - } - // th re-records itself below; the parent's record did not survive the lists - if (th->nt && th->nt->dedicated == 0) { - // surviving on an snt: put that nt back on the (just emptied) snts - // list, or the scans could not see this thread's record - ccan_list_add(&vm->ractor.sched.ntlist.snts, &th->nt->snts_node); - } - -#if USE_MN_THREADS - nt_machine_stack_atfork(); -#endif - rb_internal_thread_event_hooks_rw_lock_atfork(); - - VM_ASSERT(sched->is_running); - - if (sched->running != th) { - thread_sched_to_running(sched, th); - } - else { - thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL); - } - -#ifdef RB_THREAD_T_HAS_NATIVE_ID - if (th->nt) { - th->nt->tid = get_native_thread_id(); - } -#endif -} - -#endif - -#ifdef RB_THREAD_LOCAL_SPECIFIER -static RB_THREAD_LOCAL_SPECIFIER rb_thread_t *ruby_native_thread; -#else -static pthread_key_t ruby_native_thread_key; -#endif - -static void -null_func(int i) -{ - /* null */ - // This function can be called from signal handler - // RUBY_DEBUG_LOG("i:%d", i); -} - -rb_thread_t * -ruby_thread_from_native(void) -{ -#ifdef RB_THREAD_LOCAL_SPECIFIER - return ruby_native_thread; -#else - return pthread_getspecific(ruby_native_thread_key); -#endif -} - -int -ruby_thread_set_native(rb_thread_t *th) -{ - if (th) { -#ifdef USE_UBF_LIST - ccan_list_node_init(&th->sched.node.ubf); -#endif - } - - // setup TLS - - if (th && th->ec) { - rb_ractor_set_current_ec(th->ractor, th->ec); - } -#ifdef RB_THREAD_LOCAL_SPECIFIER - ruby_native_thread = th; - return 1; -#else - return pthread_setspecific(ruby_native_thread_key, th) == 0; -#endif -} - -static void native_thread_setup(struct rb_native_thread *nt); -static void native_thread_setup_on_thread(struct rb_native_thread *nt); - -// Internal cache of page size: -static size_t RB_THREAD_PAGE_SIZE; - -void -Init_native_thread(rb_thread_t *main_th) -{ - // Get the system page size for later use in stack allocation and stack overflow checks: - RB_THREAD_PAGE_SIZE = sysconf(_SC_PAGESIZE); - -#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) - if (condattr_monotonic) { - int r = pthread_condattr_init(condattr_monotonic); - if (r == 0) { - r = pthread_condattr_setclock(condattr_monotonic, CLOCK_MONOTONIC); - } - if (r) condattr_monotonic = NULL; - } -#endif - -#ifndef RB_THREAD_LOCAL_SPECIFIER - if (pthread_key_create(&ruby_native_thread_key, 0) == EAGAIN) { - rb_bug("pthread_key_create failed (ruby_native_thread_key)"); - } - if (pthread_key_create(&ruby_current_ec_key, 0) == EAGAIN) { - rb_bug("pthread_key_create failed (ruby_current_ec_key)"); - } -#endif - ruby_posix_signal(SIGVTALRM, null_func); - - // setup vm - rb_vm_t *vm = main_th->vm; - rb_native_mutex_initialize(&vm->ractor.sched.lock); - rb_native_cond_initialize(&vm->ractor.sched.cond); - rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond); - rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond); - - ccan_list_head_init(&vm->ractor.sched.grq); - rb_native_mutex_initialize(&vm->ractor.sched.ntlist.lock); - ccan_list_head_init(&vm->ractor.sched.ntlist.running_dnts); - ccan_list_head_init(&vm->ractor.sched.ntlist.snts); - rb_native_mutex_initialize(&vm->ractor.sched.timeslice.lock); - ccan_list_head_init(&vm->ractor.sched.timeslice.scheds); + // setup vm + rb_vm_t *vm = main_th->vm; + thread_sched_init_vm(vm); // setup main thread main_th->nt->thread_id = pthread_self(); @@ -2150,135 +454,23 @@ Init_native_thread(rb_thread_t *main_th) native_thread_setup(main_th->nt); native_thread_setup_on_thread(main_th->nt); - TH_SCHED(main_th)->running = main_th; - main_th->has_dedicated_nt = 1; - - // setup main NT (before the record below: its kind decides where it goes) - main_th->nt->dedicated = 1; - main_th->nt->running_thread = main_th; - main_th->nt->vm = vm; - - thread_sched_setup_running_threads(TH_SCHED(main_th), main_th->ractor, vm, main_th, NULL); - - // setup mn -#if USE_RUBY_DEBUG_LOG - vm->ractor.sched.dnt_cnt = 1; -#endif -} - -extern int ruby_mn_threads_enabled; - -void -ruby_mn_threads_params(void) -{ - rb_vm_t *vm = GET_VM(); - rb_ractor_t *main_ractor = GET_RACTOR(); - - const char *mn_threads_cstr = getenv("RUBY_MN_THREADS"); - bool enable_mn_threads = false; - - if (USE_MN_THREADS && mn_threads_cstr && (enable_mn_threads = atoi(mn_threads_cstr) > 0)) { - // enabled - ruby_mn_threads_enabled = 1; - } - main_ractor->threads.sched.enable_mn_threads = enable_mn_threads; - - const char *max_cpu_cstr = getenv("RUBY_MAX_CPU"); -#if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN) - long nprocessors = sysconf(_SC_NPROCESSORS_ONLN); - const int default_max_cpu = (nprocessors > 0) ? (int)nprocessors : 8; -#else - const int default_max_cpu = 8; -#endif - int max_cpu = default_max_cpu; - - if (USE_MN_THREADS && max_cpu_cstr) { - int given_max_cpu = atoi(max_cpu_cstr); - if (given_max_cpu > 0) { - max_cpu = given_max_cpu; - } - } - - vm->ractor.sched.max_cpu = max_cpu; -} - -static void -native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt) -{ - RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated + 1); - - if (nt->dedicated == 0) { - // Lock-free; pairs with ractor_sched_enq (enq: grq_cnt up then read - // snt_cnt / here: snt_cnt down then read grq_cnt) against lost wakeups. - if (RUBY_ATOMIC_FETCH_SUB(vm->ractor.sched.snt_cnt, 1) == 1) { - // the last snt went dedicated; pending entries need the timer thread - ractor_sched_lock(vm, cr); - { - if (vm->ractor.sched.grq_cnt > 0) { - timer_thread_wakeup_locked(vm); - } - } - ractor_sched_unlock(vm, cr); - } -#if USE_RUBY_DEBUG_LOG - vm->ractor.sched.dnt_cnt++; -#endif - } - - nt->dedicated++; -} + TH_SCHED(main_th)->running = main_th; + main_th->has_dedicated_nt = 1; -static void -native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt) -{ - RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated - 1); - VM_ASSERT(nt->dedicated > 0); - nt->dedicated--; - - if (nt->dedicated == 0) { - // Rejoin under the max_cpu cap; with no room this nt retires and - // belongs to neither count until it ends. - while (1) { - rb_atomic_t snt = RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt); - if (snt < vm->ractor.sched.max_cpu || (int)snt <= MINIMUM_SNT) { - if (RUBY_ATOMIC_CAS(vm->ractor.sched.snt_cnt, snt, snt + 1) == snt) break; - } - else { - nt->retiring = true; - break; - } - } -#if USE_RUBY_DEBUG_LOG - vm->ractor.sched.dnt_cnt--; -#endif - } -} + // setup main NT (before the record below: its kind decides where it goes) + main_th->nt->dedicated = 1; + main_th->nt->running_thread = main_th; + main_th->nt->vm = vm; -static void -native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th) -{ + thread_sched_setup_running_threads(TH_SCHED(main_th), main_th->ractor, vm, main_th, NULL); + + // setup mn #if USE_RUBY_DEBUG_LOG - if (nt) { - if (th->nt) { - RUBY_DEBUG_LOG("th:%d nt:%d->%d", (int)th->serial, (int)th->nt->serial, (int)nt->serial); - } - else { - RUBY_DEBUG_LOG("th:%d nt:NULL->%d", (int)th->serial, (int)nt->serial); - } - } - else { - if (th->nt) { - RUBY_DEBUG_LOG("th:%d nt:%d->NULL", (int)th->serial, (int)th->nt->serial); - } - else { - RUBY_DEBUG_LOG("th:%d nt:NULL->NULL", (int)th->serial); - } - } + vm->ractor.sched.dnt_cnt = 1; #endif - - th->nt = nt; } + static void native_thread_destroy_atfork(struct rb_native_thread *nt) { @@ -2571,8 +763,6 @@ struct nt_param { struct rb_native_thread *nt; }; -static void * -nt_start(void *ptr); static int native_thread_create0(struct rb_native_thread *nt) @@ -2645,264 +835,8 @@ native_thread_alloc(void) return nt; } -static int -native_thread_create_dedicated(rb_thread_t *th) -{ - th->nt = native_thread_alloc(); - th->nt->vm = th->vm; - th->nt->running_thread = th; - th->nt->dedicated = 1; - - // vm stack - size_t vm_stack_word_size = th->vm->default_params.thread_vm_stack_size / sizeof(VALUE); - void *vm_stack = ruby_xmalloc(vm_stack_word_size * sizeof(VALUE)); - th->sched.malloc_stack = true; - rb_ec_initialize_vm_stack(th->ec, vm_stack, vm_stack_word_size); - th->sched.context_stack = vm_stack; - th->sched.context_stack_size = vm_stack_word_size; - - int err = native_thread_create0(th->nt); - if (!err) { - // setup - thread_sched_to_ready(TH_SCHED(th), th); - } - return err; -} - -static void -call_thread_start_func_2(rb_thread_t *th) -{ - /* Capture the address of a local in this stack frame to mark the beginning of the - machine stack for this thread. This is required even if we can tell the real - stack beginning from the pthread API in native_thread_init_stack, because - glibc stores some of its own data on the stack before calling into user code - on a new thread, and replacing that data on fiber-switch would break it (see - bug #13887) */ - VALUE stack_start = 0; - VALUE *stack_start_addr = asan_get_real_stack_addr(&stack_start); - - native_thread_init_stack(th, stack_start_addr); - thread_start_func_2(th, th->ec->machine.stack_start); -} - -static void * -nt_start(void *ptr) -{ - struct rb_native_thread *nt = (struct rb_native_thread *)ptr; - rb_vm_t *vm = nt->vm; - - native_thread_setup_on_thread(nt); - - // init tid -#ifdef RB_THREAD_T_HAS_NATIVE_ID - nt->tid = get_native_thread_id(); -#endif - -#if USE_RUBY_DEBUG_LOG && defined(RUBY_NT_SERIAL) - ruby_nt_serial = nt->serial; -#endif - - RUBY_DEBUG_LOG("nt:%u", nt->serial); - - bool in_snts = false; - - if (!nt->dedicated) { - coroutine_initialize_main(nt->nt_context); - - // join the snt list that the barrier/timeslice scans walk - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); - { - ccan_list_add(&vm->ractor.sched.ntlist.snts, &nt->snts_node); - } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); - in_snts = true; - } - - bool retired = false; - - while (1) { - if (nt->dedicated) { - // wait running turn - rb_thread_t *th = nt->running_thread; - struct rb_thread_sched *sched = TH_SCHED(th); - - RUBY_DEBUG_LOG("on dedicated th:%u", rb_th_serial(th)); - ruby_thread_set_native(th); - - thread_sched_lock(sched, th); - { - if (sched->running == th) { - thread_sched_add_running_thread(sched, th); - } - thread_sched_wait_running_turn(sched, th, false, NULL); - } - thread_sched_unlock(sched, th); - - // start threads - call_thread_start_func_2(th); - break; // TODO: allow to change to the SNT - } - else { - RUBY_DEBUG_LOG("check next"); - if (nt->retiring) { // came back with no room in the shared pool - retired = true; - break; - } - - rb_ractor_t *r = ractor_sched_deq(vm, NULL); - - if (r) { - struct rb_thread_sched *sched = &r->threads.sched; - - bool locked = true; - - thread_sched_lock(sched, NULL); - { - rb_thread_t *next_th = sched->running; - - if (next_th && next_th->nt == NULL) { - RUBY_DEBUG_LOG("nt:%d next_th:%d", (int)nt->serial, (int)next_th->serial); -#if USE_MN_THREADS - thread_sched_switch0(nt->nt_context, next_th, nt, false); - - // If a coroutine terminated during the transfer, co_start - // recorded it in nt->dead_co (switch0's return value is - // backend-dependent, unusable; see thread_pthread.h). - struct coroutine_context *dead_co = nt->dead_co; - nt->dead_co = NULL; - if (thread_sched_reclaim(dead_co)) { - // it already released the sched lock before its - // transfer (its Ractor may be gone): leave sched be. - locked = false; - } -#else - thread_sched_switch0(nt->nt_context, next_th, nt, false); -#endif - } - else { - RUBY_DEBUG_LOG("no schedulable threads -- next_th:%p", next_th); - } - } - if (locked) { - thread_sched_unlock(sched, NULL); - } - } - else { - // ractor_sched_deq retired this nt. - retired = true; - break; - } - - if (nt->dedicated) { - // SNT becomes DNT while running - break; - } - } - } - - if (in_snts) { - // Leaving the shared loop: every path back here deregistered first - // (park and death both precede the transfer), so only the snts entry - // is left to remove. - VM_ASSERT(nt->running_th == NULL); - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); - { - ccan_list_del_init(&nt->snts_node); - } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); - } - - if (retired) { - // The counts dropped this nt already; nothing can reference it now. - RUBY_DEBUG_LOG("retired nt:%u", nt->serial); - native_thread_destroy_self(nt); - } - - return NULL; -} - -static int native_thread_create_shared(rb_thread_t *th); - -#if USE_MN_THREADS -static void nt_free_stack(void *mstack); - - -// Reclaim the context a coroutine thread recorded in nt->dead_co before its -// final transfer (co_start's epilogue). Our running here proves that transfer's -// register save into the block completed. Returns true when a thread did -// terminate -- it RELEASED the sched lock before transferring; NULL/false means -// a live yield, where the loop still owns the lock. -static bool -thread_sched_reclaim(struct coroutine_context *dead_co) -{ - struct rb_thread_context *tctx = (struct rb_thread_context *)dead_co; - - if (tctx != NULL && tctx->dead) { - nt_free_stack(tctx->stack); - SIZED_FREE(tctx); - // pairs with the increment at the top of coroutine_thread_terminated: - // a waiting VM destruct may proceed once this reclaim is done - VM_ASSERT(RUBY_ATOMIC_LOAD(GET_VM()->ractor.sched.winding_cnt) > 0); - RUBY_ATOMIC_DEC(GET_VM()->ractor.sched.winding_cnt); - return true; - } - return false; -} -#endif - -void -rb_thread_wake_fence(rb_thread_t *th) -{ - timer_thread_wake_fence(th); -} - -void -rb_threadptr_sched_free(rb_thread_t *th) -{ - timer_thread_wake_fence(th); -#if USE_MN_THREADS - if (th->sched.malloc_stack) { - // has dedicated - SIZED_FREE_N((VALUE *)th->sched.context_stack, th->sched.context_stack_size); - native_thread_destroy(th->nt); - } - else if (th->sched.context != NULL) { - // a coroutine thread that never reached its epilogue (never started); - // a terminated one is reclaimed by whoever resumed from its final - // transfer (thread_sched_reclaim), and cleared this pointer. - struct rb_thread_context *tctx = (struct rb_thread_context *)th->sched.context; - nt_free_stack(tctx->stack); - SIZED_FREE(tctx); - th->sched.context = NULL; - // TODO: how to free nt and nt->altstack? - } -#else - SIZED_FREE_N((VALUE *)th->sched.context_stack, th->sched.context_stack_size); - native_thread_destroy(th->nt); -#endif - - th->nt = NULL; -} - - -static int -native_thread_create(rb_thread_t *th) -{ - VM_ASSERT(th->nt == 0); - RUBY_DEBUG_LOG("th:%d has_dnt:%d", th->serial, th->has_dedicated_nt); - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_STARTED, th); - if (!th->ractor->threads.sched.enable_mn_threads) { - th->has_dedicated_nt = 1; - } - if (th->has_dedicated_nt) { - return native_thread_create_dedicated(th); - } - else { - return native_thread_create_shared(th); - } -} #if USE_NATIVE_THREAD_PRIORITY @@ -2940,118 +874,27 @@ native_fd_select(int n, rb_fdset_t *readfds, rb_fdset_t *writefds, rb_fdset_t *e return rb_fd_select(n, readfds, writefds, exceptfds, timeout); } -#ifdef USE_UBF_LIST -static CCAN_LIST_HEAD(ubf_list_head); -static rb_nativethread_lock_t ubf_list_lock = RB_NATIVETHREAD_LOCK_INIT; - -static void -ubf_list_atfork(void) -{ - ccan_list_head_init(&ubf_list_head); - rb_native_mutex_initialize(&ubf_list_lock); -} - -RBIMPL_ATTR_MAYBE_UNUSED() -static bool -ubf_list_contain_p(rb_thread_t *th) -{ - rb_thread_t *list_th; - ccan_list_for_each(&ubf_list_head, list_th, sched.node.ubf) { - if (list_th == th) return true; - } - return false; -} - -/* The thread 'th' is registered to be trying unblock. */ -static void -register_ubf_list(rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); - struct ccan_list_node *node = &th->sched.node.ubf; - - VM_ASSERT(th->unblock.func != NULL); - - rb_native_mutex_lock(&ubf_list_lock); - { - // check not connected yet - if (ccan_list_empty((struct ccan_list_head*)node)) { - VM_ASSERT(!ubf_list_contain_p(th)); - ccan_list_add(&ubf_list_head, node); - } - } - rb_native_mutex_unlock(&ubf_list_lock); - - timer_thread_wakeup(); -} - -/* The thread 'th' is unblocked. It no longer need to be registered. */ -static void -unregister_ubf_list(rb_thread_t *th) -{ - RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); - struct ccan_list_node *node = &th->sched.node.ubf; - - /* we can't allow re-entry into ubf_list_head */ - VM_ASSERT(th->unblock.func == NULL); - - if (!ccan_list_empty((struct ccan_list_head*)node)) { - rb_native_mutex_lock(&ubf_list_lock); - { - VM_ASSERT(ubf_list_contain_p(th)); - ccan_list_del_init(node); - } - rb_native_mutex_unlock(&ubf_list_lock); - } -} - /* - * send a signal to intent that a target thread return from blocking syscall. - * Maybe any signal is ok, but we chose SIGVTALRM. + * Send a signal to make the target thread return from a blocking syscall. + * Maybe any signal is ok, but we chose SIGVTALRM (see null_func). */ static void -ubf_wakeup_thread(rb_thread_t *th) +native_thread_interrupt(rb_thread_t *th) { - RUBY_DEBUG_LOG("th:%u thread_id:%p", rb_th_serial(th), (void *)th->nt->thread_id); - pthread_kill(th->nt->thread_id, SIGVTALRM); } -static void -ubf_select(void *ptr) -{ - rb_thread_t *th = (rb_thread_t *)ptr; - RUBY_DEBUG_LOG("wakeup th:%u", rb_th_serial(th)); - ubf_wakeup_thread(th); - register_ubf_list(th); -} - -static bool -ubf_threads_empty(void) -{ - return ccan_list_empty(&ubf_list_head) != 0; -} - -static void -ubf_wakeup_all_threads(void) +static int +native_thread_default_max_cpu(void) { - rb_thread_t *th; - rb_native_mutex_lock(&ubf_list_lock); - { - ccan_list_for_each(&ubf_list_head, th, sched.node.ubf) { - ubf_wakeup_thread(th); - } - } - rb_native_mutex_unlock(&ubf_list_lock); +#if defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN) + long nprocessors = sysconf(_SC_NPROCESSORS_ONLN); + return (nprocessors > 0) ? (int)nprocessors : 8; +#else + return 8; +#endif } -#else /* USE_UBF_LIST */ -#define register_ubf_list(th) (void)(th) -#define unregister_ubf_list(th) (void)(th) -#define ubf_select 0 -static void ubf_wakeup_all_threads(void) { return; } -static bool ubf_threads_empty(void) { return true; } -#define ubf_list_atfork() do {} while (0) -#endif /* USE_UBF_LIST */ #define TT_DEBUG 0 #define WRITE_CONST(fd, str) (void)(write((fd),(str),sizeof(str)-1)<0) @@ -3302,7 +1145,7 @@ static struct { #if USE_MN_THREADS /* Timed waiters, bucketed by deadline into a hierarchical timer wheel; * untimed (fd-only) waiters keep a plain list. Both under waiting_lock. - * The wheel operations all live in thread_pthread_mn.c (timer_wheel_*). */ + * The wheel operations all live in thread_sched_mn.c (timer_wheel_*). */ #define TIMER_WHEEL_LEVELS 4 #define TIMER_WHEEL_SLOT_BITS 6 #define TIMER_WHEEL_SLOTS (1 << TIMER_WHEEL_SLOT_BITS) @@ -3343,102 +1186,8 @@ static void timer_thread_check_timeslice(rb_vm_t *vm); static bool timeslice_scan(rb_vm_t *vm, bool interrupt); static int timer_thread_set_timeout(rb_vm_t *vm); -#include "thread_pthread_mn.c" - -static int -timer_thread_set_timeout(rb_vm_t *vm) -{ -#if 0 - return 10; // ms -#else - int timeout = -1; - - ractor_sched_lock(vm, NULL); - { - if ( timeslice_scan(vm, false) // (1-1) Provide time slice for active NTs - || !ubf_threads_empty() // (1-3) Periodic UBF - || vm->ractor.sched.grq_cnt > 0 // (1-4) Lazy GRQ deq start - ) { - - RUBY_DEBUG_LOG("ubf:%d grq:%d", - !ubf_threads_empty(), - (vm->ractor.sched.grq_cnt > 0)); - - timeout = 10; // ms - vm->ractor.sched.timeslice_wait_inf = false; - } - else { - vm->ractor.sched.timeslice_wait_inf = true; - } - } - ractor_sched_unlock(vm, NULL); - - timeout = timer_wheel_timeout(timeout); - - RUBY_DEBUG_LOG("timeout:%d inf:%d", timeout, (int)vm->ractor.sched.timeslice_wait_inf); - - // fprintf(stderr, "timeout:%d\n", timeout); - return timeout; -#endif -} - -static void -timer_thread_check_signal(rb_vm_t *vm) -{ - // ruby_sigchld_handler(vm); TODO - - int signum = rb_signal_buff_size(); - if (UNLIKELY(signum > 0) && vm->ractor.main_thread) { - RUBY_DEBUG_LOG("signum:%d", signum); - threadptr_trap_interrupt(vm->ractor.main_thread); - } -} - -// Tick (with `interrupt`) each listed sched's running thread and prune scheds -// whose readyq drained; returns whether any sched still needs ticks. -static bool -timeslice_scan(rb_vm_t *vm, bool interrupt) -{ - bool found = false; - struct rb_thread_sched *sched, *next; - - rb_native_mutex_lock(&vm->ractor.sched.timeslice.lock); - { - ccan_list_for_each_safe(&vm->ractor.sched.timeslice.scheds, sched, next, timeslice_node) { - // trylock: timeslice_sched_link nests sched.lock -> timeslice.lock, - // this scan holds the locks the other way around - if (rb_native_mutex_trylock(&sched->lock_) == 0) { - if (ccan_list_empty(&sched->readyq)) { - ccan_list_del_init(&sched->timeslice_node); // a later enq relinks it - } - else if (sched->is_running) { - VM_ASSERT(sched->running != NULL); - found = true; - if (interrupt) { - RUBY_DEBUG_LOG("timeslice th:%u", rb_th_serial(sched->running)); - RUBY_VM_SET_TIMER_INTERRUPT(sched->running->ec); - } - } - // else: waiters behind a blocked runner need no ticks; the - // add path wakes the timer when the sched runs again - rb_native_mutex_unlock(&sched->lock_); - } - else { - found = true; // busy switching; tick it on the next round - } - } - } - rb_native_mutex_unlock(&vm->ractor.sched.timeslice.lock); - - return found; -} +#include "thread_sched_mn.c" -static void -timer_thread_check_timeslice(rb_vm_t *vm) -{ - // TODO: check time - timeslice_scan(vm, true); -} void rb_assert_sig(void) @@ -3453,28 +1202,6 @@ rb_assert_sig(void) } } -static void * -timer_thread_func(void *ptr) -{ - rb_vm_t *vm = (rb_vm_t *)ptr; -#if defined(RUBY_NT_SERIAL) - ruby_nt_serial = (rb_atomic_t)-1; -#endif - - RUBY_DEBUG_LOG("started%s", ""); - - while (RUBY_ATOMIC_LOAD(system_working)) { - timer_thread_check_signal(vm); - timer_thread_check_timeout(vm); - ubf_wakeup_all_threads(); - - RUBY_DEBUG_LOG("system_working:%d", RUBY_ATOMIC_LOAD(system_working)); - timer_thread_polling(vm); - } - - RUBY_DEBUG_LOG("terminated"); - return NULL; -} /* only use signal-safe system calls here */ static void @@ -3517,34 +1244,6 @@ timer_thread_wakeup_force(void) signal_communication_pipe(timer_th.comm_fds[1]); } -static void -timer_thread_wakeup_locked(rb_vm_t *vm) -{ - // should be locked before. - ASSERT_ractor_sched_locked(vm, NULL); - - if (timer_th.created_fork_gen == current_fork_gen) { - if (vm->ractor.sched.timeslice_wait_inf) { - RUBY_DEBUG_LOG("wakeup with fd:%d", timer_th.comm_fds[1]); - timer_thread_wakeup_force(); - } - else { - RUBY_DEBUG_LOG("will be wakeup..."); - } - } -} - -static void -timer_thread_wakeup(void) -{ - rb_vm_t *vm = GET_VM(); - - ractor_sched_lock(vm, NULL); - { - timer_thread_wakeup_locked(vm); - } - ractor_sched_unlock(vm, NULL); -} static void rb_thread_create_timer_thread(void) @@ -3748,38 +1447,6 @@ ruby_ppoll(struct pollfd *fds, nfds_t nfds, # define ppoll(fds,nfds,ts,sigmask) ruby_ppoll((fds),(nfds),(ts),(sigmask)) #endif -static void -native_sleep(rb_thread_t *th, rb_hrtime_t *rel) -{ - struct rb_thread_sched *sched = TH_SCHED(th); - - RUBY_DEBUG_LOG("rel:%d", rel ? (int)*rel : 0); - - if (rel && !th_has_dedicated_nt(th)) { - // an M:N thread has no condvar of its own: the timer thread wakes it - thread_sched_wait_events(sched, th, -1, thread_sched_waiting_timeout, rel); - } - else if (rel) { - /* Solaris cond_timedwait() returns EINVAL if an argument is greater than - * current_time + 100,000,000. So cut up to 100,000,000. This is - * considered as a kind of spurious wakeup. The caller to native_sleep - * should care about spurious wakeup. - * - * See also [Bug #1341] [ruby-core:29702] - * http://download.oracle.com/docs/cd/E19683-01/816-0216/6m6ngupgv/index.html - */ - const rb_hrtime_t max = (rb_hrtime_t)100000000 * RB_HRTIME_PER_SEC; - if (*rel > max) *rel = max; - - rb_hrtime_t end = rb_hrtime_add(rb_hrtime_now(), *rel); - thread_sched_to_waiting_until_wakeup(sched, th, &end); - } - else { - thread_sched_to_waiting_until_wakeup(sched, th, NULL); - } - - RUBY_DEBUG_LOG("wakeup"); -} // fork read-write lock (only for pthread) static pthread_rwlock_t rb_thread_fork_rw_lock = PTHREAD_RWLOCK_INITIALIZER; @@ -3959,24 +1626,4 @@ rb_thread_execute_hooks(rb_event_flag_t event, rb_thread_t *th) } } -// return true if the current thread acquires DNT. -// return false if the current thread already acquires DNT. -bool -rb_thread_lock_native_thread(void) -{ - rb_thread_t *th = GET_THREAD(); - bool is_snt = th->nt->dedicated == 0; - native_thread_dedicated_inc(th->vm, th->ractor, th->nt); - - return is_snt; -} - -void -rb_thread_malloc_stack_set(rb_thread_t *th, void *stack, size_t stack_size) -{ - th->sched.malloc_stack = true; - th->sched.context_stack = stack; - th->sched.context_stack_size = stack_size; -} - #endif /* THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION */ diff --git a/thread_pthread.h b/thread_pthread.h index 040ae55be4697e..b8979be530cfbb 100644 --- a/thread_pthread.h +++ b/thread_pthread.h @@ -8,6 +8,9 @@ Copyright (C) 2004-2007 Koichi Sasada + This platform runs the common scheduler; see thread_sched.h for its data + structures and thread_sched.c for the implementation. + **********************************************************************/ #ifdef HAVE_PTHREAD_NP_H @@ -22,178 +25,7 @@ # define RB_THREAD_CURRENT_EC_NOINLINE #endif -// How a thread_sched_wait_events() wait ended. "unavailable" (could not be -// registered) is not "the event fired": the caller must fall back, not proceed. -enum thread_sched_wait_result { - thread_sched_wait_event, // an event the caller asked for fired - thread_sched_wait_timeout, // the timeout expired before any event - thread_sched_wait_unavailable, // not registered; the caller must fall back -}; - -// this data should be protected by timer_th.waiting_lock -struct rb_thread_sched_waiting { - enum thread_sched_waiting_flag { - thread_sched_waiting_none = 0x00, - thread_sched_waiting_timeout = 0x01, - thread_sched_waiting_io_read = 0x02, - thread_sched_waiting_io_write = 0x08, - thread_sched_waiting_io_force = 0x40, // ignore readable - } flags; - - struct { - // should be compat with hrtime.h -#ifdef MY_RUBY_BUILD_MAY_TIME_TRAVEL - int128_t timeout; -#else - uint64_t timeout; -#endif - uint32_t event_serial; - int fd; // -1 for timeout only - int result; - } data; - - // connected to a timer_th wheel slot (timed) or timer_th.waiting_untimed - struct ccan_list_node node; - - /* which wheel slot `node` is on; meaningful only while flags has - * thread_sched_waiting_timeout */ - uint8_t wheel_lvl; - uint8_t wheel_slot; - - // connected to rb_fd_waiters.waiters of data.fd - struct ccan_list_node fd_node; -}; - -// One entry per fd with waiters; fds stay dense, so a table indexed by fd fits. -// Entries live in fixed chunks: growing must not move a live list head. -struct rb_fd_waiters { - struct ccan_list_head waiters; // rb_thread_sched_waiting.fd_node - - // The io flags currently armed in epoll/kqueue for this fd: the union of - // what its waiters asked for. - uint32_t armed_flags; - - // Bumped on full disarm. Events carry the generation they were armed with, - // so one queued before the fd was disarmed (and reused) is recognised. - uint32_t generation; -}; - -// per-Thread scheduler helper data -struct rb_thread_sched_item { - struct { - struct ccan_list_node ubf; - - // connected to ractor->threads.sched.reqdyq - // locked by ractor->threads.sched.lock - struct ccan_list_node readyq; - // Indicates whether thread is on the readyq. - // There is no clear relationship between this and th->status. - bool is_ready; - - } node; - - struct rb_thread_sched_waiting waiting_reason; - uint32_t event_serial; - - // wakes pending on this thread (timer thread or an fd shard claim); - // under timer_th.wake_pending_lock - uint32_t wake_pending_cnt; - - // parked on its own condvar with a deadline; under the sched lock (see - // ubf_waiting). Always false for an M:N thread: its deadline lives on the - // timer wheel, and its early wake comes from the timer thread instead. - bool waiting_timed; - - bool malloc_stack; - void *context_stack; - size_t context_stack_size; - struct coroutine_context *context; -}; - -struct rb_native_thread { - rb_atomic_t serial; - struct rb_vm_struct *vm; - - rb_nativethread_id_t thread_id; - -#ifdef RB_THREAD_T_HAS_NATIVE_ID - int tid; -#endif - - struct rb_thread_struct *running_thread; - - // The running thread on this shared nt, for the barrier/timeslice scans. - // While a scan holds running_th_lock the thread cannot finish parking. - rb_nativethread_lock_t running_th_lock; - struct rb_thread_struct *running_th; - struct ccan_list_node snts_node; // in vm->ractor.sched.ntlist.snts - // in vm->ractor.sched.ntlist.running_dnts while running_thread runs - struct ccan_list_node running_dnts_node; - // barrier_serial stamped by the barrier's counting walk; this nt's - // deregistration during that barrier decrements the snapshot count - uint32_t barrier_counted_serial; - - // to control native thread; use sched->lock - rb_nativethread_cond_t readyq; - -#ifdef USE_SIGALTSTACK - void *altstack; -#endif - - struct coroutine_context *nt_context; - int dedicated; - - // set when this thread came back from a blocking region with no room left - // in the shared pool; it ends when it next asks for work - bool retiring; - - // A terminating coroutine records its context here before its final - // transfer; this nt's loop reclaims it. (Not via coroutine_transfer()'s - // return value: its meaning differs between the amd64 asm and ucontext.) - struct coroutine_context *dead_co; -}; - -#undef except -#undef try -#undef leave -#undef finally - -// per-Ractor -struct rb_thread_sched { - rb_nativethread_lock_t lock_; -#if VM_CHECK_MODE - struct rb_thread_struct *lock_owner; -#endif - struct rb_thread_struct *running; // running thread or NULL - // Most recently running thread or NULL. If this thread wakes up before the newly running - // thread completes the transfer of control, it can interrupt and resume running. - // The new thread clears this field when it takes control. - struct rb_thread_struct *runnable_hot_th; - int runnable_hot_th_waiting; - bool is_running; - - bool enable_mn_threads; - - struct ccan_list_head readyq; - int readyq_cnt; - // ractor scheduling - // When not linked in vm->ractor.sched.grq, this node is kept - // self-linked (ccan_list_node_init), so "linked?" can be read off the - // node itself: enqueuers assert it, and direct transfers cancel an - // outstanding entry (see ractor_sched_cancel_enq). - struct ccan_list_node grq_node; - struct ccan_list_node timeslice_node; // self-linked = not on timeslice.scheds -}; - -struct rb_thread_context; - -// A coroutine (M:N) thread's teardown runs coroutine_thread_terminated -// instead of the dedicated-thread path in thread_start_func_2; see the -// comments there and in thread_pthread_mn.c. th->sched.context is cleared in -// that epilogue, so this also reads as "did not tear down yet". -// (Only meaningful when USE_MN_THREADS -- gate uses accordingly; the macro -// itself is a plain pointer test and always compiles.) -#define th_has_coroutine(th) ((th)->sched.context != NULL) +#include "thread_sched.h" #ifdef RB_THREAD_LOCAL_SPECIFIER NOINLINE(void rb_current_ec_set(struct rb_execution_context_struct *)); @@ -228,9 +60,4 @@ native_tls_set(native_tls_key_t key, void *ptr) RUBY_EXTERN native_tls_key_t ruby_current_ec_key; #endif -struct rb_ractor_struct; -void rb_ractor_sched_wait(struct rb_execution_context_struct *ec, struct rb_ractor_struct *cr, rb_unblock_function_t *ubf, void *ptr); -void rb_ractor_sched_wakeup(struct rb_ractor_struct *r, struct rb_thread_struct *th); -void rb_thread_wake_fence(struct rb_thread_struct *th); - #endif /* RUBY_THREAD_PTHREAD_H */ diff --git a/thread_sched.c b/thread_sched.c new file mode 100644 index 00000000000000..a7d89cc1316eef --- /dev/null +++ b/thread_sched.c @@ -0,0 +1,2501 @@ +/* -*-c-*- */ +/********************************************************************** + + thread_sched.c - platform independent thread/ractor scheduler + + This file is #included from thread.c. It pulls in the platform + implementation (THREAD_IMPL_SRC: thread_pthread.c or thread_win32.c) + first, then builds the scheduler on top of the primitives that file + provides. It implements: + + - the per-Ractor thread scheduler (GVL): struct rb_thread_sched + - the Ractor scheduler: global ready queue (grq) and the VM barrier + - the native thread (NT) main loop + - the timer thread main loop and time slice management + - the unblocking function (UBF) list + + See thread_sched.h for the data structures and for the primitives the + platform layer has to supply. + +**********************************************************************/ + +/* ------------------------------------------------------------------------ + * The scheduler <-> platform contract. + * + * The platform implementation is included below, ahead of the scheduler + * body, so it can use every primitive it defines without declaring them. + * The traffic in the other direction -- the scheduler entry points the + * platform layer (and the M:N scheduler it includes) calls back into -- + * has to be declared here instead. + * ------------------------------------------------------------------------ */ + +#define thread_sched_dump(s) thread_sched_dump_(__FILE__, __LINE__, s) +#define ractor_sched_dump(s) ractor_sched_dump_(__FILE__, __LINE__, s) + +#define thread_sched_lock(a, b) thread_sched_lock_(a, b, __FILE__, __LINE__) +#define thread_sched_unlock(a, b) thread_sched_unlock_(a, b, __FILE__, __LINE__) +#define ractor_sched_lock(a, b) ractor_sched_lock_(a, b, __FILE__, __LINE__) +#define ractor_sched_unlock(a, b) ractor_sched_unlock_(a, b, __FILE__, __LINE__) + +#ifndef MINIMUM_SNT +// make at least MINIMUM_SNT snts for debug. +#define MINIMUM_SNT 0 +#endif + +struct coroutine_context; + +#include "probes.h" + +// thread.c +static void threadptr_trap_interrupt(rb_thread_t *); + +// thread scheduler (GVL) +static void thread_sched_to_running(struct rb_thread_sched *sched, rb_thread_t *th); +static void thread_sched_to_waiting(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately); +static void thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th); +static void coroutine_transfer0(struct coroutine_context *transfer_from, + struct coroutine_context *transfer_to, bool to_dead); +static void thread_sched_lock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line); +static void thread_sched_unlock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line); +static void thread_sched_unlock_no_log(struct rb_thread_sched *sched, rb_thread_t *th); +static void thread_sched_set_locked(struct rb_thread_sched *sched, rb_thread_t *th); +static void thread_sched_setup_running_threads(struct rb_thread_sched *sched, rb_ractor_t *cr, rb_vm_t *vm, + rb_thread_t *add_th, rb_thread_t *del_th); +static void thread_sched_add_running_thread(struct rb_thread_sched *sched, rb_thread_t *th); +static void thread_sched_to_ready(struct rb_thread_sched *sched, rb_thread_t *th); +static void thread_sched_to_ready_common(struct rb_thread_sched *sched, rb_thread_t *th, bool wakeup, bool will_switch); +static void thread_sched_to_dead_common(struct rb_thread_sched *sched, rb_thread_t *th); +static void thread_sched_to_waiting_until_wakeup(struct rb_thread_sched *sched, rb_thread_t *th, const rb_hrtime_t *end); +static void thread_sched_wait_running_turn(struct rb_thread_sched *sched, rb_thread_t *th, bool can_direct_transfer, const rb_hrtime_t *end); +static void thread_sched_wakeup_next_thread(struct rb_thread_sched *sched, rb_thread_t *th, bool will_switch); + +// native thread <-> ractor assignment +static void native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt); +static void native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt); +static void native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th); + +// ractor scheduler +static void ractor_sched_lock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line); +static void ractor_sched_unlock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line); +static void ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r); +static void ractor_sched_cancel_enq(rb_vm_t *vm, struct rb_thread_sched *sched); + +// VM wide scheduler state; the platform's Init_native_thread() calls this +static void thread_sched_init_vm(rb_vm_t *vm); + +// implemented by the M:N scheduler (thread_sched_mn.c) or stubbed out +static bool ractor_sched_timeout_arm(rb_thread_t *th, const rb_hrtime_t *rel); +static bool ractor_sched_timeout_disarm(rb_thread_t *th); +static void timer_thread_wake_fence(struct rb_thread_struct *th); + +// unblocking function (UBF) +static bool ubf_set(rb_thread_t *th, rb_unblock_function_t *func, void *arg, rb_atomic_t *event_serial); +static void ubf_clear(rb_thread_t *th, bool clear_serial); + +// native thread main loops +static void call_thread_start_func_2(rb_thread_t *th); +static void *nt_start(void *ptr); + +// timer thread +static void *timer_thread_func(void *ptr); +static int timer_thread_set_timeout(rb_vm_t *vm); +static void timer_thread_check_timeslice(rb_vm_t *vm); +static bool timeslice_scan(rb_vm_t *vm, bool interrupt); +static void timer_thread_wakeup(void); +static void timer_thread_wakeup_locked(rb_vm_t *vm); +static void timer_thread_wakeup_force(void); + +#include THREAD_IMPL_SRC + +// Defaults for what the platform above did not opt out of. + +#ifndef RB_NATIVE_MUTEX_TRYLOCK_DETECTS_SELF +// Whether rb_native_mutex_trylock() reports EBUSY when the calling thread is +// itself the owner. A recursive lock (a Windows CRITICAL_SECTION) grants it +// again instead, so it cannot back a "somebody holds this" assertion. +#define RB_NATIVE_MUTEX_TRYLOCK_DETECTS_SELF 1 +#endif + +/* ------------------------------------------------------------------------ + * The scheduler itself. + * ------------------------------------------------------------------------ */ + +static bool +th_has_dedicated_nt(const rb_thread_t *th) +{ + // TODO: th->has_dedicated_nt + return th->nt->dedicated > 0; +} + +RBIMPL_ATTR_MAYBE_UNUSED() +static void +thread_sched_dump_(const char *file, int line, struct rb_thread_sched *sched) +{ + fprintf(stderr, "@%s:%d running:%d\n", file, line, sched->running ? (int)sched->running->serial : -1); + rb_thread_t *th; + int i = 0; + ccan_list_for_each(&sched->readyq, th, sched.node.readyq) { + i++; if (i>10) rb_bug("too many"); + fprintf(stderr, " ready:%d (%sNT:%d)\n", th->serial, + th->nt ? (th->nt->dedicated ? "D" : "S") : "x", + th->nt ? (int)th->nt->serial : -1); + } +} + + +RBIMPL_ATTR_MAYBE_UNUSED() +static void +ractor_sched_dump_(const char *file, int line, rb_vm_t *vm) +{ + rb_ractor_t *r; + + fprintf(stderr, "ractor_sched_dump %s:%d\n", file, line); + + int i = 0; + ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) { + i++; + if (i>10) rb_bug("!!"); + fprintf(stderr, " %d ready:%d\n", i, rb_ractor_id(r)); + } +} + + +static void +thread_sched_set_locked(struct rb_thread_sched *sched, rb_thread_t *th) +{ +#if VM_CHECK_MODE > 0 + VM_ASSERT(sched->lock_owner == NULL); + + sched->lock_owner = th; +#endif +} + +static void +thread_sched_set_unlocked(struct rb_thread_sched *sched, rb_thread_t *th) +{ +#if VM_CHECK_MODE > 0 + VM_ASSERT(sched->lock_owner == th); + + sched->lock_owner = NULL; +#endif +} + +static void +thread_sched_lock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line) +{ + rb_native_mutex_lock(&sched->lock_); + +#if VM_CHECK_MODE + RUBY_DEBUG_LOG2(file, line, "r:%d th:%u", th ? (int)rb_ractor_id(th->ractor) : -1, rb_th_serial(th)); +#else + RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th)); +#endif + + thread_sched_set_locked(sched, th); +} + +static void +thread_sched_unlock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line) +{ + RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th)); + + thread_sched_set_unlocked(sched, th); + + rb_native_mutex_unlock(&sched->lock_); +} + +// Like thread_sched_unlock(), but never dereferences th (the debug log above +// reads th->serial). For the MN termination epilogue, which unlocks after th +// may already be collectable. Keep in sync with thread_sched_unlock_. +RBIMPL_ATTR_MAYBE_UNUSED() +static void +thread_sched_unlock_no_log(struct rb_thread_sched *sched, rb_thread_t *th) +{ + thread_sched_set_unlocked(sched, th); // pointer compare only + + rb_native_mutex_unlock(&sched->lock_); +} + +static void +ASSERT_thread_sched_locked(struct rb_thread_sched *sched, rb_thread_t *th) +{ +#if RB_NATIVE_MUTEX_TRYLOCK_DETECTS_SELF + VM_ASSERT(rb_native_mutex_trylock(&sched->lock_) == EBUSY); +#endif + +#if VM_CHECK_MODE + if (th) { + VM_ASSERT(sched->lock_owner == th); + } + else { + VM_ASSERT(sched->lock_owner != NULL); + } +#endif +} + + +RBIMPL_ATTR_MAYBE_UNUSED() +static unsigned int +rb_ractor_serial(const rb_ractor_t *r) +{ + if (r) { + return rb_ractor_id(r); + } + else { + return 0; + } +} + +static void +ractor_sched_set_locked(rb_vm_t *vm, rb_ractor_t *cr) +{ +#if VM_CHECK_MODE > 0 + VM_ASSERT(vm->ractor.sched.lock_owner == NULL); + VM_ASSERT(vm->ractor.sched.locked == false); + + vm->ractor.sched.lock_owner = cr; + vm->ractor.sched.locked = true; +#endif +} + +static void +ractor_sched_set_unlocked(rb_vm_t *vm, rb_ractor_t *cr) +{ +#if VM_CHECK_MODE > 0 + VM_ASSERT(vm->ractor.sched.locked); + VM_ASSERT(vm->ractor.sched.lock_owner == cr); + + vm->ractor.sched.locked = false; + vm->ractor.sched.lock_owner = NULL; +#endif +} + + +static void +ractor_sched_lock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line) +{ + rb_native_mutex_lock(&vm->ractor.sched.lock); + +#if VM_CHECK_MODE + RUBY_DEBUG_LOG2(file, line, "cr:%u prev_owner:%u", rb_ractor_serial(cr), rb_ractor_serial(vm->ractor.sched.lock_owner)); +#else + RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr)); +#endif + + ractor_sched_set_locked(vm, cr); +} + +static void +ractor_sched_unlock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line) +{ + RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr)); + + ractor_sched_set_unlocked(vm, cr); + rb_native_mutex_unlock(&vm->ractor.sched.lock); +} + +static void +ASSERT_ractor_sched_locked(rb_vm_t *vm, rb_ractor_t *cr) +{ +#if RB_NATIVE_MUTEX_TRYLOCK_DETECTS_SELF + VM_ASSERT(rb_native_mutex_trylock(&vm->ractor.sched.lock) == EBUSY); +#endif + VM_ASSERT(vm->ractor.sched.locked); + VM_ASSERT(cr == NULL || vm->ractor.sched.lock_owner == cr); +} + +static void ractor_sched_barrier_join_signal_locked(rb_vm_t *vm); + +/* ntlist registration: a thread that executes Ruby code is always registered, + * in its snt's nt->running_th or on running_dnts via its dedicated nt. The + * only unregistered execution is scheduler glue (parking, resuming), which + * touches no Ruby heap, and the barrier wait below. */ +static void +ntlist_add_running(rb_vm_t *vm, rb_thread_t *th) +{ + struct rb_native_thread *nt = th->nt; + + // a dedicated nt is not on the snts list the scans walk: running_dnts instead + if (nt != NULL && nt->dedicated == 0) { + rb_native_mutex_lock(&nt->running_th_lock); + { + VM_ASSERT(nt->running_th == NULL); + nt->running_th = th; + } + rb_native_mutex_unlock(&nt->running_th_lock); + } + else { + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + // an snt gone dedicated (rb_thread_lock_native_thread) has no + // creation-time running_thread: the registration supplies it + nt->running_thread = th; + ccan_list_add(&vm->ractor.sched.ntlist.running_dnts, &nt->running_dnts_node); + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); + } +} + +// Returns whether the active barrier's walk had counted this registration: +// such a deregistration owes the snapshot count a decrement. Read and +// cleared under the registration's own lock, so it pairs with the walk. +static bool +ntlist_del_running(rb_vm_t *vm, rb_thread_t *th) +{ + struct rb_native_thread *nt = th->nt; + uint32_t serial; + bool counted; + bool in_running_th; + + // The registration itself says where it is: nt->running_th holds th, or + // th's nt hangs on running_dnts. barrier_serial is read inside the + // registration's lock, ordered with the walk that stamped there. + rb_native_mutex_lock(&nt->running_th_lock); + { + in_running_th = (nt->running_th == th); + if (in_running_th) { + nt->running_th = NULL; + serial = vm->ractor.sched.barrier_serial; + counted = (nt->barrier_counted_serial == serial); + nt->barrier_counted_serial = serial - 1; // only once per barrier + } + } + rb_native_mutex_unlock(&nt->running_th_lock); + + if (!in_running_th) { + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + ccan_list_del_init(&nt->running_dnts_node); + serial = vm->ractor.sched.barrier_serial; + counted = (nt->barrier_counted_serial == serial); + nt->barrier_counted_serial = serial - 1; + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); + } + return counted; +} + +// Stamp a registration into the active barrier's snapshot unless the walk +// already counted it; returns whether it stamped. Called under sched.lock, +// so it is serialized with the walk: the stamp says exactly whether the +// registration came first. +static bool +ntlist_stamp_if_uncounted(rb_vm_t *vm, rb_thread_t *th) +{ + struct rb_native_thread *nt = th->nt; + uint32_t serial = vm->ractor.sched.barrier_serial; // sched.lock is held + bool stamped; + bool in_running_th; + + rb_native_mutex_lock(&nt->running_th_lock); + { + in_running_th = (nt->running_th == th); + if (in_running_th) { + stamped = (nt->barrier_counted_serial != serial); + nt->barrier_counted_serial = serial; + } + } + rb_native_mutex_unlock(&nt->running_th_lock); + + if (!in_running_th) { + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + stamped = (nt->barrier_counted_serial != serial); + nt->barrier_counted_serial = serial; + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); + } + return stamped; +} + +// Record a thread entering/leaving the running set, with no global lock and +// no count: the records themselves are what the barrier counts. Pairing: +// the barrier sets barrier_is_waiting and then walks the records under their +// locks; we move a record and then read the flag, so one side sees the other. +// List sched for the timer's timeslice ticks. The caller holds sched->lock_ +// with the readyq non-empty, so the timer cannot prune the entry meanwhile. +static void +timeslice_sched_link(rb_vm_t *vm, struct rb_thread_sched *sched) +{ + rb_native_mutex_lock(&vm->ractor.sched.timeslice.lock); + { + if (sched->timeslice_node.next == &sched->timeslice_node) { + ccan_list_add_tail(&vm->ractor.sched.timeslice.scheds, &sched->timeslice_node); + } + } + rb_native_mutex_unlock(&vm->ractor.sched.timeslice.lock); +} + +static void +thread_sched_setup_running_threads(struct rb_thread_sched *sched, rb_ractor_t *cr, rb_vm_t *vm, + rb_thread_t *add_th, rb_thread_t *del_th) +{ + RUBY_DEBUG_LOG("+:%u -:%u", rb_th_serial(add_th), rb_th_serial(del_th)); + + if (del_th) { + bool counted = ntlist_del_running(vm, del_th); + sched->is_running = false; + + // The first load is only a filter; the one under sched.lock decides. + // A missed flag means this deregistration preceded the barrier's walk. + if (UNLIKELY(RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting))) { + ractor_sched_lock(vm, cr); + { + if (RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting)) { + if (counted) { + VM_ASSERT(vm->ractor.sched.barrier_running_cnt > 0); + vm->ractor.sched.barrier_running_cnt--; + } + ractor_sched_barrier_join_signal_locked(vm); + } + } + ractor_sched_unlock(vm, cr); + } + } + + if (add_th) { + ntlist_add_running(vm, add_th); + + if (UNLIKELY(RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting))) { + // A stop-the-world section. In its waiting phase sched.lock is + // takable: join the snapshot count and take the interrupt (this + // thread joins at its next check, like any walked runner). In + // the GC phase the barrier holds sched.lock to its end, so this + // blocks here, as the old global-lock design did. + ractor_sched_lock(vm, cr); + { + if (RUBY_ATOMIC_LOAD(vm->ractor.sched.barrier_is_waiting) && + ntlist_stamp_if_uncounted(vm, add_th)) { + // the walk ran before this registration; count it in + RUBY_DEBUG_LOG("barrier_is_waiting"); + vm->ractor.sched.barrier_running_cnt++; + RUBY_VM_SET_VM_BARRIER_INTERRUPT(add_th->ec); + } + } + ractor_sched_unlock(vm, cr); + } + + sched->is_running = true; + + // taking a turn with waiters already queued needs the timeslice ticks + if (!ccan_list_empty(&sched->readyq)) { + timeslice_sched_link(vm, sched); + ractor_sched_lock(vm, cr); + { + if (vm->ractor.sched.timeslice_wait_inf) { + timer_thread_wakeup_locked(vm); + } + } + ractor_sched_unlock(vm, cr); + } + } +} + +static void +thread_sched_add_running_thread(struct rb_thread_sched *sched, rb_thread_t *th) +{ + ASSERT_thread_sched_locked(sched, th); + VM_ASSERT(sched->running == th); + + rb_vm_t *vm = th->vm; + thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL); +} + +static void +thread_sched_del_running_thread(struct rb_thread_sched *sched, rb_thread_t *th) +{ + ASSERT_thread_sched_locked(sched, th); + + rb_vm_t *vm = th->vm; + thread_sched_setup_running_threads(sched, th->ractor, vm, NULL, th); +} + +void +rb_add_running_thread(rb_thread_t *th) +{ + struct rb_thread_sched *sched = TH_SCHED(th); + + thread_sched_lock(sched, th); + { + thread_sched_add_running_thread(sched, th); + } + thread_sched_unlock(sched, th); +} + +void +rb_del_running_thread(rb_thread_t *th) +{ + struct rb_thread_sched *sched = TH_SCHED(th); + + thread_sched_lock(sched, th); + { + thread_sched_del_running_thread(sched, th); + } + thread_sched_unlock(sched, th); +} + +// setup current or next running thread +// sched->running should be set only on this function. +// +// if th is NULL, there is no running threads. +static void +thread_sched_set_running(struct rb_thread_sched *sched, rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u->th:%u", rb_th_serial(sched->running), rb_th_serial(th)); + VM_ASSERT(sched->running != th); + + if (RUBY_DTRACE_RTS_SET_RUNNING_ENABLED()) { + RUBY_DTRACE_RTS_SET_RUNNING(sched, sched->running, th); + } + + sched->running = th; +} + +RBIMPL_ATTR_MAYBE_UNUSED() +static bool +thread_sched_readyq_contain_p(struct rb_thread_sched *sched, rb_thread_t *th) +{ + rb_thread_t *rth; + ccan_list_for_each(&sched->readyq, rth, sched.node.readyq) { + if (rth == th) { + VM_ASSERT(th->sched.node.is_ready); + return true; + } + } + VM_ASSERT(!th->sched.node.is_ready); + return false; +} + +// deque thread from the ready queue. +// if the ready queue is empty, return NULL. +// +// return deque'ed running thread (or NULL). +static rb_thread_t * +thread_sched_deq(struct rb_thread_sched *sched) +{ + ASSERT_thread_sched_locked(sched, NULL); + rb_thread_t *next_th; + + VM_ASSERT(sched->running != NULL); + + if (ccan_list_empty(&sched->readyq)) { + next_th = NULL; + } + else { + next_th = ccan_list_pop(&sched->readyq, rb_thread_t, sched.node.readyq); + VM_ASSERT(next_th->sched.node.is_ready); + next_th->sched.node.is_ready = false; + + VM_ASSERT(sched->readyq_cnt > 0); + sched->readyq_cnt--; + ccan_list_node_init(&next_th->sched.node.readyq); + } + + RUBY_DEBUG_LOG("next_th:%u readyq_cnt:%d", rb_th_serial(next_th), sched->readyq_cnt); + + return next_th; +} + +// enqueue ready thread to the ready queue. +static void +thread_sched_enq(struct rb_thread_sched *sched, rb_thread_t *ready_th) +{ + ASSERT_thread_sched_locked(sched, NULL); + RUBY_DEBUG_LOG("ready_th:%u readyq_cnt:%d", rb_th_serial(ready_th), sched->readyq_cnt); + + VM_ASSERT(sched->running != NULL); + VM_ASSERT(!thread_sched_readyq_contain_p(sched, ready_th)); + + bool timeslice_onset = sched->is_running && ccan_list_empty(&sched->readyq); + + ccan_list_add_tail(&sched->readyq, &ready_th->sched.node.readyq); + ready_th->sched.node.is_ready = true; + sched->readyq_cnt++; + + if (timeslice_onset) { + // The running thread needs the timeslice ticks now. Linked before + // the check under sched.lock: either the timer's scan (same lock) + // sees the sched, or this sees timeslice_wait_inf. + rb_vm_t *vm = ready_th->vm; + timeslice_sched_link(vm, sched); + ractor_sched_lock(vm, NULL); + { + if (vm->ractor.sched.timeslice_wait_inf) { + timer_thread_wakeup_locked(vm); + } + } + ractor_sched_unlock(vm, NULL); + } +} + +// DNT: kick condvar +// SNT: TODO +static void +thread_sched_wakeup_running_thread(struct rb_thread_sched *sched, rb_thread_t *next_th, bool will_switch) +{ + ASSERT_thread_sched_locked(sched, NULL); + VM_ASSERT(sched->running == next_th); + + if (next_th) { + if (next_th->nt) { + if (th_has_dedicated_nt(next_th)) { + RUBY_DEBUG_LOG("pinning th:%u", next_th->serial); + rb_native_cond_signal(&next_th->nt->readyq); + } + else { + // TODO + RUBY_DEBUG_LOG("th:%u is already running.", next_th->serial); + } + } + else { + if (will_switch) { + RUBY_DEBUG_LOG("th:%u (do nothing)", rb_th_serial(next_th)); + } + else { + RUBY_DEBUG_LOG("th:%u (enq)", rb_th_serial(next_th)); + ractor_sched_enq(next_th->vm, next_th->ractor); + } + } + } + else { + RUBY_DEBUG_LOG("no waiting threads%s", ""); + } +} + +// waiting -> ready (locked) +static void +thread_sched_to_ready_common(struct rb_thread_sched *sched, rb_thread_t *th, bool wakeup, bool will_switch) +{ + RUBY_DEBUG_LOG("th:%u running:%u redyq_cnt:%d", rb_th_serial(th), rb_th_serial(sched->running), sched->readyq_cnt); + + VM_ASSERT(sched->running != th); + VM_ASSERT(!thread_sched_readyq_contain_p(sched, th)); + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_READY, th); + + if (sched->running == NULL) { + thread_sched_set_running(sched, th); + if (wakeup) thread_sched_wakeup_running_thread(sched, th, will_switch); + } + else { + thread_sched_enq(sched, th); + } +} + +// waiting -> ready +// +// `th` had became "waiting" state by `thread_sched_to_waiting` +// and `thread_sched_to_ready` enqueue `th` to the thread ready queue. +RBIMPL_ATTR_MAYBE_UNUSED() +static void +thread_sched_to_ready(struct rb_thread_sched *sched, rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); + + thread_sched_lock(sched, th); + { + thread_sched_to_ready_common(sched, th, true, false); + } + thread_sched_unlock(sched, th); +} + +// wait until sched->running is `th`. `end` is an absolute deadline for a dedicated +static void +thread_sched_wait_running_turn(struct rb_thread_sched *sched, rb_thread_t *th, bool can_direct_transfer, const rb_hrtime_t *end) +{ + RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); + + ASSERT_thread_sched_locked(sched, th); + VM_ASSERT(th == rb_ec_thread_ptr(rb_current_ec_noinline())); + + bool timedout = false; + + if (th != sched->running) { + // TODO: This optimization should also be made to work for MN_THREADS + if (th->has_dedicated_nt && th == sched->runnable_hot_th && (sched->running == NULL || sched->running->has_dedicated_nt)) { + RUBY_DEBUG_LOG("(nt) stealing: hot-th:%u. running:%u", rb_th_serial(th), rb_th_serial(sched->running)); + + // th serves itself on its own nt, displacing the enqueued + // running thread back to the readyq: cancel the entry that was + // posted for it (a later dequeue would find this Ractor served + // and its next enqueue would double-list the node) + ractor_sched_cancel_enq(th->vm, sched); + + // If there is a thread set to run, move it back to the front of the readyq + if (sched->running != NULL) { + rb_thread_t *running = sched->running; + VM_ASSERT(!thread_sched_readyq_contain_p(sched, running)); + running->sched.node.is_ready = true; + ccan_list_add(&sched->readyq, &running->sched.node.readyq); + sched->readyq_cnt++; + } + + // Pull off the ready queue and start running. + if (th->sched.node.is_ready) { + VM_ASSERT(thread_sched_readyq_contain_p(sched, th)); + ccan_list_del_init(&th->sched.node.readyq); + th->sched.node.is_ready = false; + sched->readyq_cnt--; + } + thread_sched_set_running(sched, th); + rb_ractor_thread_switch(th->ractor, th, false); + } + else if (th == sched->runnable_hot_th) { + // The hot thread cannot steal the control (e.g. the running thread + // is an MN thread). It is going to sleep, so it is no longer spinning; + // drop the hint so that other threads don't yield the lock to it. + sched->runnable_hot_th = NULL; + sched->runnable_hot_th_waiting = 0; + } + + // already deleted from running threads + + + // wait for execution right + rb_thread_t *next_th; + while((next_th = sched->running) != th) { + if (th_has_dedicated_nt(th)) { + RUBY_DEBUG_LOG("(nt) sleep th:%u running:%u", rb_th_serial(th), rb_th_serial(sched->running)); + + thread_sched_set_unlocked(sched, th); + { + RUBY_DEBUG_LOG("nt:%d cond:%p", th->nt->serial, &th->nt->readyq); + rb_nativethread_cond_t *cond = &th->nt->readyq; + + // Once someone has queued this thread the deadline is spent: it + // is waiting for a turn, not for the time, and arming a kernel + // timer for every round of that costs more than the wait. + // Once someone has queued this thread the deadline is spent: it + // is waiting for a turn, not for the time, and arming a kernel + // timer for every round of that costs more than the wait. + if (end && !th->sched.node.is_ready) { + rb_hrtime_t abs = *end; + + if (!RB_NATIVE_COND_HRTIME_DEADLINE_P()) { + // the condvar counts in another clock: restate it there + rb_hrtime_t now = rb_hrtime_now(); + abs = native_cond_timeout(cond, *end > now ? *end - now : 0); + } + timedout = native_cond_timedwait(cond, &sched->lock_, &abs) == ETIMEDOUT; + } + else { + rb_native_cond_wait(cond, &sched->lock_); + } + } + thread_sched_set_locked(sched, th); + + if (timedout && + sched->running != th && !th->sched.node.is_ready) { + // the deadline passed and nobody woke this thread: get back in + // line for the running turn, then wait for it without a deadline + thread_sched_to_ready_common(sched, th, false, false); + end = NULL; + } + + if (sched->runnable_hot_th != NULL && sched->runnable_hot_th_waiting) { + VM_ASSERT(sched->runnable_hot_th != th); + // Give the hot thread a chance to preempt, if it's actively spinning. + // On multicore, this reduces the rate of core-switching. On single-core it + // should mostly be a nop, since the other thread can't be concurrently spinning. + thread_sched_unlock(sched, th); + thread_sched_lock(sched, th); + } + + RUBY_DEBUG_LOG("(nt) wakeup %s", sched->running == th ? "success" : "failed"); + if (th == sched->running) { + rb_ractor_thread_switch(th->ractor, th, false); + } + } + else { + // search another ready thread + if (can_direct_transfer && + (next_th = sched->running) != NULL && + !next_th->nt // next_th is running or has dedicated nt + ) { + + RUBY_DEBUG_LOG("th:%u->%u (direct)", rb_th_serial(th), rb_th_serial(next_th)); + + thread_sched_set_unlocked(sched, th); + { + rb_ractor_set_current_ec(th->ractor, NULL); + thread_sched_switch(th, next_th); + } + thread_sched_set_locked(sched, th); + } + else { + // search another ready ractor + struct rb_native_thread *nt = th->nt; + native_thread_assign(NULL, th); + + RUBY_DEBUG_LOG("th:%u->%u (ractor scheduling)", rb_th_serial(th), rb_th_serial(next_th)); + + thread_sched_set_unlocked(sched, th); + { + rb_ractor_set_current_ec(th->ractor, NULL); + coroutine_transfer0(th->sched.context, nt->nt_context, false); + } + thread_sched_set_locked(sched, th); + } + + VM_ASSERT(rb_current_ec_noinline() == th->ec); + } + } + + VM_ASSERT(th->nt != NULL); + VM_ASSERT(rb_current_ec_noinline() == th->ec); + VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none); + + // add th to running threads + thread_sched_add_running_thread(sched, th); + } + + // Control transfer to the current thread is now complete. The original thread + // cannot steal control at this point. + sched->runnable_hot_th = NULL; + sched->runnable_hot_th_waiting = 0; + + + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_RESUMED, th); +} + +// waiting -> ready -> running (locked) +static void +thread_sched_to_running_common(struct rb_thread_sched *sched, rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u dedicated:%d", rb_th_serial(th), th_has_dedicated_nt(th)); + + VM_ASSERT(sched->running != th); + VM_ASSERT(th_has_dedicated_nt(th)); + VM_ASSERT(GET_THREAD() == th); + + native_thread_dedicated_dec(th->vm, th->ractor, th->nt); + + // waiting -> ready + thread_sched_to_ready_common(sched, th, false, false); + + if (sched->running == th) { + thread_sched_add_running_thread(sched, th); + } + + // TODO: check SNT number + thread_sched_wait_running_turn(sched, th, false, NULL); +} + +// waiting -> ready -> running +// +// `th` had been waiting by `thread_sched_to_waiting()` +// and run a dedicated task (like waitpid and so on). +// After the dedicated task, this function is called +// to join a normal thread-scheduling. +static void +thread_sched_to_running(struct rb_thread_sched *sched, rb_thread_t *th) +{ + // We are reading and writing these sched fields without lock cover, but + // there are no correctness issues resulting from stale cache or delayed writeback. + // When it works, this causes the next-scheduled thread to yield the sched lock + // briefly so that we can grab it if we're still spinning (not descheduled yet). + if (sched->runnable_hot_th == th) { + sched->runnable_hot_th_waiting = 1; + } + thread_sched_lock(sched, th); + { + thread_sched_to_running_common(sched, th); + } + thread_sched_unlock(sched, th); +} + +// resume a next thread in the thread ready queue. +// +// deque next running thread from the ready thread queue and +// resume this thread if available. +// +// If the next therad has a dedicated native thraed, simply signal to resume. +// Otherwise, make the ractor ready and other nt will run the ractor and the thread. +static void +thread_sched_wakeup_next_thread(struct rb_thread_sched *sched, rb_thread_t *th, bool will_switch) +{ + ASSERT_thread_sched_locked(sched, th); + + VM_ASSERT(sched->running == th); + VM_ASSERT(sched->running->nt != NULL); + + rb_thread_t *next_th = thread_sched_deq(sched); + + RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th)); + VM_ASSERT(th != next_th); + + thread_sched_set_running(sched, next_th); + VM_ASSERT(next_th == sched->running); + thread_sched_wakeup_running_thread(sched, next_th, will_switch); + + if (th != next_th) { + thread_sched_del_running_thread(sched, th); + } +} + +// running -> dead (locked) +static void +thread_sched_to_dead_common(struct rb_thread_sched *sched, rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated); + + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); + + // A dying coroutine thread (will_switch=true here) does NOT wake the + // next thread now: it is still winding down (co_start's epilogue), and + // the same Ractor must not have two threads executing at once. The + // epilogue enqueues the Ractor after its last rb_ractor_t access. + thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th)); + + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_EXITED, th); +} + +// running -> dead +static void +thread_sched_to_dead(struct rb_thread_sched *sched, rb_thread_t *th) +{ + // wait out any pending wake here, while th's Ractor is still alive + timer_thread_wake_fence(th); + + thread_sched_lock(sched, th); + { + thread_sched_to_dead_common(sched, th); + } + thread_sched_unlock(sched, th); +} + +// running -> waiting (locked) +// +// This thread will run dedicated task (th->nt->dedicated++). +static void +thread_sched_to_waiting_common(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately) +{ + RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated); + + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); + + native_thread_dedicated_inc(th->vm, th->ractor, th->nt); + if (!yield_immediately) { + sched->runnable_hot_th = th; + sched->runnable_hot_th_waiting = 0; + } + thread_sched_wakeup_next_thread(sched, th, false); +} + +// running -> waiting +// +// This thread will run a dedicated task. +static void +thread_sched_to_waiting(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately) +{ + thread_sched_lock(sched, th); + { + thread_sched_to_waiting_common(sched, th, yield_immediately); + } + thread_sched_unlock(sched, th); +} + +// mini utility func +// return true if any there are any interrupts +static bool +ubf_set(rb_thread_t *th, rb_unblock_function_t *func, void *arg, rb_atomic_t *event_serial) +{ + VM_ASSERT(func != NULL); + + retry: + if (RUBY_VM_INTERRUPTED(th->ec)) { + RUBY_DEBUG_LOG("interrupted:0x%x", th->ec->interrupt_flag); + return true; + } + + rb_native_mutex_lock(&th->interrupt_lock); + { + if (!th->ec->raised_flag && RUBY_VM_INTERRUPTED(th->ec)) { + rb_native_mutex_unlock(&th->interrupt_lock); + goto retry; + } + + VM_ASSERT(th->unblock.func == NULL); + th->unblock.func = func; + th->unblock.arg = arg; + if (event_serial) { + rb_atomic_t prev_serial = RUBY_ATOMIC_FETCH_ADD(th->unblock.event_serial, 1); + *event_serial = prev_serial+1; + } + } + rb_native_mutex_unlock(&th->interrupt_lock); + + return false; +} + +static void +ubf_clear(rb_thread_t *th, bool clear_serial) +{ + rb_native_mutex_lock(&th->interrupt_lock); + { + th->unblock.func = NULL; + th->unblock.arg = NULL; + if (clear_serial) { + RUBY_ATOMIC_ADD(th->unblock.event_serial, 1); + } + } + rb_native_mutex_unlock(&th->interrupt_lock); +} + +static void +ubf_waiting(void *ptr) +{ + rb_thread_t *th = (rb_thread_t *)ptr; + struct rb_thread_sched *sched = TH_SCHED(th); + + // only once. it is safe because th->interrupt_lock is already acquired. + th->unblock.func = NULL; + th->unblock.arg = NULL; + + RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); + + thread_sched_lock(sched, th); + { + if (sched->running == th || th->sched.node.is_ready) { + // not sleeping yet, or a deadline already put it back in line + } + else { + thread_sched_to_ready_common(sched, th, true, false); + + // If the turn is taken, th stays parked until the running thread yields. + // For a timed wait, wake it early anyway: it re-parks at once, but its + // wakeup then runs on another core in parallel with the running thread, + // off the handoff path. An untimed wait has no post-wake bookkeeping + // worth pipelining, so it skips the extra futex round. + if (sched->running != th && th->sched.waiting_timed && + th->nt != NULL && th_has_dedicated_nt(th)) { + rb_native_cond_signal(&th->nt->readyq); + } + } + } + thread_sched_unlock(sched, th); +} + +// running -> waiting +// +// This thread will sleep until other thread wakeup the thread. `end` is an +// absolute deadline, NULL to sleep until woken; only a dedicated native thread, +// which parks on its own condvar, can take one. +static void +thread_sched_to_waiting_until_wakeup(struct rb_thread_sched *sched, rb_thread_t *th, const rb_hrtime_t *end) +{ + RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); + + VM_ASSERT(end == NULL || th_has_dedicated_nt(th)); + + RB_VM_SAVE_MACHINE_CONTEXT(th); + + + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); + + thread_sched_lock(sched, th); + { + // NOTE: there's a lock ordering inversion here with the ubf call, but it's benign. + if (ubf_set(th, ubf_waiting, (void *)th, NULL)) { + RUBY_DEBUG_LOG("th:%u interrupted", rb_th_serial(th)); + } + else { + bool can_direct_transfer = !th_has_dedicated_nt(th); + th->sched.waiting_timed = (end != NULL); // never true here for M:N (end is NULL) + // NOTE: th->status is set before and after this sleep outside of this function in `sleep_forever` + thread_sched_wakeup_next_thread(sched, th, can_direct_transfer); + thread_sched_wait_running_turn(sched, th, can_direct_transfer, end); + th->sched.waiting_timed = false; + } + } + thread_sched_unlock(sched, th); + + ubf_clear(th, false); +} + +// run another thread in the ready queue. +// continue to run if there are no ready threads. +static void +thread_sched_yield(struct rb_thread_sched *sched, rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%d sched->readyq_cnt:%d", (int)th->serial, sched->readyq_cnt); + + thread_sched_lock(sched, th); + { + if (!ccan_list_empty(&sched->readyq)) { + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); + thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th)); + bool can_direct_transfer = !th_has_dedicated_nt(th); + thread_sched_to_ready_common(sched, th, false, can_direct_transfer); + thread_sched_wait_running_turn(sched, th, can_direct_transfer, NULL); + th->status = THREAD_RUNNABLE; + } + else { + VM_ASSERT(sched->readyq_cnt == 0); + } + } + thread_sched_unlock(sched, th); +} + +void +rb_thread_sched_init(struct rb_thread_sched *sched, bool atfork) +{ + rb_native_mutex_initialize(&sched->lock_); + +#if VM_CHECK_MODE + sched->lock_owner = NULL; +#endif + + ccan_list_head_init(&sched->readyq); + sched->readyq_cnt = 0; + ccan_list_node_init(&sched->grq_node); // self-linked = not enqueued + ccan_list_node_init(&sched->timeslice_node); + +#if USE_MN_THREADS + if (!atfork) sched->enable_mn_threads = true; // MN is enabled on Ractors +#endif +} + +static void +coroutine_transfer0(struct coroutine_context *transfer_from, struct coroutine_context *transfer_to, bool to_dead) +{ +#ifdef RUBY_ASAN_ENABLED + void **fake_stack = to_dead ? NULL : &transfer_from->fake_stack; + __sanitizer_start_switch_fiber(fake_stack, transfer_to->stack_base, transfer_to->stack_size); +#endif + +#if defined(COROUTINE_SANITIZE_THREAD) + /* Tell TSan we are switching to transfer_to's fiber before the stack + * switch, so its per-thread shadow stack stays bound to the right + * coroutine. */ + __tsan_switch_to_fiber(transfer_to->tsan_fiber, 0); +#endif + + RBIMPL_ATTR_MAYBE_UNUSED() + struct coroutine_context *returning_from = coroutine_transfer(transfer_from, transfer_to); + + /* if to_dead was passed, the caller is promising that this coroutine is finished and it should + * never be resumed! */ + VM_ASSERT(!to_dead); +#ifdef RUBY_ASAN_ENABLED + __sanitizer_finish_switch_fiber(transfer_from->fake_stack, + (const void**)&returning_from->stack_base, &returning_from->stack_size); +#endif +} + +static void +thread_sched_switch0(struct coroutine_context *current_cont, rb_thread_t *next_th, struct rb_native_thread *nt, bool to_dead) +{ + VM_ASSERT(!nt->dedicated); + VM_ASSERT(next_th->nt == NULL); + + RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th)); + + // this direct transfer serves next_th without a dequeue; cancel its + // Ractor's outstanding grq entry (no-op when nothing is enqueued) + ractor_sched_cancel_enq(next_th->vm, TH_SCHED(next_th)); + + ruby_thread_set_native(next_th); + native_thread_assign(nt, next_th); + + coroutine_transfer0(current_cont, next_th->sched.context, to_dead); +} + +static void +thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th) +{ + struct rb_native_thread *nt = cth->nt; + native_thread_assign(NULL, cth); + RUBY_DEBUG_LOG("th:%u->%u on nt:%d", rb_th_serial(cth), rb_th_serial(next_th), nt->serial); + thread_sched_switch0(cth->sched.context, next_th, nt, cth->status == THREAD_KILLED); +} + +#if VM_CHECK_MODE > 0 +RBIMPL_ATTR_MAYBE_UNUSED() +static unsigned int +grq_size(rb_vm_t *vm, rb_ractor_t *cr) +{ + ASSERT_ractor_sched_locked(vm, cr); + + rb_ractor_t *r, *prev_r = NULL; + unsigned int i = 0; + + ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) { + i++; + + VM_ASSERT(r != prev_r); + prev_r = r; + } + return i; +} +#endif + +// A native thread enters/leaves an epilogue that outlives its Ractor: from +// the increment until the decrement, ruby_vm_destruct waits for it below. +// The increment must happen while the VM still counts the thread's Ractor, +// so that the two never look absent at the same time. +void +rb_thread_sched_winding_begin(rb_vm_t *vm) +{ + RUBY_ATOMIC_INC(vm->ractor.sched.winding_cnt); +} + +void +rb_thread_sched_winding_end(rb_vm_t *vm) +{ + VM_ASSERT(RUBY_ATOMIC_LOAD(vm->ractor.sched.winding_cnt) > 0); + RUBY_ATOMIC_DEC(vm->ractor.sched.winding_cnt); +} + +// ruby_vm_destruct: wait until no native thread is between a coroutine +// epilogue and its reclaim -- past that point the reclaim frees through the +// (about to be destroyed) objspace and reads the (about to be unset) VM. +// Runs without the VM lock, which the epilogue needs to progress. +void +rb_thread_sched_wait_winding(rb_vm_t *vm) +{ + while (RUBY_ATOMIC_LOAD(vm->ractor.sched.winding_cnt) > 0) { + native_thread_yield(); + } +} + +// A direct service of a runnable thread (direct transfer or the hot-thread +// steal) bypasses the grq; cancel the Ractor's outstanding entry so that +// "enqueued <=> runnable and unserved" keeps holding. The caller holds the +// per-Ractor sched lock, so no concurrent enqueue can relink the node: a +// self-linked read needs no lock (the common case -- direct switches whose +// transition never enqueued). A linked read can race only with a dequeue, +// hence the recheck under the grq lock. +static void +ractor_sched_cancel_enq(rb_vm_t *vm, struct rb_thread_sched *sched) +{ + if (sched->grq_node.next != &sched->grq_node) { + ractor_sched_lock(vm, NULL); + { + if (sched->grq_node.next != &sched->grq_node) { + ccan_list_del_init(&sched->grq_node); + VM_ASSERT(vm->ractor.sched.grq_cnt > 0); + vm->ractor.sched.grq_cnt--; + } + } + ractor_sched_unlock(vm, NULL); + } +} + +static void +ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r) +{ + struct rb_thread_sched *sched = &r->threads.sched; + rb_ractor_t *cr = NULL; // timer thread can call this function + + VM_ASSERT(sched->running != NULL); + VM_ASSERT(sched->running->nt == NULL); + + ractor_sched_lock(vm, cr); + { + // Precondition: not already enqueued (the grq_node is self-linked). + // This holds because every service of a runnable-but-unserved thread + // either dequeues the entry (the nt scheduling loop) or cancels it + // (direct transfers / the hot-thread steal; see + // ractor_sched_cancel_enq) -- re-adding a linked node would corrupt + // the queue, so check unconditionally (a CHECK-mode-only assert + // would miss it: the race needs timing that CHECK builds perturb). + if (sched->grq_node.next != &sched->grq_node) { + rb_bug("ractor_sched_enq: already enqueued"); + } + ccan_list_add_tail(&vm->ractor.sched.grq, &sched->grq_node); + vm->ractor.sched.grq_cnt++; + VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt); + + RUBY_DEBUG_LOG("r:%u th:%u grq_cnt:%u", rb_ractor_id(r), rb_th_serial(sched->running), vm->ractor.sched.grq_cnt); + + rb_native_cond_signal(&vm->ractor.sched.cond); + + // The signal reaches a parked snt, and a running one revisits the + // queue in ractor_sched_deq before it can wait (same lock as here). + // With every snt dedicated or retired, only the timer thread's + // timeout branch can serve the entry or widen the pool: wake it + // (a no-op unless it sleeps untimed). + if (RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt) == 0) { + timer_thread_wakeup_locked(vm); + } + + // ractor_sched_dump(vm); + } + ractor_sched_unlock(vm, cr); +} + + +#ifndef MINIMUM_SNT +// make at least MINIMUM_SNT snts for debug. +#define MINIMUM_SNT 0 +#endif + +/* A shared thread woken with nothing to run is one whose turn another thread + * took first. After this many in a row it gives itself back: the queue keeps + * running dry, so the pool is wider than the work. 0 retires on the first one + * and is too eager to be useful; a negative value keeps every thread. */ +#ifndef SNT_IDLE_RETIRE +#define SNT_IDLE_RETIRE 3 +#endif + +/* Never give the last shared thread back. With none left an enqueue has nobody + * to signal, and the only code that makes one runs on the timer thread's + * timeout branch, which is reached only once it has seen a backlog. */ +#define SNT_KEEP_MINIMUM (MINIMUM_SNT > 1 ? MINIMUM_SNT : 1) + +static rb_ractor_t * +ractor_sched_deq(rb_vm_t *vm, rb_ractor_t *cr) +{ + rb_ractor_t *r; + int idle_streak = 0; // consecutive pops that found the queue empty + + ractor_sched_lock(vm, cr); + { + RUBY_DEBUG_LOG("empty? %d", ccan_list_empty(&vm->ractor.sched.grq)); + // ractor_sched_dump(vm); + + VM_ASSERT(rb_current_execution_context(false) == NULL); + VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt); + + while ((r = ccan_list_pop(&vm->ractor.sched.grq, rb_ractor_t, threads.sched.grq_node)) == NULL) { + RUBY_DEBUG_LOG("wait grq_cnt:%d", (int)vm->ractor.sched.grq_cnt); + + if (SNT_IDLE_RETIRE >= 0 && ++idle_streak > SNT_IDLE_RETIRE && + (int)RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt) > SNT_KEEP_MINIMUM) { + RUBY_ATOMIC_DEC(vm->ractor.sched.snt_cnt); + RUBY_DEBUG_LOG("retire, snt_cnt:%d", (int)vm->ractor.sched.snt_cnt); + break; // returning NULL ends this nt; see the caller + } + + ractor_sched_set_unlocked(vm, cr); + rb_native_cond_wait(&vm->ractor.sched.cond, &vm->ractor.sched.lock); + ractor_sched_set_locked(vm, cr); + + RUBY_DEBUG_LOG("wakeup grq_cnt:%d", (int)vm->ractor.sched.grq_cnt); + } + + VM_ASSERT(rb_current_execution_context(false) == NULL); + + if (r) { + ccan_list_node_init(&r->threads.sched.grq_node); // back to self-linked + VM_ASSERT(vm->ractor.sched.grq_cnt > 0); + vm->ractor.sched.grq_cnt--; + RUBY_DEBUG_LOG("r:%d grq_cnt:%u", (int)rb_ractor_id(r), vm->ractor.sched.grq_cnt); + } + else { + // the retire branch is the only way out of the loop without a ractor + VM_ASSERT(idle_streak > SNT_IDLE_RETIRE); + } + } + ractor_sched_unlock(vm, cr); + + return r; +} + +void rb_ractor_lock_self(rb_ractor_t *r); +void rb_ractor_unlock_self(rb_ractor_t *r); + +// The current thread for a ractor is put to "sleep" (descheduled in the STOPPED_FOREVER state) waiting for +// a ractor action to wake it up. +void +rb_ractor_sched_wait(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf, void *ubf_arg) +{ + // ractor lock of cr is acquired + + RUBY_DEBUG_LOG("start%s", ""); + + rb_thread_t * volatile th = rb_ec_thread_ptr(ec); + struct rb_thread_sched *sched = TH_SCHED(th); + struct ractor_waiter *waiter = (struct ractor_waiter*)ubf_arg; + + if (ubf_set(th, ubf, ubf_arg, &waiter->event_serial)) { + // interrupted + return; + } + + thread_sched_lock(sched, th); + rb_ractor_unlock_self(cr); + { + // A dedicated native thread takes the deadline on the very condvar a wakeup + // signals. An M:N thread has no condvar of its own, so its deadline goes to + // the timer thread, which then wakes it the way rb_ractor_sched_wakeup() does. + bool dedicated = th_has_dedicated_nt(th); + const rb_hrtime_t *end_p = NULL; + bool armed = false, expired = false; + + if (waiter->end) { + if (dedicated) { + end_p = waiter->end; + } + else { + // the timer wheel takes a relative timeout + rb_hrtime_t now = rb_hrtime_now(); + rb_hrtime_t rel = *waiter->end > now ? *waiter->end - now : 0; + + armed = ractor_sched_timeout_arm(th, &rel); + expired = !armed; + } + } + + if (expired) { + RUBY_DEBUG_LOG("expired before sleep%s", ""); + } + else if (armed && th->sched.waiting_reason.flags == thread_sched_waiting_none) { + // the timer thread already took this thread out of the wheel; bump the + // serial so that it does not try to wake a thread that never slept + th->sched.event_serial++; + } + else { + // setup sleep + bool can_direct_transfer = !dedicated; + RB_VM_SAVE_MACHINE_CONTEXT(th); + th->status = THREAD_STOPPED_FOREVER; + th->sched.waiting_timed = (end_p != NULL); // never true here for M:N (end_p is NULL) + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); + thread_sched_wakeup_next_thread(sched, th, can_direct_transfer); + // sleep + thread_sched_wait_running_turn(sched, th, can_direct_transfer, end_p); + th->sched.waiting_timed = false; + th->status = THREAD_RUNNABLE; + + // whoever woke this thread took the timeout back first + VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none); + } + } + thread_sched_unlock(sched, th); + rb_ractor_lock_self(cr); + + ubf_clear(th, true); + + RUBY_DEBUG_LOG("end%s", ""); +} + +void +rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *r_th) +{ + // ractor lock of r acquired + struct rb_thread_sched *sched = TH_SCHED(r_th); + + RUBY_DEBUG_LOG("r:%u th:%d", (unsigned int)rb_ractor_id(r), r_th->serial); + + thread_sched_lock(sched, r_th); + { + if (r_th->status == THREAD_STOPPED_FOREVER) { + RUBY_ATOMIC_ADD(r_th->unblock.event_serial, 1); + + // r_th must not resume with a wheel entry left behind: take its timeout + // back, as ubf_event_waiting() does. Only r_th arms it, and it is + // parked here, so reading the flags without the timer lock is safe. + if (r_th->sched.waiting_reason.flags != thread_sched_waiting_none) { + ractor_sched_timeout_disarm(r_th); + } + + // a timeout that fired first may have made r_th runnable already: waking + // it twice would put it on the readyq twice + if (sched->running != r_th && !r_th->sched.node.is_ready) { + r_th->sched.event_serial++; // a timeout still armed must not wake it again + thread_sched_to_ready_common(sched, r_th, true, false); + } + } + } + thread_sched_unlock(sched, r_th); +} + +static bool +ractor_sched_barrier_completed_p(rb_vm_t *vm) +{ + // The snapshot barrier_running_cnt is taken by the barrier's walk and + // decremented by counted deregistrations; no rescan is needed here. + RUBY_DEBUG_LOG("run:%u wait:%u", vm->ractor.sched.barrier_running_cnt, vm->ractor.sched.barrier_joined_cnt); + VM_ASSERT(vm->ractor.sched.barrier_running_cnt - 1 >= vm->ractor.sched.barrier_joined_cnt); + + return (vm->ractor.sched.barrier_running_cnt - vm->ractor.sched.barrier_joined_cnt) == 1; +} + +void +rb_ractor_sched_barrier_start(rb_vm_t *vm, rb_ractor_t *cr) +{ + VM_ASSERT(cr == GET_RACTOR()); + VM_ASSERT(vm->ractor.sync.lock_owner == cr); // VM is locked + VM_ASSERT(!vm->ractor.sched.barrier_is_waiting); + VM_ASSERT(vm->ractor.sched.barrier_joined_cnt == 0); + VM_ASSERT(vm->ractor.sched.barrier_ractor == NULL); + VM_ASSERT(vm->ractor.sched.barrier_lock_rec == 0); + + RUBY_DEBUG_LOG("start serial:%u", vm->ractor.sched.barrier_serial); + + unsigned int lock_rec; + + ractor_sched_lock(vm, cr); + { + RUBY_ATOMIC_SET(vm->ractor.sched.barrier_is_waiting, 1); + vm->ractor.sched.barrier_ractor = cr; + vm->ractor.sched.barrier_lock_rec = vm->ractor.sync.lock_rec; + + // release VM lock + lock_rec = vm->ractor.sync.lock_rec; + vm->ractor.sync.lock_rec = 0; + vm->ractor.sync.lock_owner = NULL; + rb_native_mutex_unlock(&vm->ractor.sync.lock); + + // Interrupt all running threads: running_dnts plus each snt's running_th. + // A switch before this scan is visible to it; one after it sees + // barrier_is_waiting (set above) and waits. + // Interrupt and count every registered runner, stamping each nt so a + // deregistration during this barrier knows it was counted. + rb_thread_t *ith; + unsigned int running_cnt = 0; + uint32_t serial = vm->ractor.sched.barrier_serial; + + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + struct rb_native_thread *dnt; + ccan_list_for_each(&vm->ractor.sched.ntlist.running_dnts, dnt, running_dnts_node) { + ith = dnt->running_thread; + dnt->barrier_counted_serial = serial; + running_cnt++; + if (ith->ractor != cr) { + RUBY_DEBUG_LOG("barrier request to th:%u", rb_th_serial(ith)); + RUBY_VM_SET_VM_BARRIER_INTERRUPT(ith->ec); + } + } + + struct rb_native_thread *nt; + ccan_list_for_each(&vm->ractor.sched.ntlist.snts, nt, snts_node) { + rb_native_mutex_lock(&nt->running_th_lock); + { + ith = nt->running_th; + if (ith != NULL) { + nt->barrier_counted_serial = serial; + running_cnt++; + if (ith->ractor != cr) { + RUBY_DEBUG_LOG("barrier request to th:%u", rb_th_serial(ith)); + RUBY_VM_SET_VM_BARRIER_INTERRUPT(ith->ec); + } + } + } + rb_native_mutex_unlock(&nt->running_th_lock); + } + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); + + vm->ractor.sched.barrier_running_cnt = running_cnt; + + // wait for other ractors + while (!ractor_sched_barrier_completed_p(vm)) { + ractor_sched_set_unlocked(vm, cr); + rb_native_cond_wait(&vm->ractor.sched.barrier_complete_cond, &vm->ractor.sched.lock); + ractor_sched_set_locked(vm, cr); + } + + RUBY_DEBUG_LOG("completed seirial:%u", vm->ractor.sched.barrier_serial); + + // no other ractors are there + vm->ractor.sched.barrier_serial++; + vm->ractor.sched.barrier_joined_cnt = 0; + rb_native_cond_broadcast(&vm->ractor.sched.barrier_release_cond); + + // acquire VM lock + rb_native_mutex_lock(&vm->ractor.sync.lock); + vm->ractor.sync.lock_rec = lock_rec; + vm->ractor.sync.lock_owner = cr; + } + + // do not release ractor_sched_lock and there is no newly added (resumed) thread + // thread_sched_setup_running_threads +} + +// called from vm_lock_leave if the vm_lock used for barrierred +void +rb_ractor_sched_barrier_end(rb_vm_t *vm, rb_ractor_t *cr) +{ + RUBY_DEBUG_LOG("serial:%u", (unsigned int)vm->ractor.sched.barrier_serial - 1); + VM_ASSERT(vm->ractor.sched.barrier_is_waiting); + VM_ASSERT(vm->ractor.sched.barrier_ractor); + VM_ASSERT(vm->ractor.sched.barrier_lock_rec > 0); + + RUBY_ATOMIC_SET(vm->ractor.sched.barrier_is_waiting, 0); + vm->ractor.sched.barrier_ractor = NULL; + vm->ractor.sched.barrier_lock_rec = 0; + ractor_sched_unlock(vm, cr); +} + +static void +ractor_sched_barrier_join_signal_locked(rb_vm_t *vm) +{ + if (ractor_sched_barrier_completed_p(vm)) { + rb_native_cond_signal(&vm->ractor.sched.barrier_complete_cond); + } +} + +static void +ractor_sched_barrier_join_wait_locked(rb_vm_t *vm, rb_thread_t *th) +{ + VM_ASSERT(vm->ractor.sched.barrier_is_waiting); + + unsigned int barrier_serial = vm->ractor.sched.barrier_serial; + + while (vm->ractor.sched.barrier_serial == barrier_serial) { + RUBY_DEBUG_LOG("sleep serial:%u", barrier_serial); + RB_VM_SAVE_MACHINE_CONTEXT(th); + + rb_ractor_t *cr = th->ractor; + ractor_sched_set_unlocked(vm, cr); + rb_native_cond_wait(&vm->ractor.sched.barrier_release_cond, &vm->ractor.sched.lock); + ractor_sched_set_locked(vm, cr); + + RUBY_DEBUG_LOG("wakeup serial:%u", barrier_serial); + } +} + +void +rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr) +{ + VM_ASSERT(cr->threads.sched.running != NULL); // running ractor + VM_ASSERT(cr == GET_RACTOR()); + VM_ASSERT(vm->ractor.sync.lock_owner == NULL); // VM is locked, but owner == NULL + VM_ASSERT(vm->ractor.sched.barrier_is_waiting); // VM needs barrier sync + +#if USE_RUBY_DEBUG_LOG || VM_CHECK_MODE > 0 + unsigned int barrier_serial = vm->ractor.sched.barrier_serial; +#endif + + RUBY_DEBUG_LOG("join"); + + rb_native_mutex_unlock(&vm->ractor.sync.lock); + { + VM_ASSERT(vm->ractor.sched.barrier_is_waiting); // VM needs barrier sync + VM_ASSERT(vm->ractor.sched.barrier_serial == barrier_serial); + + ractor_sched_lock(vm, cr); + { + // running_cnt + /* Every joiner is a member of the running set: a dying thread + * leaves the living set before handing over its scheduler slot. */ + vm->ractor.sched.barrier_joined_cnt++; + RUBY_DEBUG_LOG("waiting_cnt:%u serial:%u", vm->ractor.sched.barrier_joined_cnt, barrier_serial); + + ractor_sched_barrier_join_signal_locked(vm); + ractor_sched_barrier_join_wait_locked(vm, cr->threads.sched.running); + } + ractor_sched_unlock(vm, cr); + } + + rb_native_mutex_lock(&vm->ractor.sync.lock); + // VM locked here +} + +// Called when the ractor holding this sched is freed. A drained sched can +// still be on timeslice.scheds (pruning is lazy); an unlisted node is +// self-linked (fork re-inits them all), making this del a no-op. +void +rb_thread_sched_destroy(struct rb_thread_sched *sched) +{ + rb_vm_t *vm = GET_VM(); + + rb_native_mutex_lock(&vm->ractor.sched.timeslice.lock); + { + ccan_list_del_init(&sched->timeslice_node); + } + rb_native_mutex_unlock(&vm->ractor.sched.timeslice.lock); +} + +#if defined(HAVE_WORKING_FORK) +static void rb_internal_thread_event_hooks_rw_lock_atfork(void); + +static void +thread_sched_atfork(struct rb_thread_sched *sched) +{ + current_fork_gen++; + rb_thread_sched_init(sched, true); + rb_thread_t *th = GET_THREAD(); + rb_vm_t *vm = GET_VM(); + + if (th_has_dedicated_nt(th)) { + vm->ractor.sched.snt_cnt = 0; +#if USE_RUBY_DEBUG_LOG + vm->ractor.sched.dnt_cnt = 1; +#endif + } + else { + vm->ractor.sched.snt_cnt = 1; +#if USE_RUBY_DEBUG_LOG + vm->ractor.sched.dnt_cnt = 0; +#endif + } + + rb_native_mutex_initialize(&vm->ractor.sched.lock); +#if VM_CHECK_MODE > 0 + vm->ractor.sched.lock_owner = NULL; + vm->ractor.sched.locked = false; +#endif + + // rb_native_cond_destroy(&vm->ractor.sched.cond); + rb_native_cond_initialize(&vm->ractor.sched.cond); + rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond); + rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond); + + ccan_list_head_init(&vm->ractor.sched.grq); + vm->ractor.sched.grq_cnt = 0; // the list was just emptied; reset the count with it + // A fork during a VM barrier leaves the child with barrier state that can + // never complete (the other ractors are gone); reset it like the rest. + vm->ractor.sched.barrier_is_waiting = 0; // single-threaded child + vm->ractor.sched.barrier_joined_cnt = 0; + vm->ractor.sched.barrier_ractor = NULL; + vm->ractor.sched.barrier_lock_rec = 0; + // Threads that were winding down in the parent do not exist in the child; + // without this reset the child's ruby_vm_destruct would wait for their + // reclaim (which never comes) forever. + vm->ractor.sched.winding_cnt = 0; + rb_native_mutex_initialize(&vm->ractor.sched.ntlist.lock); + ccan_list_head_init(&vm->ractor.sched.ntlist.running_dnts); + ccan_list_head_init(&vm->ractor.sched.ntlist.snts); // those nts are gone + rb_native_mutex_initialize(&vm->ractor.sched.timeslice.lock); + ccan_list_head_init(&vm->ractor.sched.timeslice.scheds); + rb_native_mutex_initialize(&th->nt->running_th_lock); // a scan could hold it at fork + // Fork can copy nodes linked (or torn mid-link); re-init every sched's + // node so rb_thread_sched_destroy's del_init stays a no-op for them. + rb_ractor_t *r; + ccan_list_for_each(&vm->ractor.set, r, vmlr_node) { + ccan_list_node_init(&r->threads.sched.timeslice_node); + } + ccan_list_for_each(&vm->ractor.terminated_set, r, vmlr_node) { + ccan_list_node_init(&r->threads.sched.timeslice_node); + } + // th re-records itself below; the parent's record did not survive the lists + if (th->nt && th->nt->dedicated == 0) { + // surviving on an snt: put that nt back on the (just emptied) snts + // list, or the scans could not see this thread's record + ccan_list_add(&vm->ractor.sched.ntlist.snts, &th->nt->snts_node); + } + +#if USE_MN_THREADS + nt_machine_stack_atfork(); +#endif + rb_internal_thread_event_hooks_rw_lock_atfork(); + + VM_ASSERT(sched->is_running); + + if (sched->running != th) { + thread_sched_to_running(sched, th); + } + else { + thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL); + } + +#ifdef RB_THREAD_T_HAS_NATIVE_ID + if (th->nt) { + th->nt->tid = get_native_thread_id(); + } +#endif +} + +#endif + +extern int ruby_mn_threads_enabled; + +void +ruby_mn_threads_params(void) +{ + rb_vm_t *vm = GET_VM(); + rb_ractor_t *main_ractor = GET_RACTOR(); + + const char *mn_threads_cstr = getenv("RUBY_MN_THREADS"); + bool enable_mn_threads = false; + + if (USE_MN_THREADS && mn_threads_cstr && (enable_mn_threads = atoi(mn_threads_cstr) > 0)) { + // enabled + ruby_mn_threads_enabled = 1; + } + main_ractor->threads.sched.enable_mn_threads = enable_mn_threads; + + const char *max_cpu_cstr = getenv("RUBY_MAX_CPU"); + int max_cpu = native_thread_default_max_cpu(); + + if (USE_MN_THREADS && max_cpu_cstr) { + int given_max_cpu = atoi(max_cpu_cstr); + if (given_max_cpu > 0) { + max_cpu = given_max_cpu; + } + } + + vm->ractor.sched.max_cpu = max_cpu; +} + +static void +native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt) +{ + RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated + 1); + + if (nt->dedicated == 0) { + // Lock-free; pairs with ractor_sched_enq (enq: grq_cnt up then read + // snt_cnt / here: snt_cnt down then read grq_cnt) against lost wakeups. + if (RUBY_ATOMIC_FETCH_SUB(vm->ractor.sched.snt_cnt, 1) == 1) { + // the last snt went dedicated; pending entries need the timer thread + ractor_sched_lock(vm, cr); + { + if (vm->ractor.sched.grq_cnt > 0) { + timer_thread_wakeup_locked(vm); + } + } + ractor_sched_unlock(vm, cr); + } +#if USE_RUBY_DEBUG_LOG + vm->ractor.sched.dnt_cnt++; +#endif + } + + nt->dedicated++; +} + +static void +native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt) +{ + RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated - 1); + VM_ASSERT(nt->dedicated > 0); + nt->dedicated--; + + if (nt->dedicated == 0) { + // Rejoin under the max_cpu cap; with no room this nt retires and + // belongs to neither count until it ends. + while (1) { + rb_atomic_t snt = RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt); + if (snt < vm->ractor.sched.max_cpu || (int)snt <= MINIMUM_SNT) { + if (RUBY_ATOMIC_CAS(vm->ractor.sched.snt_cnt, snt, snt + 1) == snt) break; + } + else { + nt->retiring = true; + break; + } + } +#if USE_RUBY_DEBUG_LOG + vm->ractor.sched.dnt_cnt--; +#endif + } +} + +static void +native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th) +{ +#if USE_RUBY_DEBUG_LOG + if (nt) { + if (th->nt) { + RUBY_DEBUG_LOG("th:%d nt:%d->%d", (int)th->serial, (int)th->nt->serial, (int)nt->serial); + } + else { + RUBY_DEBUG_LOG("th:%d nt:NULL->%d", (int)th->serial, (int)nt->serial); + } + } + else { + if (th->nt) { + RUBY_DEBUG_LOG("th:%d nt:%d->NULL", (int)th->serial, (int)th->nt->serial); + } + else { + RUBY_DEBUG_LOG("th:%d nt:NULL->NULL", (int)th->serial); + } + } +#endif + + th->nt = nt; +} + +static int +native_thread_create_dedicated(rb_thread_t *th) +{ + th->nt = native_thread_alloc(); + th->nt->vm = th->vm; + th->nt->running_thread = th; + th->nt->dedicated = 1; + + // vm stack + size_t vm_stack_word_size = th->vm->default_params.thread_vm_stack_size / sizeof(VALUE); + void *vm_stack = ruby_xmalloc(vm_stack_word_size * sizeof(VALUE)); + th->sched.malloc_stack = true; + rb_ec_initialize_vm_stack(th->ec, vm_stack, vm_stack_word_size); + th->sched.context_stack = vm_stack; + th->sched.context_stack_size = vm_stack_word_size; + + int err = native_thread_create0(th->nt); + if (!err) { + // setup + thread_sched_to_ready(TH_SCHED(th), th); + } + return err; +} + +static void +call_thread_start_func_2(rb_thread_t *th) +{ + /* Capture the address of a local in this stack frame to mark the beginning of the + machine stack for this thread. This is required even if we can tell the real + stack beginning from the pthread API in native_thread_init_stack, because + glibc stores some of its own data on the stack before calling into user code + on a new thread, and replacing that data on fiber-switch would break it (see + bug #13887) */ + VALUE stack_start = 0; + VALUE *stack_start_addr = asan_get_real_stack_addr(&stack_start); + + native_thread_init_stack(th, stack_start_addr); + thread_start_func_2(th, th->ec->machine.stack_start); +} + +static void * +nt_start(void *ptr) +{ + struct rb_native_thread *nt = (struct rb_native_thread *)ptr; + rb_vm_t *vm = nt->vm; + + native_thread_setup_on_thread(nt); + + // init tid +#ifdef RB_THREAD_T_HAS_NATIVE_ID + nt->tid = get_native_thread_id(); +#endif + +#if USE_RUBY_DEBUG_LOG && defined(RUBY_NT_SERIAL) + ruby_nt_serial = nt->serial; +#endif + + RUBY_DEBUG_LOG("nt:%u", nt->serial); + + bool in_snts = false; + + if (!nt->dedicated) { + coroutine_initialize_main(nt->nt_context); + + // join the snt list that the barrier/timeslice scans walk + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + ccan_list_add(&vm->ractor.sched.ntlist.snts, &nt->snts_node); + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); + in_snts = true; + } + + bool retired = false; + + while (1) { + if (nt->dedicated) { + // wait running turn + rb_thread_t *th = nt->running_thread; + struct rb_thread_sched *sched = TH_SCHED(th); + + RUBY_DEBUG_LOG("on dedicated th:%u", rb_th_serial(th)); + ruby_thread_set_native(th); + + thread_sched_lock(sched, th); + { + if (sched->running == th) { + thread_sched_add_running_thread(sched, th); + } + thread_sched_wait_running_turn(sched, th, false, NULL); + } + thread_sched_unlock(sched, th); + + // start threads + call_thread_start_func_2(th); + break; // TODO: allow to change to the SNT + } + else { + RUBY_DEBUG_LOG("check next"); + if (nt->retiring) { // came back with no room in the shared pool + retired = true; + break; + } + + rb_ractor_t *r = ractor_sched_deq(vm, NULL); + + if (r) { + struct rb_thread_sched *sched = &r->threads.sched; + + bool locked = true; + + thread_sched_lock(sched, NULL); + { + rb_thread_t *next_th = sched->running; + + if (next_th && next_th->nt == NULL) { + RUBY_DEBUG_LOG("nt:%d next_th:%d", (int)nt->serial, (int)next_th->serial); +#if USE_MN_THREADS + thread_sched_switch0(nt->nt_context, next_th, nt, false); + + // If a coroutine terminated during the transfer, co_start + // recorded it in nt->dead_co (switch0's return value is + // backend-dependent, unusable; see thread_pthread.h). + struct coroutine_context *dead_co = nt->dead_co; + nt->dead_co = NULL; + if (thread_sched_reclaim(dead_co)) { + // it already released the sched lock before its + // transfer (its Ractor may be gone): leave sched be. + locked = false; + } +#else + thread_sched_switch0(nt->nt_context, next_th, nt, false); +#endif + } + else { + RUBY_DEBUG_LOG("no schedulable threads -- next_th:%p", next_th); + } + } + if (locked) { + thread_sched_unlock(sched, NULL); + } + } + else { + // ractor_sched_deq retired this nt. + retired = true; + break; + } + + if (nt->dedicated) { + // SNT becomes DNT while running + break; + } + } + } + + if (in_snts) { + // Leaving the shared loop: every path back here deregistered first + // (park and death both precede the transfer), so only the snts entry + // is left to remove. + VM_ASSERT(nt->running_th == NULL); + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + ccan_list_del_init(&nt->snts_node); + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); + } + + if (retired) { + // The counts dropped this nt already; nothing can reference it now. + RUBY_DEBUG_LOG("retired nt:%u", nt->serial); + native_thread_destroy_self(nt); + } + + return NULL; +} + +static int native_thread_create_shared(rb_thread_t *th); + +#if USE_MN_THREADS +static void nt_free_stack(void *mstack); + + +// Reclaim the context a coroutine thread recorded in nt->dead_co before its +// final transfer (co_start's epilogue). Our running here proves that transfer's +// register save into the block completed. Returns true when a thread did +// terminate -- it RELEASED the sched lock before transferring; NULL/false means +// a live yield, where the loop still owns the lock. +static bool +thread_sched_reclaim(struct coroutine_context *dead_co) +{ + struct rb_thread_context *tctx = (struct rb_thread_context *)dead_co; + + if (tctx != NULL && tctx->dead) { + nt_free_stack(tctx->stack); + SIZED_FREE(tctx); + // pairs with the increment at the top of coroutine_thread_terminated: + // a waiting VM destruct may proceed once this reclaim is done + VM_ASSERT(RUBY_ATOMIC_LOAD(GET_VM()->ractor.sched.winding_cnt) > 0); + RUBY_ATOMIC_DEC(GET_VM()->ractor.sched.winding_cnt); + return true; + } + return false; +} +#endif + +void +rb_thread_wake_fence(rb_thread_t *th) +{ + timer_thread_wake_fence(th); +} + +void +rb_threadptr_sched_free(rb_thread_t *th) +{ + timer_thread_wake_fence(th); +#if USE_MN_THREADS + if (th->sched.malloc_stack) { + // has dedicated + SIZED_FREE_N((VALUE *)th->sched.context_stack, th->sched.context_stack_size); + native_thread_destroy(th->nt); + } + else if (th->sched.context != NULL) { + // a coroutine thread that never reached its epilogue (never started); + // a terminated one is reclaimed by whoever resumed from its final + // transfer (thread_sched_reclaim), and cleared this pointer. + struct rb_thread_context *tctx = (struct rb_thread_context *)th->sched.context; + nt_free_stack(tctx->stack); + SIZED_FREE(tctx); + th->sched.context = NULL; + // TODO: how to free nt and nt->altstack? + } +#else + SIZED_FREE_N((VALUE *)th->sched.context_stack, th->sched.context_stack_size); + native_thread_destroy(th->nt); +#endif + + th->nt = NULL; +} + + +static int +native_thread_create(rb_thread_t *th) +{ + VM_ASSERT(th->nt == 0); + RUBY_DEBUG_LOG("th:%d has_dnt:%d", th->serial, th->has_dedicated_nt); + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_STARTED, th); + + if (!th->ractor->threads.sched.enable_mn_threads) { + th->has_dedicated_nt = 1; + } + + if (th->has_dedicated_nt) { + return native_thread_create_dedicated(th); + } + else { + return native_thread_create_shared(th); + } +} + +#ifdef USE_UBF_LIST +static CCAN_LIST_HEAD(ubf_list_head); +#ifdef RB_NATIVETHREAD_LOCK_INIT +static rb_nativethread_lock_t ubf_list_lock = RB_NATIVETHREAD_LOCK_INIT; +#else +// no static initializer on this platform; thread_sched_init_vm() does it +static rb_nativethread_lock_t ubf_list_lock; +#endif + +static void +ubf_list_atfork(void) +{ + ccan_list_head_init(&ubf_list_head); + rb_native_mutex_initialize(&ubf_list_lock); +} + +RBIMPL_ATTR_MAYBE_UNUSED() +static bool +ubf_list_contain_p(rb_thread_t *th) +{ + rb_thread_t *list_th; + ccan_list_for_each(&ubf_list_head, list_th, sched.node.ubf) { + if (list_th == th) return true; + } + return false; +} + +/* The thread 'th' is registered to be trying unblock. */ +static void +register_ubf_list(rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); + struct ccan_list_node *node = &th->sched.node.ubf; + + VM_ASSERT(th->unblock.func != NULL); + + rb_native_mutex_lock(&ubf_list_lock); + { + // check not connected yet + if (ccan_list_empty((struct ccan_list_head*)node)) { + VM_ASSERT(!ubf_list_contain_p(th)); + ccan_list_add(&ubf_list_head, node); + } + } + rb_native_mutex_unlock(&ubf_list_lock); + + timer_thread_wakeup(); +} + +/* The thread 'th' is unblocked. It no longer need to be registered. */ +static void +unregister_ubf_list(rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u", rb_th_serial(th)); + struct ccan_list_node *node = &th->sched.node.ubf; + + /* we can't allow re-entry into ubf_list_head */ + VM_ASSERT(th->unblock.func == NULL); + + if (!ccan_list_empty((struct ccan_list_head*)node)) { + rb_native_mutex_lock(&ubf_list_lock); + { + VM_ASSERT(ubf_list_contain_p(th)); + ccan_list_del_init(node); + } + rb_native_mutex_unlock(&ubf_list_lock); + } +} + +/* + * Poke the target thread so that it returns from a blocking syscall. + * How that is done is up to the platform (native_thread_interrupt). + */ +static void +ubf_wakeup_thread(rb_thread_t *th) +{ + RUBY_DEBUG_LOG("th:%u thread_id:%p", rb_th_serial(th), (void *)th->nt->thread_id); + + native_thread_interrupt(th); +} + +static void +ubf_select(void *ptr) +{ + rb_thread_t *th = (rb_thread_t *)ptr; + RUBY_DEBUG_LOG("wakeup th:%u", rb_th_serial(th)); + ubf_wakeup_thread(th); + register_ubf_list(th); +} + +static bool +ubf_threads_empty(void) +{ + return ccan_list_empty(&ubf_list_head) != 0; +} + +static void +ubf_wakeup_all_threads(void) +{ + rb_thread_t *th; + rb_native_mutex_lock(&ubf_list_lock); + { + ccan_list_for_each(&ubf_list_head, th, sched.node.ubf) { + ubf_wakeup_thread(th); + } + } + rb_native_mutex_unlock(&ubf_list_lock); +} + +#else /* USE_UBF_LIST */ +#define register_ubf_list(th) (void)(th) +#define unregister_ubf_list(th) (void)(th) +#define ubf_select 0 +static void ubf_wakeup_all_threads(void) { return; } +static bool ubf_threads_empty(void) { return true; } +#define ubf_list_atfork() do {} while (0) +#endif /* USE_UBF_LIST */ + +static int +timer_thread_set_timeout(rb_vm_t *vm) +{ +#if 0 + return 10; // ms +#else + int timeout = -1; + + ractor_sched_lock(vm, NULL); + { + if ( timeslice_scan(vm, false) // (1-1) Provide time slice for active NTs + || !ubf_threads_empty() // (1-3) Periodic UBF + || vm->ractor.sched.grq_cnt > 0 // (1-4) Lazy GRQ deq start + ) { + + RUBY_DEBUG_LOG("ubf:%d grq:%d", + !ubf_threads_empty(), + (vm->ractor.sched.grq_cnt > 0)); + + timeout = 10; // ms + vm->ractor.sched.timeslice_wait_inf = false; + } + else { + vm->ractor.sched.timeslice_wait_inf = true; + } + } + ractor_sched_unlock(vm, NULL); + + timeout = timer_wheel_timeout(timeout); + + RUBY_DEBUG_LOG("timeout:%d inf:%d", timeout, (int)vm->ractor.sched.timeslice_wait_inf); + + // fprintf(stderr, "timeout:%d\n", timeout); + return timeout; +#endif +} + +static void +timer_thread_check_signal(rb_vm_t *vm) +{ + // ruby_sigchld_handler(vm); TODO + + int signum = rb_signal_buff_size(); + if (UNLIKELY(signum > 0) && vm->ractor.main_thread) { + RUBY_DEBUG_LOG("signum:%d", signum); + threadptr_trap_interrupt(vm->ractor.main_thread); + } +} + +// Tick (with `interrupt`) each listed sched's running thread and prune scheds +// whose readyq drained; returns whether any sched still needs ticks. +static bool +timeslice_scan(rb_vm_t *vm, bool interrupt) +{ + bool found = false; + struct rb_thread_sched *sched, *next; + + rb_native_mutex_lock(&vm->ractor.sched.timeslice.lock); + { + ccan_list_for_each_safe(&vm->ractor.sched.timeslice.scheds, sched, next, timeslice_node) { + // trylock: timeslice_sched_link nests sched.lock -> timeslice.lock, + // this scan holds the locks the other way around + if (rb_native_mutex_trylock(&sched->lock_) == 0) { + if (ccan_list_empty(&sched->readyq)) { + ccan_list_del_init(&sched->timeslice_node); // a later enq relinks it + } + else if (sched->is_running) { + VM_ASSERT(sched->running != NULL); + found = true; + if (interrupt) { + RUBY_DEBUG_LOG("timeslice th:%u", rb_th_serial(sched->running)); + RUBY_VM_SET_TIMER_INTERRUPT(sched->running->ec); + } + } + // else: waiters behind a blocked runner need no ticks; the + // add path wakes the timer when the sched runs again + rb_native_mutex_unlock(&sched->lock_); + } + else { + found = true; // busy switching; tick it on the next round + } + } + } + rb_native_mutex_unlock(&vm->ractor.sched.timeslice.lock); + + return found; +} + +static void +timer_thread_check_timeslice(rb_vm_t *vm) +{ + // TODO: check time + timeslice_scan(vm, true); +} + +static void * +timer_thread_func(void *ptr) +{ + rb_vm_t *vm = (rb_vm_t *)ptr; +#if defined(RUBY_NT_SERIAL) + ruby_nt_serial = (rb_atomic_t)-1; +#endif + + RUBY_DEBUG_LOG("started%s", ""); + + while (RUBY_ATOMIC_LOAD(system_working)) { + timer_thread_check_signal(vm); + timer_thread_check_timeout(vm); + ubf_wakeup_all_threads(); + + RUBY_DEBUG_LOG("system_working:%d", RUBY_ATOMIC_LOAD(system_working)); + timer_thread_polling(vm); + } + + RUBY_DEBUG_LOG("terminated"); + return NULL; +} + +static void +timer_thread_wakeup_locked(rb_vm_t *vm) +{ + // should be locked before. + ASSERT_ractor_sched_locked(vm, NULL); + + if (TIMER_THREAD_CREATED_P()) { + if (vm->ractor.sched.timeslice_wait_inf) { + RUBY_DEBUG_LOG("wakeup%s", ""); + timer_thread_wakeup_force(); + } + else { + RUBY_DEBUG_LOG("will be wakeup..."); + } + } +} + +static void +timer_thread_wakeup(void) +{ + rb_vm_t *vm = GET_VM(); + + ractor_sched_lock(vm, NULL); + { + timer_thread_wakeup_locked(vm); + } + ractor_sched_unlock(vm, NULL); +} + +static void +native_sleep(rb_thread_t *th, rb_hrtime_t *rel) +{ + struct rb_thread_sched *sched = TH_SCHED(th); + + RUBY_DEBUG_LOG("rel:%d", rel ? (int)*rel : 0); + + if (rel && !th_has_dedicated_nt(th)) { + // an M:N thread has no condvar of its own: the timer thread wakes it + thread_sched_wait_events(sched, th, -1, thread_sched_waiting_timeout, rel); + } + else if (rel) { + /* Solaris cond_timedwait() returns EINVAL if an argument is greater than + * current_time + 100,000,000. So cut up to 100,000,000. This is + * considered as a kind of spurious wakeup. The caller to native_sleep + * should care about spurious wakeup. + * + * See also [Bug #1341] [ruby-core:29702] + * http://download.oracle.com/docs/cd/E19683-01/816-0216/6m6ngupgv/index.html + */ + const rb_hrtime_t max = (rb_hrtime_t)100000000 * RB_HRTIME_PER_SEC; + if (*rel > max) *rel = max; + + rb_hrtime_t end = rb_hrtime_add(rb_hrtime_now(), *rel); + thread_sched_to_waiting_until_wakeup(sched, th, &end); + } + else { + thread_sched_to_waiting_until_wakeup(sched, th, NULL); + } + + RUBY_DEBUG_LOG("wakeup"); +} + + +// return true if the current thread acquires DNT. +// return false if the current thread already acquires DNT. +bool +rb_thread_lock_native_thread(void) +{ + rb_thread_t *th = GET_THREAD(); + bool is_snt = th->nt->dedicated == 0; + native_thread_dedicated_inc(th->vm, th->ractor, th->nt); + + return is_snt; +} + +void +rb_thread_malloc_stack_set(rb_thread_t *th, void *stack, size_t stack_size) +{ + th->sched.malloc_stack = true; + th->sched.context_stack = stack; + th->sched.context_stack_size = stack_size; +} + +// VM wide scheduler state, shared by every platform. Called from +// Init_native_thread() before the main thread is recorded. +static void +thread_sched_init_vm(rb_vm_t *vm) +{ + rb_native_mutex_initialize(&vm->ractor.sched.lock); + rb_native_cond_initialize(&vm->ractor.sched.cond); + rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond); + rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond); + + ccan_list_head_init(&vm->ractor.sched.grq); + rb_native_mutex_initialize(&vm->ractor.sched.ntlist.lock); + ccan_list_head_init(&vm->ractor.sched.ntlist.running_dnts); + ccan_list_head_init(&vm->ractor.sched.ntlist.snts); + rb_native_mutex_initialize(&vm->ractor.sched.timeslice.lock); + ccan_list_head_init(&vm->ractor.sched.timeslice.scheds); + +#ifndef RB_NATIVETHREAD_LOCK_INIT + // ubf_list_lock could not be initialized statically + ubf_list_atfork(); +#endif +} diff --git a/thread_sched.h b/thread_sched.h new file mode 100644 index 00000000000000..0fe5bd951133d3 --- /dev/null +++ b/thread_sched.h @@ -0,0 +1,264 @@ +#ifndef RUBY_THREAD_SCHED_H +#define RUBY_THREAD_SCHED_H +/********************************************************************** + + thread_sched.h - data structures of the thread/ractor scheduler + + The scheduler itself lives in thread_sched.c and is shared by every + platform. This header holds the types it works on; it is included from + the platform header (thread_pthread.h / thread_win32.h), which adds the + platform specific members and the thread local storage plumbing. + + == platform primitive layer == + + thread_sched.c is built on top of the following, which the platform + implementation (THREAD_IMPL_SRC) has to provide before including it: + + * rb_native_mutex_*() / rb_native_cond_*() (ruby/thread_native.h) + * native_thread_create0() / native_thread_destroy() and friends + * native_thread_interrupt() -- poke a thread out of a blocking call + * timer_thread_polling() -- the timer thread's blocking wait + * timer_thread_wakeup_force() -- wake that wait up + * TIMER_THREAD_CREATED_P() + * USE_MN_THREADS -- 1 enables the M:N scheduler + (when 0 the platform supplies the stubs listed at the bottom of + thread_sched_mn.c) + +**********************************************************************/ + +// How a thread_sched_wait_events() wait ended. "unavailable" (could not be +// registered) is not "the event fired": the caller must fall back, not proceed. +enum thread_sched_wait_result { + thread_sched_wait_event, // an event the caller asked for fired + thread_sched_wait_timeout, // the timeout expired before any event + thread_sched_wait_unavailable, // not registered; the caller must fall back +}; + +// this data should be protected by timer_th.waiting_lock +struct rb_thread_sched_waiting { + enum thread_sched_waiting_flag { + thread_sched_waiting_none = 0x00, + thread_sched_waiting_timeout = 0x01, + thread_sched_waiting_io_read = 0x02, + thread_sched_waiting_io_write = 0x08, + thread_sched_waiting_io_force = 0x40, // ignore readable + } flags; + + struct { + // should be compat with hrtime.h +#ifdef MY_RUBY_BUILD_MAY_TIME_TRAVEL + int128_t timeout; +#else + uint64_t timeout; +#endif + uint32_t event_serial; + int fd; // -1 for timeout only + int result; + } data; + + // connected to a timer_th wheel slot (timed) or timer_th.waiting_untimed + struct ccan_list_node node; + + /* which wheel slot `node` is on; meaningful only while flags has + * thread_sched_waiting_timeout */ + uint8_t wheel_lvl; + uint8_t wheel_slot; + + // connected to rb_fd_waiters.waiters of data.fd + struct ccan_list_node fd_node; +}; + +// One entry per fd with waiters; fds stay dense, so a table indexed by fd fits. +// Entries live in fixed chunks: growing must not move a live list head. +struct rb_fd_waiters { + struct ccan_list_head waiters; // rb_thread_sched_waiting.fd_node + + // The io flags currently armed in epoll/kqueue for this fd: the union of + // what its waiters asked for. + uint32_t armed_flags; + + // Bumped on full disarm. Events carry the generation they were armed with, + // so one queued before the fd was disarmed (and reused) is recognised. + uint32_t generation; +}; + +// per-Thread scheduler helper data +struct rb_thread_sched_item { + struct { + struct ccan_list_node ubf; + + // connected to ractor->threads.sched.reqdyq + // locked by ractor->threads.sched.lock + struct ccan_list_node readyq; + // Indicates whether thread is on the readyq. + // There is no clear relationship between this and th->status. + bool is_ready; + + } node; + + struct rb_thread_sched_waiting waiting_reason; + uint32_t event_serial; + + // wakes pending on this thread (timer thread or an fd shard claim); + // under timer_th.wake_pending_lock + uint32_t wake_pending_cnt; + + // parked on its own condvar with a deadline; under the sched lock (see + // ubf_waiting). Always false for an M:N thread: its deadline lives on the + // timer wheel, and its early wake comes from the timer thread instead. + bool waiting_timed; + + bool malloc_stack; + void *context_stack; + size_t context_stack_size; + struct coroutine_context *context; +}; + +struct rb_native_thread { + rb_atomic_t serial; + struct rb_vm_struct *vm; + + rb_nativethread_id_t thread_id; + +#ifdef RB_THREAD_T_HAS_NATIVE_ID + int tid; +#endif + +#if defined(_WIN32) + // signalled by native_thread_interrupt() to break this thread out of a + // blocking w32_wait_events() + HANDLE interrupt_event; +#endif + + struct rb_thread_struct *running_thread; + + // The running thread on this shared nt, for the barrier/timeslice scans. + // While a scan holds running_th_lock the thread cannot finish parking. + rb_nativethread_lock_t running_th_lock; + struct rb_thread_struct *running_th; + struct ccan_list_node snts_node; // in vm->ractor.sched.ntlist.snts + // in vm->ractor.sched.ntlist.running_dnts while running_thread runs + struct ccan_list_node running_dnts_node; + // barrier_serial stamped by the barrier's counting walk; this nt's + // deregistration during that barrier decrements the snapshot count + uint32_t barrier_counted_serial; + + // to control native thread; use sched->lock + rb_nativethread_cond_t readyq; + +#ifdef USE_SIGALTSTACK + void *altstack; +#endif + + struct coroutine_context *nt_context; + int dedicated; + + // set when this thread came back from a blocking region with no room left + // in the shared pool; it ends when it next asks for work + bool retiring; + + // A terminating coroutine records its context here before its final + // transfer; this nt's loop reclaims it. (Not via coroutine_transfer()'s + // return value: its meaning differs between the amd64 asm and ucontext.) + struct coroutine_context *dead_co; +}; + +// defines these as macros, and the field names below (and in the +// rest of the interpreter) would be rewritten by them. +#undef except +#undef try +#undef leave +#undef finally + +// per-Ractor +struct rb_thread_sched { + rb_nativethread_lock_t lock_; +#if VM_CHECK_MODE + struct rb_thread_struct *lock_owner; +#endif + struct rb_thread_struct *running; // running thread or NULL + // Most recently running thread or NULL. If this thread wakes up before the newly running + // thread completes the transfer of control, it can interrupt and resume running. + // The new thread clears this field when it takes control. + struct rb_thread_struct *runnable_hot_th; + int runnable_hot_th_waiting; + bool is_running; + + bool enable_mn_threads; + + struct ccan_list_head readyq; + int readyq_cnt; + // ractor scheduling + // When not linked in vm->ractor.sched.grq, this node is kept + // self-linked (ccan_list_node_init), so "linked?" can be read off the + // node itself: enqueuers assert it, and direct transfers cancel an + // outstanding entry (see ractor_sched_cancel_enq). + struct ccan_list_node grq_node; + struct ccan_list_node timeslice_node; // self-linked = not on timeslice.scheds +}; + +struct rb_thread_context; + +// A coroutine (M:N) thread's teardown runs coroutine_thread_terminated +// instead of the dedicated-thread path in thread_start_func_2; see the +// comments there and in thread_sched_mn.c. th->sched.context is cleared in +// that epilogue, so this also reads as "did not tear down yet". +// (Only meaningful when USE_MN_THREADS -- gate uses accordingly; the macro +// itself is a plain pointer test and always compiles.) +#define th_has_coroutine(th) ((th)->sched.context != NULL) + +struct rb_ractor_struct; + +// VM wide: what schedules Ractors onto native threads. One per VM, in +// rb_vm_struct.ractor.sched. +struct rb_ractor_sched { + rb_nativethread_lock_t lock; + struct rb_ractor_struct *lock_owner; + bool locked; + + rb_nativethread_cond_t cond; // GRQ + rb_atomic_t snt_cnt; // count of shared NTs; lock-free (see native_thread_dedicated_inc) + unsigned int dnt_cnt; // count of dedicated NTs; logging only (USE_RUBY_DEBUG_LOG), not atomic + + unsigned int max_cpu; + struct ccan_list_head grq; // // Global Ready Queue + rb_atomic_t winding_cnt; // native threads between a coroutine epilogue and its reclaim; ruby_vm_destruct waits for 0 + unsigned int grq_cnt; + + // What the barrier walk visits: threads running on dedicated + // nts, and the shared nts (whose running_th fields hold the rest). + struct { + rb_nativethread_lock_t lock; + struct ccan_list_head running_dnts; + struct ccan_list_head snts; + } ntlist; + + // scheds whose readyq holds waiters: the timer ticks their + // running thread (timeslice_scan) and prunes drained entries. + struct { + rb_nativethread_lock_t lock; + struct ccan_list_head scheds; + } timeslice; + + // true if timeslice timer is not enable + bool timeslice_wait_inf; + + // barrier + rb_nativethread_cond_t barrier_complete_cond; + rb_nativethread_cond_t barrier_release_cond; + // bool; nonzero while a stop-the-world section is active. Set + // before the barrier walks the running records; a record moved + // after the walk sees it (thread_sched_setup_running_threads). + rb_atomic_t barrier_is_waiting; + unsigned int barrier_joined_cnt; // threads joined so far; under sched.lock + unsigned int barrier_running_cnt; // runners counted by the barrier's walk; under sched.lock + unsigned int barrier_serial; + struct rb_ractor_struct *barrier_ractor; + unsigned int barrier_lock_rec; +}; + +void rb_ractor_sched_wait(struct rb_execution_context_struct *ec, struct rb_ractor_struct *cr, rb_unblock_function_t *ptr, void *arg); +void rb_ractor_sched_wakeup(struct rb_ractor_struct *r, struct rb_thread_struct *th); +void rb_thread_wake_fence(struct rb_thread_struct *th); + +#endif /* RUBY_THREAD_SCHED_H */ diff --git a/thread_pthread_mn.c b/thread_sched_mn.c similarity index 97% rename from thread_pthread_mn.c rename to thread_sched_mn.c index a57008da7b0f31..ddf35b459b8602 100644 --- a/thread_pthread_mn.c +++ b/thread_sched_mn.c @@ -1,4 +1,22 @@ -// included by "thread_pthread.c" +/* -*-c-*- */ +/********************************************************************** + + thread_sched_mn.c - the M:N scheduler + + Included by the platform implementation (currently only thread_pthread.c) + when USE_MN_THREADS is 1. A platform that cannot run coroutine threads + defines it to 0 and supplies the stubs at the bottom of this file itself + (see thread_win32.c). + + Most of what is here is platform independent: the coroutine threads + themselves, the native thread stack pool, the timer wheel, and the + fd -> waiters map. The part that is not is the readiness backend -- + arming an fd and waiting for events -- which is epoll on Linux and kqueue + elsewhere. Those pieces are marked "backend" below; they are the natural + seam for a thread_sched_epoll.c / thread_sched_kqueue.c split, and for an + IOCP backend that would let Windows run M:N threads too. + +**********************************************************************/ #if USE_MN_THREADS @@ -1358,6 +1376,14 @@ verify_waiting_list(void) #endif } +/* ------------------------------------------------------------------------ + * backend: the readiness notification mechanism (epoll / kqueue). + * + * Everything below that names epoll or kqueue is this backend; the rest of + * the M:N scheduler only asks it to arm an fd (fd_waiters_arm) and to wait + * for what fired (event_wait / timer_thread_polling). + * ------------------------------------------------------------------------ */ + #if HAVE_SYS_EVENT_H // kqueue helpers static enum thread_sched_waiting_flag diff --git a/thread_win32.c b/thread_win32.c index a7163fa8066acf..456ad8114bd86d 100644 --- a/thread_win32.c +++ b/thread_win32.c @@ -7,6 +7,10 @@ Copyright (C) 2004-2007 Koichi Sasada + Windows platform primitives for the common thread/ractor scheduler. The + scheduler itself is in thread_sched.c, which includes this file and then + builds on the primitives below; see thread_sched.h for the contract. + **********************************************************************/ #ifdef THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION @@ -14,21 +18,43 @@ #include "internal/sanitizers.h" #include -#define TIME_QUANTUM_USEC (10 * 1000) -#define RB_CONDATTR_CLOCK_MONOTONIC 1 /* no effect */ - #undef Sleep #define native_thread_yield() Sleep(0) -#define unregister_ubf_list(th) -#define ubf_wakeup_all_threads() do {} while (0) -#define ubf_threads_empty() (1) -#define ubf_timer_disarm() do {} while (0) -#define ubf_list_atfork() do {} while (0) + +// A CRITICAL_SECTION is recursive, so trylock cannot tell "held by me" from +// "free"; see thread_sched.c. +#define RB_NATIVE_MUTEX_TRYLOCK_DETECTS_SELF 0 + +// M:N threads need an event backend to park a coroutine on (epoll/kqueue on +// the POSIX side; IOCP would be the Windows counterpart). Until there is one +// every thread here is dedicated, and this file supplies the stubs that +// thread_sched_mn.c provides elsewhere. +#define USE_MN_THREADS 0 + +// Interruption is delivered through a per-native-thread event object rather +// than a signal, but the bookkeeping is the same as everywhere else. +#define USE_UBF_LIST 1 + +#include COROUTINE_H + +// Thread event hooks are not implemented on this platform. +#define RB_INTERNAL_THREAD_HOOK(event, th) ((void)0) + +// No fork(), so this never advances; it only keeps TIMER_THREAD_CREATED_P() +// spelled the same way on both platforms. +static rb_serial_t current_fork_gen = 1; + +// Always: native_cond_timedwait() below takes the rb_hrtime_t deadline and +// converts it to the relative timeout the Win32 wait wants itself. +#define RB_NATIVE_COND_HRTIME_DEADLINE_P() 1 static volatile DWORD ruby_native_thread_key = TLS_OUT_OF_INDEXES; static int w32_wait_events(HANDLE *events, int count, DWORD timeout, rb_thread_t *th); +static void native_thread_destroy(struct rb_native_thread *nt); +static void timer_thread_wakeup_force(void); +static void ubf_select(void *ptr); // thread_sched.c rb_internal_thread_event_hook_t * rb_internal_thread_add_event_hook(rb_internal_thread_event_callback callback, rb_event_flag_t internal_event, void *user_data) @@ -129,159 +155,6 @@ w32_mutex_create(void) return lock; } -#define GVL_DEBUG 0 - -static void -thread_sched_to_running(struct rb_thread_sched *sched, rb_thread_t *th) -{ - w32_mutex_lock(sched->lock, false); - if (GVL_DEBUG) fprintf(stderr, "gvl acquire (%p): acquire\n", th); -} - -static void -thread_sched_to_waiting(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately) -{ - ReleaseMutex(sched->lock); -} - -static void -thread_sched_to_dead(struct rb_thread_sched *sched, rb_thread_t *th) -{ - thread_sched_to_waiting(sched, th, true); -} - -static void -thread_sched_yield(struct rb_thread_sched *sched, rb_thread_t *th) -{ - thread_sched_to_waiting(sched, th, true); - native_thread_yield(); - thread_sched_to_running(sched, th); -} - -void -rb_thread_sched_init(struct rb_thread_sched *sched, bool atfork) -{ - if (GVL_DEBUG) fprintf(stderr, "sched init\n"); - sched->lock = w32_mutex_create(); -} - -#if 0 -// per-ractor -void -rb_thread_sched_destroy(struct rb_thread_sched *sched) -{ - if (GVL_DEBUG) fprintf(stderr, "sched destroy\n"); - CloseHandle(sched->lock); -} -#endif - -rb_thread_t * -ruby_thread_from_native(void) -{ - return TlsGetValue(ruby_native_thread_key); -} - -int -ruby_thread_set_native(rb_thread_t *th) -{ - if (th && th->ec) { - rb_ractor_set_current_ec(th->ractor, th->ec); - } - return TlsSetValue(ruby_native_thread_key, th); -} - -void -Init_native_thread(rb_thread_t *main_th) -{ - if ((ruby_current_ec_key = TlsAlloc()) == TLS_OUT_OF_INDEXES) { - rb_bug("TlsAlloc() for ruby_current_ec_key fails"); - } - if ((ruby_native_thread_key = TlsAlloc()) == TLS_OUT_OF_INDEXES) { - rb_bug("TlsAlloc() for ruby_native_thread_key fails"); - } - - // setup main thread - - ruby_thread_set_native(main_th); - main_th->nt->interrupt_event = CreateEvent(0, TRUE, FALSE, 0); - - DuplicateHandle(GetCurrentProcess(), - GetCurrentThread(), - GetCurrentProcess(), - &main_th->nt->thread_id, 0, FALSE, DUPLICATE_SAME_ACCESS); - - RUBY_DEBUG_LOG("initial thread th:%u thid:%p, event: %p", - rb_th_serial(main_th), - main_th->nt->thread_id, - main_th->nt->interrupt_event); -} - -void -ruby_mn_threads_params(void) -{ -} - -static int -w32_wait_events(HANDLE *events, int count, DWORD timeout, rb_thread_t *th) -{ - HANDLE *targets = events; - HANDLE intr; - const int initcount = count; - DWORD ret; - - w32_event_debug("events:%p, count:%d, timeout:%ld, th:%u\n", - events, count, timeout, th ? rb_th_serial(th) : UINT_MAX); - - if (th && (intr = th->nt->interrupt_event)) { - if (ResetEvent(intr) && (!RUBY_VM_INTERRUPTED(th->ec) || SetEvent(intr))) { - targets = ALLOCA_N(HANDLE, count + 1); - memcpy(targets, events, sizeof(HANDLE) * count); - - targets[count++] = intr; - w32_event_debug("handle:%p (count:%d, intr)\n", intr, count); - } - else if (intr == th->nt->interrupt_event) { - w32_error("w32_wait_events"); - } - } - - w32_event_debug("WaitForMultipleObjects start count:%d\n", count); - ret = WaitForMultipleObjects(count, targets, FALSE, timeout); - w32_event_debug("WaitForMultipleObjects end ret:%lu\n", ret); - - if (ret == (DWORD)(WAIT_OBJECT_0 + initcount) && th) { - errno = EINTR; - } - if (ret == WAIT_FAILED && W32_EVENT_DEBUG) { - int i; - DWORD dmy; - for (i = 0; i < count; i++) { - w32_event_debug("i:%d %s\n", i, GetHandleInformation(targets[i], &dmy) ? "OK" : "NG"); - } - } - return ret; -} - -static void ubf_handle(void *ptr); -#define ubf_select ubf_handle - -int -rb_w32_wait_events_blocking(HANDLE *events, int num, DWORD timeout) -{ - return w32_wait_events(events, num, timeout, ruby_thread_from_native()); -} - -int -rb_w32_wait_events(HANDLE *events, int num, DWORD timeout) -{ - int ret; - rb_thread_t *th = GET_THREAD(); - - BLOCKING_REGION(th, ret = rb_w32_wait_events_blocking(events, num, timeout), - ubf_handle, ruby_thread_from_native(), FALSE); - return ret; -} - static void w32_close_handle(HANDLE handle) { @@ -290,90 +163,9 @@ w32_close_handle(HANDLE handle) } } -static void -w32_resume_thread(HANDLE handle) -{ - if (ResumeThread(handle) == (DWORD)-1) { - w32_error("w32_resume_thread"); - } -} - -#ifdef _MSC_VER -#define HAVE__BEGINTHREADEX 1 -#else -#undef HAVE__BEGINTHREADEX -#endif - -#ifdef HAVE__BEGINTHREADEX -#define start_thread (HANDLE)_beginthreadex -#define thread_errno errno -typedef unsigned long (__stdcall *w32_thread_start_func)(void*); -#else -#define start_thread CreateThread -#define thread_errno rb_w32_map_errno(GetLastError()) -typedef LPTHREAD_START_ROUTINE w32_thread_start_func; -#endif - -static HANDLE -w32_create_thread(DWORD stack_size, w32_thread_start_func func, void *val) -{ - return start_thread(0, stack_size, func, val, CREATE_SUSPENDED | STACK_SIZE_PARAM_IS_A_RESERVATION, 0); -} - -int -rb_w32_sleep(unsigned long msec) -{ - return w32_wait_events(0, 0, msec, ruby_thread_from_native()); -} - -int WINAPI -rb_w32_Sleep(unsigned long msec) -{ - int ret; - rb_thread_t *th = GET_THREAD(); - - BLOCKING_REGION(th, ret = rb_w32_sleep(msec), - ubf_handle, ruby_thread_from_native(), FALSE); - return ret; -} - -static DWORD -hrtime2msec(rb_hrtime_t hrt) -{ - return (DWORD)hrt / (DWORD)RB_HRTIME_PER_MSEC; -} - -static void -native_sleep(rb_thread_t *th, rb_hrtime_t *rel) -{ - const volatile DWORD msec = rel ? hrtime2msec(*rel) : INFINITE; - - THREAD_BLOCKING_BEGIN(th); - { - DWORD ret; - - rb_native_mutex_lock(&th->interrupt_lock); - th->unblock.func = ubf_handle; - th->unblock.arg = th; - rb_native_mutex_unlock(&th->interrupt_lock); - - if (RUBY_VM_INTERRUPTED(th->ec)) { - /* interrupted. return immediate */ - } - else { - RUBY_DEBUG_LOG("start msec:%lu", msec); - ret = w32_wait_events(0, 0, msec, th); - RUBY_DEBUG_LOG("done ret:%lu", ret); - (void)ret; - } - - rb_native_mutex_lock(&th->interrupt_lock); - th->unblock.func = 0; - th->unblock.arg = 0; - rb_native_mutex_unlock(&th->interrupt_lock); - } - THREAD_BLOCKING_END(th); -} +/* ------------------------------------------------------------------------- + * native mutex / condition variable + * ------------------------------------------------------------------------- */ void rb_native_mutex_lock(rb_nativethread_lock_t *lock) @@ -510,75 +302,38 @@ rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex) native_cond_timedwait_ms(cond, mutex, INFINITE); } -static unsigned long -abs_timespec_to_timeout_ms(const struct timespec *ts) +void +rb_native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, unsigned long msec) { - struct timeval tv; - struct timeval now; - - gettimeofday(&now, NULL); - tv.tv_sec = ts->tv_sec; - tv.tv_usec = ts->tv_nsec / 1000; - - if (!rb_w32_time_subtract(&tv, &now)) - return 0; - - return (tv.tv_sec * 1000) + (tv.tv_usec / 1000); + native_cond_timedwait_ms(cond, mutex, msec); } -static int -native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, const struct timespec *ts) +// The scheduler parks threads with an absolute deadline; on this platform the +// wait itself is relative, so the conversion happens here. +static rb_hrtime_t +native_cond_timeout(rb_nativethread_cond_t *cond, const rb_hrtime_t rel) { - unsigned long timeout_ms; - - timeout_ms = abs_timespec_to_timeout_ms(ts); - if (!timeout_ms) - return ETIMEDOUT; - - return native_cond_timedwait_ms(cond, mutex, timeout_ms); + if (rel > 0) { + rb_hrtime_t now = rb_hrtime_now(); + return (rel > RB_HRTIME_MAX - now) ? RB_HRTIME_MAX : now + rel; + } + return rb_hrtime_now(); } -static struct timespec native_cond_timeout(rb_nativethread_cond_t *cond, struct timespec timeout_rel); - -void -rb_native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, unsigned long msec) +static int +native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, const rb_hrtime_t *abs) { - struct timespec rel = { - .tv_sec = msec / 1000, - .tv_nsec = (msec % 1000) * 1000 * 1000, - }; - struct timespec ts = native_cond_timeout(cond, rel); - native_cond_timedwait(cond, mutex, &ts); -} + rb_hrtime_t now = rb_hrtime_now(); -static struct timespec -native_cond_timeout(rb_nativethread_cond_t *cond, struct timespec timeout_rel) -{ - int ret; - struct timeval tv; - struct timespec timeout; - struct timespec now; - - ret = gettimeofday(&tv, 0); - if (ret != 0) - rb_sys_fail(0); - now.tv_sec = tv.tv_sec; - now.tv_nsec = tv.tv_usec * 1000; - - timeout.tv_sec = now.tv_sec; - timeout.tv_nsec = now.tv_nsec; - timeout.tv_sec += timeout_rel.tv_sec; - timeout.tv_nsec += timeout_rel.tv_nsec; - - if (timeout.tv_nsec >= 1000*1000*1000) { - timeout.tv_sec++; - timeout.tv_nsec -= 1000*1000*1000; - } + if (*abs <= now) return ETIMEDOUT; - if (timeout.tv_sec < now.tv_sec) - timeout.tv_sec = TIMET_MAX; + rb_hrtime_t rel = *abs - now; + unsigned long msec = (unsigned long)(rel / RB_HRTIME_PER_MSEC); - return timeout; + // do not busy loop on a sub-millisecond deadline + if (msec == 0) msec = 1; + + return native_cond_timedwait_ms(cond, mutex, msec); } void @@ -594,109 +349,292 @@ rb_native_cond_destroy(rb_nativethread_cond_t *cond) /* */ } +/* ------------------------------------------------------------------------- + * thread local storage + * ------------------------------------------------------------------------- */ -#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < 0x602 -/* declared in processthreadsapi.h only when _WIN32_WINNT >= 0x0602, - * but exported from kernel32.dll since Windows 8 */ -WINBASEAPI VOID WINAPI GetCurrentThreadStackLimits(PULONG_PTR, PULONG_PTR); -#endif +rb_thread_t * +ruby_thread_from_native(void) +{ + return TlsGetValue(ruby_native_thread_key); +} -static void -native_thread_init_stack(rb_thread_t *th, void *local_in_parent_frame) +int +ruby_thread_set_native(rb_thread_t *th) { - ULONG_PTR low, high; - SIZE_T size, space; + if (th) { + ccan_list_node_init(&th->sched.node.ubf); + } - /* VirtualQuery against the current stack pointer may return a region - * that does not span the whole stack when the interpreter is - * initialized deep in the stack, which makes stack_check() misfire. - * [Bug #11438] */ - GetCurrentThreadStackLimits(&low, &high); - size = high - low; - space = size / 5; - if (space > 1024*1024) space = 1024*1024; - th->ec->machine.stack_start = (VALUE *)high - 1; - th->ec->machine.stack_maxsize = size - space; + if (th && th->ec) { + rb_ractor_set_current_ec(th->ractor, th->ec); + } + return TlsSetValue(ruby_native_thread_key, th); } -static void -native_thread_destroy_atfork(struct rb_native_thread *nt) +/* ------------------------------------------------------------------------- + * waiting on Windows objects, with interruption + * ------------------------------------------------------------------------- */ + +static int +w32_wait_events(HANDLE *events, int count, DWORD timeout, rb_thread_t *th) { - /* no-op */ -} + HANDLE *targets = events; + HANDLE intr; + const int initcount = count; + DWORD ret; + + w32_event_debug("events:%p, count:%d, timeout:%ld, th:%u\n", + events, count, timeout, th ? rb_th_serial(th) : UINT_MAX); + + if (th && (intr = th->nt->interrupt_event)) { + if (ResetEvent(intr) && (!RUBY_VM_INTERRUPTED(th->ec) || SetEvent(intr))) { + targets = ALLOCA_N(HANDLE, count + 1); + memcpy(targets, events, sizeof(HANDLE) * count); + + targets[count++] = intr; + w32_event_debug("handle:%p (count:%d, intr)\n", intr, count); + } + else if (intr == th->nt->interrupt_event) { + w32_error("w32_wait_events"); + } + } + + w32_event_debug("WaitForMultipleObjects start count:%d\n", count); + ret = WaitForMultipleObjects(count, targets, FALSE, timeout); + w32_event_debug("WaitForMultipleObjects end ret:%lu\n", ret); + + if (ret == (DWORD)(WAIT_OBJECT_0 + initcount) && th) { + errno = EINTR; + } + if (ret == WAIT_FAILED && W32_EVENT_DEBUG) { + int i; + DWORD dmy; + for (i = 0; i < count; i++) { + w32_event_debug("i:%d %s\n", i, GetHandleInformation(targets[i], &dmy) ? "OK" : "NG"); + } + } + return ret; +} + +int +rb_w32_wait_events_blocking(HANDLE *events, int num, DWORD timeout) +{ + return w32_wait_events(events, num, timeout, ruby_thread_from_native()); +} + +int +rb_w32_wait_events(HANDLE *events, int num, DWORD timeout) +{ + int ret; + rb_thread_t *th = GET_THREAD(); + + BLOCKING_REGION(th, ret = rb_w32_wait_events_blocking(events, num, timeout), + ubf_select, ruby_thread_from_native(), FALSE); + return ret; +} + +int +rb_w32_sleep(unsigned long msec) +{ + return w32_wait_events(0, 0, msec, ruby_thread_from_native()); +} + +int WINAPI +rb_w32_Sleep(unsigned long msec) +{ + int ret; + rb_thread_t *th = GET_THREAD(); + + BLOCKING_REGION(th, ret = rb_w32_sleep(msec), + ubf_select, ruby_thread_from_native(), FALSE); + return ret; +} + +/* @internal */ +int +rb_w32_check_interrupt(rb_thread_t *th) +{ + return w32_wait_events(0, 0, 0, th); +} + +/* + * Pull the target thread out of a blocking w32_wait_events(). This is the + * counterpart of the SIGVTALRM the POSIX implementation sends. + */ +static void +native_thread_interrupt(rb_thread_t *th) +{ + // the caller (ubf_wakeup_thread) logs this + if (!SetEvent(th->nt->interrupt_event)) { + w32_error("native_thread_interrupt"); + } +} + +/* ------------------------------------------------------------------------- + * native thread + * ------------------------------------------------------------------------- */ + +static void +w32_resume_thread(HANDLE handle) +{ + if (ResumeThread(handle) == (DWORD)-1) { + w32_error("w32_resume_thread"); + } +} + +#ifdef _MSC_VER +#define HAVE__BEGINTHREADEX 1 +#else +#undef HAVE__BEGINTHREADEX +#endif + +#ifdef HAVE__BEGINTHREADEX +#define start_thread (HANDLE)_beginthreadex +#define thread_errno errno +typedef unsigned long (__stdcall *w32_thread_start_func)(void*); +#else +#define start_thread CreateThread +#define thread_errno rb_w32_map_errno(GetLastError()) +typedef LPTHREAD_START_ROUTINE w32_thread_start_func; +#endif + +static HANDLE +w32_create_thread(DWORD stack_size, w32_thread_start_func func, void *val) +{ + return start_thread(0, stack_size, func, val, CREATE_SUSPENDED | STACK_SIZE_PARAM_IS_A_RESERVATION, 0); +} + +static void +native_thread_join(HANDLE th) +{ + w32_wait_events(&th, 1, INFINITE, 0); +} + +#if !defined(_WIN32_WINNT_WIN8) || _WIN32_WINNT < 0x602 +/* declared in processthreadsapi.h only when _WIN32_WINNT >= 0x0602, + * but exported from kernel32.dll since Windows 8 */ +WINBASEAPI VOID WINAPI GetCurrentThreadStackLimits(PULONG_PTR, PULONG_PTR); +#endif + +static void +native_thread_init_stack(rb_thread_t *th, void *local_in_parent_frame) +{ + ULONG_PTR low, high; + SIZE_T size, space; + + /* VirtualQuery against the current stack pointer may return a region + * that does not span the whole stack when the interpreter is + * initialized deep in the stack, which makes stack_check() misfire. + * [Bug #11438] */ + GetCurrentThreadStackLimits(&low, &high); + size = high - low; + space = size / 5; + if (space > 1024*1024) space = 1024*1024; + th->ec->machine.stack_start = (VALUE *)high - 1; + th->ec->machine.stack_maxsize = size - space; +} + +static void +native_thread_setup(struct rb_native_thread *nt) +{ + rb_native_cond_initialize(&nt->readyq); + rb_native_mutex_initialize(&nt->running_th_lock); + + // Created here rather than on the new thread itself: ubf can fire before + // that thread gets a chance to run. + nt->interrupt_event = CreateEvent(0, TRUE, FALSE, 0); + if (nt->interrupt_event == NULL) { + w32_error("native_thread_setup"); + } +} + +static void +native_thread_setup_on_thread(struct rb_native_thread *nt) +{ + // nothing to do: there is no altstack and no thread id to cache +} + +static struct rb_native_thread * +native_thread_alloc(void) +{ + struct rb_native_thread *nt = ZALLOC(struct rb_native_thread); + native_thread_setup(nt); + +#if USE_RUBY_DEBUG_LOG + static rb_atomic_t nt_serial = 2; + nt->serial = RUBY_ATOMIC_FETCH_ADD(nt_serial, 1); +#endif + return nt; +} + +static void +native_thread_destroy_atfork(struct rb_native_thread *nt) +{ + /* no fork() on this platform */ +} #ifndef InterlockedExchangePointer #define InterlockedExchangePointer(t, v) \ (void *)InterlockedExchange((long *)(t), (long)(v)) #endif + static void native_thread_destroy(struct rb_native_thread *nt) { if (nt) { HANDLE intr = InterlockedExchangePointer(&nt->interrupt_event, 0); RUBY_DEBUG_LOG("close handle intr:%p, thid:%p\n", intr, nt->thread_id); - w32_close_handle(intr); + if (intr) w32_close_handle(intr); + + rb_native_cond_destroy(&nt->readyq); + rb_native_mutex_destroy(&nt->running_th_lock); + + ruby_xfree(nt); } } -static unsigned long __stdcall -thread_start_func_1(void *th_ptr) +static void +native_thread_destroy_self(struct rb_native_thread *nt) { - rb_thread_t *th = th_ptr; - volatile HANDLE thread_id = th->nt->thread_id; - - native_thread_init_stack(th, &th); - th->nt->interrupt_event = CreateEvent(0, TRUE, FALSE, 0); - - /* run */ - RUBY_DEBUG_LOG("thread created th:%u, thid: %p, event: %p", - rb_th_serial(th), th->nt->thread_id, th->nt->interrupt_event); + native_thread_destroy(nt); +} - thread_sched_to_running(TH_SCHED(th), th); - ruby_thread_set_native(th); +static unsigned long __stdcall +nt_start_trampoline(void *nt_ptr) +{ + struct rb_native_thread *nt = (struct rb_native_thread *)nt_ptr; + HANDLE thread_id = nt->thread_id; - // kick threads - thread_start_func_2(th, th->ec->machine.stack_start); + nt_start(nt); w32_close_handle(thread_id); - RUBY_DEBUG_LOG("thread deleted th:%u", rb_th_serial(th)); - return 0; } static int -native_thread_create(rb_thread_t *th) +native_thread_create0(struct rb_native_thread *nt) { - // setup nt - const size_t stack_size = th->vm->default_params.thread_machine_stack_size; - th->nt = ZALLOC(struct rb_native_thread); - th->nt->thread_id = w32_create_thread(stack_size, thread_start_func_1, th); + const size_t stack_size = nt->vm->default_params.thread_machine_stack_size; - // setup vm stack - size_t vm_stack_word_size = th->vm->default_params.thread_vm_stack_size / sizeof(VALUE); - void *vm_stack = ruby_xmalloc(vm_stack_word_size * sizeof(VALUE)); - th->sched.vm_stack = vm_stack; - rb_ec_initialize_vm_stack(th->ec, vm_stack, vm_stack_word_size); - - if ((th->nt->thread_id) == 0) { + nt->thread_id = w32_create_thread(stack_size, nt_start_trampoline, nt); + if (nt->thread_id == 0) { return thread_errno; } - w32_resume_thread(th->nt->thread_id); + w32_resume_thread(nt->thread_id); - if (USE_RUBY_DEBUG_LOG) { - Sleep(0); - RUBY_DEBUG_LOG("th:%u thid:%p intr:%p), stack size: %"PRIuSIZE"", - rb_th_serial(th), th->nt->thread_id, - th->nt->interrupt_event, stack_size); - } + RUBY_DEBUG_LOG("nt:%u thid:%p stack size:%"PRIuSIZE"", + nt->serial, nt->thread_id, stack_size); return 0; } -static void -native_thread_join(HANDLE th) +static int +native_thread_default_max_cpu(void) { - w32_wait_events(&th, 1, INFINITE, 0); + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwNumberOfProcessors > 0 ? (int)si.dwNumberOfProcessors : 8; } #if USE_NATIVE_THREAD_PRIORITY @@ -741,66 +679,166 @@ native_fd_select(int n, rb_fdset_t *readfds, rb_fdset_t *writefds, rb_fdset_t *e return rb_w32_select_with_thread(n, r, w, e, timeout, th); } -/* @internal */ -int -rb_w32_check_interrupt(rb_thread_t *th) +int rb_w32_set_thread_description(HANDLE th, const WCHAR *name); +int rb_w32_set_thread_description_str(HANDLE th, VALUE name); +#define native_set_another_thread_name rb_w32_set_thread_description_str + +static void +native_set_thread_name(rb_thread_t *th) { - return w32_wait_events(0, 0, 0, th); } -static void -ubf_handle(void *ptr) +static VALUE +native_thread_native_thread_id(rb_thread_t *th) { - rb_thread_t *th = (rb_thread_t *)ptr; - RUBY_DEBUG_LOG("th:%u\n", rb_th_serial(th)); + DWORD tid = GetThreadId(th->nt->thread_id); + if (tid == 0) rb_sys_fail("GetThreadId"); + return ULONG2NUM(tid); +} +#define USE_NATIVE_THREAD_NATIVE_THREAD_ID 1 - if (!SetEvent(th->nt->interrupt_event)) { - w32_error("ubf_handle"); +void +Init_native_thread(rb_thread_t *main_th) +{ + if ((ruby_current_ec_key = TlsAlloc()) == TLS_OUT_OF_INDEXES) { + rb_bug("TlsAlloc() for ruby_current_ec_key fails"); + } + if ((ruby_native_thread_key = TlsAlloc()) == TLS_OUT_OF_INDEXES) { + rb_bug("TlsAlloc() for ruby_native_thread_key fails"); } + + // setup vm + rb_vm_t *vm = main_th->vm; + thread_sched_init_vm(vm); + + // setup main thread + native_thread_setup(main_th->nt); + DuplicateHandle(GetCurrentProcess(), + GetCurrentThread(), + GetCurrentProcess(), + &main_th->nt->thread_id, 0, FALSE, DUPLICATE_SAME_ACCESS); + main_th->nt->serial = 1; + ruby_thread_set_native(main_th); + + TH_SCHED(main_th)->running = main_th; + main_th->has_dedicated_nt = 1; + + // setup main NT (before the record below: its kind decides where it goes) + main_th->nt->dedicated = 1; + main_th->nt->running_thread = main_th; + main_th->nt->vm = vm; + + thread_sched_setup_running_threads(TH_SCHED(main_th), main_th->ractor, vm, main_th, NULL); + +#if USE_RUBY_DEBUG_LOG + vm->ractor.sched.dnt_cnt = 1; +#endif + + RUBY_DEBUG_LOG("initial thread th:%u thid:%p, event: %p", + rb_th_serial(main_th), + main_th->nt->thread_id, + main_th->nt->interrupt_event); } -int rb_w32_set_thread_description(HANDLE th, const WCHAR *name); -int rb_w32_set_thread_description_str(HANDLE th, VALUE name); -#define native_set_another_thread_name rb_w32_set_thread_description_str +/* ------------------------------------------------------------------------- + * timer thread + * ------------------------------------------------------------------------- */ static struct { - HANDLE id; - HANDLE lock; -} timer_thread; -#define TIMER_THREAD_CREATED_P() (timer_thread.id != 0) + rb_serial_t created_fork_gen; + HANDLE thread_id; + HANDLE wakeup_event; // manual reset; the "comm pipe" of this platform +} timer_th = { + .created_fork_gen = 0, +}; -static unsigned long __stdcall -timer_thread_func(void *dummy) +#define TIMER_THREAD_CREATED_P() (timer_th.created_fork_gen == current_fork_gen) + +static void +timer_thread_wakeup_force(void) { - rb_vm_t *vm = GET_VM(); - RUBY_DEBUG_LOG("start"); - rb_w32_set_thread_description(GetCurrentThread(), L"ruby-timer-thread"); - while (WaitForSingleObject(timer_thread.lock, - TIME_QUANTUM_USEC/1000) == WAIT_TIMEOUT) { - vm->clock++; - rb_threadptr_check_signal(vm->ractor.main_thread); + if (timer_th.wakeup_event) { + SetEvent(timer_th.wakeup_event); } - RUBY_DEBUG_LOG("end"); - return 0; } void rb_thread_wakeup_timer_thread(int sig) { - /* do nothing */ + timer_thread_wakeup_force(); + + if (RUBY_ATOMIC_LOAD(system_working)) { + rb_vm_t *vm = GET_VM(); + rb_thread_t *main_th = vm->ractor.main_thread; + + if (main_th) { + volatile rb_execution_context_t *main_th_ec = ACCESS_ONCE(rb_execution_context_t *, main_th->ec); + + if (main_th_ec) { + RUBY_VM_SET_TRAP_INTERRUPT(main_th_ec); + + if (vm->ubf_async_safe && main_th->unblock.func) { + (main_th->unblock.func)(main_th->unblock.arg); + } + } + } + } +} + +// The blocking part of the timer thread loop: this is what +// timer_thread_wakeup_force() interrupts. +static void +timer_thread_polling(rb_vm_t *vm) +{ + int timeout = timer_thread_set_timeout(vm); + DWORD msec = (timeout < 0) ? INFINITE : (DWORD)timeout; + + DWORD ret = WaitForSingleObject(timer_th.wakeup_event, msec); + + switch (ret) { + case WAIT_TIMEOUT: + ractor_sched_lock(vm, NULL); + { + timer_thread_check_timeslice(vm); + } + ractor_sched_unlock(vm, NULL); + break; + + case WAIT_OBJECT_0: + ResetEvent(timer_th.wakeup_event); + break; + + default: + w32_error("timer_thread_polling"); + } +} + +static unsigned long __stdcall +timer_thread_trampoline(void *vm_ptr) +{ + rb_w32_set_thread_description(GetCurrentThread(), L"ruby-timer-thread"); + timer_thread_func(vm_ptr); + return 0; } static void rb_thread_create_timer_thread(void) { - if (timer_thread.id == 0) { - if (!timer_thread.lock) { - timer_thread.lock = CreateEvent(0, TRUE, FALSE, 0); + timer_th.created_fork_gen = current_fork_gen; + + if (timer_th.wakeup_event == NULL) { + timer_th.wakeup_event = CreateEvent(0, TRUE, FALSE, 0); + if (timer_th.wakeup_event == NULL) { + w32_error("rb_thread_create_timer_thread"); } - timer_thread.id = w32_create_thread(1024 + (USE_RUBY_DEBUG_LOG ? BUFSIZ : 0), - timer_thread_func, 0); - w32_resume_thread(timer_thread.id); } + + timer_th.thread_id = w32_create_thread(1024 + (USE_RUBY_DEBUG_LOG ? BUFSIZ : 0), + timer_thread_trampoline, GET_VM()); + if (timer_th.thread_id == 0) { + rb_bug("rb_thread_create_timer_thread: failed to create the timer thread"); + } + w32_resume_thread(timer_th.thread_id); } static int @@ -808,10 +846,11 @@ native_stop_timer_thread(void) { RUBY_ATOMIC_SET(system_working, 0); - SetEvent(timer_thread.lock); - native_thread_join(timer_thread.id); - CloseHandle(timer_thread.lock); - timer_thread.lock = 0; + timer_thread_wakeup_force(); + native_thread_join(timer_th.thread_id); + + w32_close_handle(timer_th.wakeup_event); + timer_th.wakeup_event = NULL; return 1; } @@ -819,12 +858,67 @@ native_stop_timer_thread(void) static void native_reset_timer_thread(void) { - if (timer_thread.id) { - CloseHandle(timer_thread.id); - timer_thread.id = 0; + if (timer_th.thread_id) { + CloseHandle(timer_th.thread_id); + timer_th.thread_id = 0; } } +/* ------------------------------------------------------------------------- + * M:N scheduler stubs + * + * These are what thread_sched_mn.c provides on platforms that have an event + * backend. Every thread here is dedicated, so the scheduler never reaches + * the ones that rb_bug(). + * ------------------------------------------------------------------------- */ + +static int +native_thread_create_shared(rb_thread_t *th) +{ + rb_bug("unreachable"); +} + +static enum thread_sched_wait_result +thread_sched_wait_events(struct rb_thread_sched *sched, rb_thread_t *th, int fd, + enum thread_sched_waiting_flag events, rb_hrtime_t *rel) +{ + return thread_sched_wait_unavailable; +} + +static bool +ractor_sched_timeout_arm(rb_thread_t *th, const rb_hrtime_t *rel) +{ + rb_bug("unreachable"); +} + +static bool +ractor_sched_timeout_disarm(rb_thread_t *th) +{ + rb_bug("unreachable"); +} + +static int +timer_wheel_timeout(int timeout) +{ + return timeout; // no M:N threads, no timed waiters +} + +static void +timer_thread_wake_fence(rb_thread_t *th) +{ + // no timer wheel, no wake batches +} + +static void +timer_thread_check_timeout(rb_vm_t *vm) +{ + // no M:N threads, no timed waiters +} + +/* ------------------------------------------------------------------------- + * misc + * ------------------------------------------------------------------------- */ + int ruby_stack_overflowed_p(const rb_thread_t *th, const void *addr) { @@ -856,6 +950,7 @@ ruby_alloca_chkstk(size_t len, void *sp) } } #endif + int rb_reserved_fd_p(int fd) { @@ -868,175 +963,10 @@ rb_nativethread_self(void) return GetCurrentThread(); } -static void -native_set_thread_name(rb_thread_t *th) -{ -} - -static VALUE -native_thread_native_thread_id(rb_thread_t *th) -{ - DWORD tid = GetThreadId(th->nt->thread_id); - if (tid == 0) rb_sys_fail("GetThreadId"); - return ULONG2NUM(tid); -} -#define USE_NATIVE_THREAD_NATIVE_THREAD_ID 1 - -void -rb_add_running_thread(rb_thread_t *th) -{ - // do nothing -} - -void -rb_del_running_thread(rb_thread_t *th) -{ - // do nothing -} - -static bool -th_has_dedicated_nt(const rb_thread_t *th) -{ - return true; -} - -void -rb_threadptr_sched_free(rb_thread_t *th) -{ - native_thread_destroy(th->nt); - ruby_xfree(th->nt); - ruby_xfree(th->sched.vm_stack); -} - - - -static bool -vm_barrier_finish_p(rb_vm_t *vm) -{ - RUBY_DEBUG_LOG("cnt:%u living:%u blocking:%u", - vm->ractor.blocking_cnt == vm->ractor.cnt, - vm->ractor.sync.barrier_cnt, - vm->ractor.cnt, - vm->ractor.blocking_cnt); - - VM_ASSERT(vm->ractor.blocking_cnt <= vm->ractor.cnt); - - return vm->ractor.blocking_cnt == vm->ractor.cnt; -} - -void -rb_ractor_sched_barrier_start(rb_vm_t *vm, rb_ractor_t *cr) -{ - vm->ractor.sync.barrier_waiting = true; - - RUBY_DEBUG_LOG("barrier start. cnt:%u living:%u blocking:%u", - vm->ractor.sync.barrier_cnt, - vm->ractor.cnt, - vm->ractor.blocking_cnt); - - rb_vm_ractor_blocking_cnt_inc(vm, cr, __FILE__, __LINE__); - - // send signal - rb_ractor_t *r = 0; - ccan_list_for_each(&vm->ractor.set, r, vmlr_node) { - if (r != cr) { - rb_ractor_vm_barrier_interrupt_running_thread(r); - } - } - - // wait - while (!vm_barrier_finish_p(vm)) { - rb_vm_cond_wait(vm, &vm->ractor.sync.barrier_complete_cond); - } - - RUBY_DEBUG_LOG("cnt:%u barrier success", vm->ractor.sync.barrier_cnt); - - rb_vm_ractor_blocking_cnt_dec(vm, cr, __FILE__, __LINE__); - - vm->ractor.sync.barrier_waiting = false; - vm->ractor.sync.barrier_cnt++; - - rb_native_cond_broadcast(&vm->ractor.sync.barrier_release_cond); -} - -void -rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr) -{ - vm->ractor.sync.lock_owner = cr; - unsigned int barrier_cnt = vm->ractor.sync.barrier_cnt; - rb_thread_t *th = GET_THREAD(); - bool running; - - RB_VM_SAVE_MACHINE_CONTEXT(th); - - if (rb_ractor_status_p(cr, ractor_running)) { - rb_vm_ractor_blocking_cnt_inc(vm, cr, __FILE__, __LINE__); - running = true; - } - else { - running = false; - } - VM_ASSERT(rb_ractor_status_p(cr, ractor_blocking)); - - if (vm_barrier_finish_p(vm)) { - RUBY_DEBUG_LOG("wakeup barrier owner"); - rb_native_cond_signal(&vm->ractor.sync.barrier_complete_cond); - } - else { - RUBY_DEBUG_LOG("wait for barrier finish"); - } - - // wait for restart - while (barrier_cnt == vm->ractor.sync.barrier_cnt) { - rb_vm_cond_wait(vm, &vm->ractor.sync.barrier_release_cond); - } - - RUBY_DEBUG_LOG("barrier is released. Acquire vm_lock"); - - if (running) { - rb_vm_ractor_blocking_cnt_dec(vm, cr, __FILE__, __LINE__); - } - - vm->ractor.sync.lock_owner = NULL; -} - -bool -rb_thread_lock_native_thread(void) -{ - return false; -} - void * rb_thread_prevent_fork(void *(*func)(void *), void *data) { return func(data); } -void -rb_thread_malloc_stack_set(rb_thread_t *th, void *stack, size_t stack_size) -{ - // no-op -} - #endif /* THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION */ - -void -rb_thread_sched_winding_begin(rb_vm_t *vm) -{ - // nothing to count: rb_thread_sched_wait_winding below never waits - (void)vm; -} - -void -rb_thread_sched_winding_end(rb_vm_t *vm) -{ - (void)vm; -} - -void -rb_thread_sched_wait_winding(rb_vm_t *vm) -{ - // no coroutine (M:N) threads on this implementation: nothing winds down - // after leaving the living set (see thread_pthread.c) - (void)vm; -} diff --git a/thread_win32.h b/thread_win32.h index 23cd71fcfe08d9..402cdbde37bd95 100644 --- a/thread_win32.h +++ b/thread_win32.h @@ -8,6 +8,9 @@ Copyright (C) 2004-2007 Koichi Sasada + This platform runs the common scheduler; see thread_sched.h for its data + structures and thread_sched.c for the implementation. + **********************************************************************/ /* interface */ @@ -16,23 +19,10 @@ # undef _WIN32 # endif -#define USE_VM_CLOCK 1 - WINBASEAPI BOOL WINAPI TryEnterCriticalSection(IN OUT LPCRITICAL_SECTION lpCriticalSection); -struct rb_native_thread { - HANDLE thread_id; - HANDLE interrupt_event; -}; - -struct rb_thread_sched_item { - void *vm_stack; -}; - -struct rb_thread_sched { - HANDLE lock; -}; +#include "thread_sched.h" typedef DWORD native_tls_key_t; // TLS index diff --git a/vm.c b/vm.c index d2d1f55d5e88d0..95a75af8ca31ea 100644 --- a/vm.c +++ b/vm.c @@ -4903,10 +4903,6 @@ Init_BareVM(void) vm_opt_method_def_table = st_init_numtable(); vm_opt_mid_table = st_init_numtable(); -#ifdef RUBY_THREAD_WIN32_H - rb_native_cond_initialize(&vm->ractor.sync.barrier_complete_cond); - rb_native_cond_initialize(&vm->ractor.sync.barrier_release_cond); -#endif } void diff --git a/vm_core.h b/vm_core.h index 0e2a22305ee572..2c416859573d63 100644 --- a/vm_core.h +++ b/vm_core.h @@ -726,69 +726,14 @@ typedef struct rb_vm_struct { // join at exit rb_nativethread_cond_t terminate_cond; bool terminate_waiting; - -#ifndef RUBY_THREAD_PTHREAD_H - // win32 - bool barrier_waiting; - unsigned int barrier_cnt; - rb_nativethread_cond_t barrier_complete_cond; - rb_nativethread_cond_t barrier_release_cond; -#endif } sync; /* VM-wide locks for the Ractor transfer/inheritance machinery. All of them * are leaf locks: no safepoint inside a critical section. */ rb_nativethread_lock_t generic_fields_lock; /* the shared generic-fields table in variable.c */ -#ifdef RUBY_THREAD_PTHREAD_H - // ractor scheduling - struct { - rb_nativethread_lock_t lock; - struct rb_ractor_struct *lock_owner; - bool locked; - - rb_nativethread_cond_t cond; // GRQ - rb_atomic_t snt_cnt; // count of shared NTs; lock-free (see native_thread_dedicated_inc) - unsigned int dnt_cnt; // count of dedicated NTs; logging only (USE_RUBY_DEBUG_LOG), not atomic - - - unsigned int max_cpu; - struct ccan_list_head grq; // // Global Ready Queue - rb_atomic_t winding_cnt; // native threads between a coroutine epilogue and its reclaim; ruby_vm_destruct waits for 0 - unsigned int grq_cnt; - - // What the barrier walk visits: threads running on dedicated - // nts, and the shared nts (whose running_th fields hold the rest). - struct { - rb_nativethread_lock_t lock; - struct ccan_list_head running_dnts; - struct ccan_list_head snts; - } ntlist; - - // scheds whose readyq holds waiters: the timer ticks their - // running thread (timeslice_scan) and prunes drained entries. - struct { - rb_nativethread_lock_t lock; - struct ccan_list_head scheds; - } timeslice; - - // true if timeslice timer is not enable - bool timeslice_wait_inf; - - // barrier - rb_nativethread_cond_t barrier_complete_cond; - rb_nativethread_cond_t barrier_release_cond; - // bool; nonzero while a stop-the-world section is active. Set - // before the barrier walks the running records; a record moved - // after the walk sees it (thread_sched_setup_running_threads). - rb_atomic_t barrier_is_waiting; - unsigned int barrier_joined_cnt; // threads joined so far; under sched.lock - unsigned int barrier_running_cnt; // runners counted by the barrier's walk; under sched.lock - unsigned int barrier_serial; - struct rb_ractor_struct *barrier_ractor; - unsigned int barrier_lock_rec; - } sched; -#endif + // ractor scheduling; see thread_sched.h + struct rb_ractor_sched sched; } ractor; #ifdef USE_SIGALTSTACK diff --git a/vm_sync.c b/vm_sync.c index 40adafe09290e1..7b6a587a1442f6 100644 --- a/vm_sync.c +++ b/vm_sync.c @@ -65,21 +65,13 @@ rb_vm_locked_p(void) static bool vm_need_barrier_waiting(const rb_vm_t *vm) { -#ifdef RUBY_THREAD_PTHREAD_H return vm->ractor.sched.barrier_is_waiting; -#else - return vm->ractor.sync.barrier_waiting; -#endif } static bool vm_need_barrier(bool no_barrier, const rb_ractor_t *cr, const rb_vm_t *vm) { -#ifdef RUBY_THREAD_PTHREAD_H return !no_barrier && cr->threads.sched.running != NULL && vm_need_barrier_waiting(vm); // ractor has running threads. -#else - return !no_barrier && vm_need_barrier_waiting(vm); -#endif } static void @@ -142,13 +134,11 @@ vm_lock_leave(rb_vm_t *vm, bool no_barrier, unsigned int *lev APPEND_LOCATION_AR VM_ASSERT(vm->ractor.sync.lock_rec == *lev); VM_ASSERT(cr == GET_RACTOR()); -#ifdef RUBY_THREAD_PTHREAD_H if (vm->ractor.sched.barrier_ractor == cr && vm->ractor.sched.barrier_lock_rec == vm->ractor.sync.lock_rec) { VM_ASSERT(!no_barrier); rb_ractor_sched_barrier_end(vm, cr); } -#endif if (RUBY_DTRACE_GVL_RELEASE_ENABLED()) { RUBY_DTRACE_GVL_RELEASE(); @@ -258,11 +248,7 @@ rb_vm_cond_timedwait(rb_vm_t *vm, rb_nativethread_cond_t *cond, unsigned long ms static bool vm_barrier_acquired_p(const rb_vm_t *vm, const rb_ractor_t *cr) { -#ifdef RUBY_THREAD_PTHREAD_H return vm->ractor.sched.barrier_ractor == cr; -#else - return false; -#endif } void