[pull] master from ruby:master

array.c

@@ -6916,7 +6916,7 @@ rb_ary_flatten_bang(int argc, VALUE *argv, VALUE ary)
         }
     }
 
-    if (!(mod = ARY_EMBED_P(result) && result != child)) rb_ary_freeze(result);
+    if (result != child && !(mod = ARY_EMBED_P(result))) rb_ary_freeze(result);
     rb_ary_replace(ary, result);
     if (mod) ARY_SET_EMBED_LEN(result, 0);
 

lib/mkmf/depend.rb

@@ -800,30 +800,6 @@ def compact_dependencies(rules, group: true)
     lines.uniq.sort.join
   end
 
-  # Returns target and dependency pairs from Make rules.
-  def dependency_pairs(rules)
-    rules = normalize_dependency_rules(rules)
-    rules.each_line.each_with_object(Set.new) do |line, pairs|
-      next unless /\A(\S+(?:\s+\S+)*):\s*(.*?)\s*\z/ =~ line
-
-      targets = $1.split
-      dependencies = expand_dependency_variables($2.split).map do |dependency|
-        normalize_dependency_rules(dependency)
-      end
-      targets.product(dependencies) {|pair| pairs << pair}
-    end
-  end
-
-  # Removes generated dependencies already covered by +manual_rules+.
-  def remove_manual_dependencies(generated, manual_rules)
-    manual = dependency_pairs(manual_rules)
-    generated.each_line.reject do |line|
-      normalized = normalize_dependency_rules(line)
-      /\A(\S+):\s+(\S+)\s*\z/ =~ normalized &&
-        manual.include?([$1, $2])
-    end.join
-  end
-
   # Removes VPATH markers that are unnecessary in build-directory output.
   def normalize_dependency_rules(rules)
     rules.gsub(/\{(?:\.;)?\$\(VPATH\)\}/, '')
@@ -972,9 +948,7 @@ def update_extension(input, source_map, make_variables: {}, nmake: false,
         source, generated, target: target, input: input, project: true
       )
     end
-    manual = match.pre_match + match.post_match
-    generated = remove_manual_dependencies(generated.join, manual)
-    expected = compact_dependencies(generated, group: !nmake)
+    expected = compact_dependencies(generated.join, group: !nmake)
     updated = match.pre_match + expected + match.post_match
     return false if same_dependency_rules?(match[0], expected)
 

ractor.c

@@ -237,6 +237,33 @@ mark_targeted_hook_list(st_data_t key, st_data_t value, st_data_t _arg)
     return ST_CONTINUE;
 }
 
+static void
+ractor_mark_thread(rb_thread_t *th)
+{
+    rb_gc_mark(th->self);
+
+    /* A thread's ec lives inside the root fiber struct and is freed with that
+     * fiber's wrapper object, so keep the fiber wrappers alive from here too. */
+    if (th->root_fiber) {
+        VALUE root_fiber_self = rb_fiberptr_self(th->root_fiber);
+        if (root_fiber_self) rb_gc_mark(root_fiber_self);
+    }
+    /* The ec sits inside its fiber, so marking that fiber's wrapper scans the ec
+     * as well.  Only when there is no wrapper yet (mid-creation, teardown) does
+     * the ec need marking of its own. */
+    VALUE ec_fiber_self = (th->ec && th->ec->fiber_ptr) ? rb_fiberptr_self(th->ec->fiber_ptr) : 0;
+    if (ec_fiber_self) {
+        rb_gc_mark(ec_fiber_self);
+    }
+    else if (th->ec) {
+        rb_execution_context_mark(th->ec);
+    }
+
+    /* Root the thread's remaining possessions directly as well; thgroup in
+     * particular has no other root. */
+    rb_thread_mark_owned_roots(th);
+}
+
 static void
 ractor_mark_unshareable_parts(rb_ractor_t *r)
 {
@@ -263,31 +290,19 @@ ractor_mark_unshareable_parts(rb_ractor_t *r)
         rb_thread_t *th = 0;
         ccan_list_for_each(&r->threads.set, th, lt_node) {
             VM_ASSERT(th != NULL);
-            rb_gc_mark(th->self);
-
-            /* A thread's ec lives inside the root fiber struct and is freed with that
-             * fiber's wrapper object, so keep the fiber wrappers alive from here too. */
-            if (th->root_fiber) {
-                VALUE root_fiber_self = rb_fiberptr_self(th->root_fiber);
-                if (root_fiber_self) rb_gc_mark(root_fiber_self);
-            }
-            /* The ec sits inside its fiber, so marking that fiber's wrapper scans the ec
-             * as well.  Only when there is no wrapper yet (mid-creation, teardown) does
-             * the ec need marking of its own. */
-            VALUE ec_fiber_self = (th->ec && th->ec->fiber_ptr) ? rb_fiberptr_self(th->ec->fiber_ptr) : 0;
-            if (ec_fiber_self) {
-                rb_gc_mark(ec_fiber_self);
-            }
-            else if (th->ec) {
-                rb_execution_context_mark(th->ec);
-            }
-
-            /* Root the thread's remaining possessions directly as well; thgroup in
-             * particular has no other root. */
-            rb_thread_mark_owned_roots(th);
+            ractor_mark_thread(th);
         }
     }
 
+    /* A thread in the MN termination epilogue has left the set but is still
+     * running on its coroutine stack; it stays a root until its last use.
+     * Read once: the epilogue clears the slot concurrently.  The thread is
+     * past rb_fiber_close/thread_cleanup_func by then -- the same state
+     * thread_mark walks whenever a terminated Thread's wrapper is still
+     * referenced, and ractor_mark_thread performs the same marks. */
+    rb_thread_t *dying_th = RUBY_ATOMIC_PTR_LOAD(r->threads.dying_th);
+    if (dying_th) ractor_mark_thread(dying_th);
+
     ractor_local_storage_mark(r);
 }
 
@@ -329,7 +344,11 @@ rb_ractor_mark_local_roots(rb_ractor_t *r)
 {
     if (r->postmortem) {
         /* The final self collection: everything else -- the Thread and Fiber
-         * wrappers, stdio, stack leftovers -- is what it exists to reclaim. */
+         * wrappers, stdio, stack leftovers -- is what it exists to reclaim.
+         * Skipping the walk below cannot drop dying_th: postmortem runs on the
+         * Ractor's last thread, which can only run after any predecessor's
+         * epilogue cleared the slot (under the same scheduler lock). */
+        VM_ASSERT(RUBY_ATOMIC_PTR_LOAD(r->threads.dying_th) == NULL);
         rb_ractor_mark_terminated_join_value(r);
         rb_gc_mark_vm_stack_values((long)r->registered_marks_cnt, r->registered_marks);
         return;
@@ -761,6 +780,8 @@ rb_ractor_terminate_atfork(rb_vm_t *vm, rb_ractor_t *r)
     rb_gc_ractor_cache_free(r->newobj_cache);
     r->newobj_cache = NULL;
     r->status_ = ractor_terminated;
+    // a termination epilogue in the parent did not survive the fork
+    r->threads.dying_th = NULL;
     /* In a forked child every other Ractor is terminated-unjoined, so keep its objspace
      * enumerable until a join or a global GC merges it. */
     if (r->objspace) {
@@ -779,6 +800,8 @@ rb_ractor_living_threads_init(rb_ractor_t *r)
     r->threads.cnt = 0;
     r->threads.blocking_cnt = 0;
     r->threads.terminating = false;
+    // atfork: a sibling's termination epilogue did not survive the fork
+    r->threads.dying_th = NULL;
 }
 
 static void

ractor_core.h

@@ -101,6 +101,9 @@ struct rb_ractor_struct {
         struct rb_thread_sched sched;
         rb_execution_context_t *running_ec;
         rb_thread_t *main;
+        // MN termination epilogue: keeps the dying thread marked (like a set
+        // member) between leaving the living set and its last use
+        rb_thread_t *dying_th;
 
         // `main` is in rb_thread_terminate_all(), waiting for the others to go
         bool terminating;

test/mkmf/test_depend.rb

@@ -42,6 +42,7 @@ def test_update_extension_dependencies
         # AUTOGENERATED DEPENDENCIES START
         example.o: $(srcdir)/../shared.h
         example.o: example.c
+        example.o: local.h
         # AUTOGENERATED DEPENDENCIES END
       DEPEND
     end

test/ruby/test_array.rb

@@ -1018,6 +1018,12 @@ def test_flatten!
     assert_equal(@cls[1, 2, 3, 4, 5, 6], a5)
   end
 
+  def test_flatten_bang_does_not_freeze_nested_array
+    child = []
+    [child].flatten!
+    assert_not_predicate(child, :frozen?)
+  end
+
   def test_flatten_empty!
     assert_nil(@cls[].flatten!)
     assert_equal(@cls[],

test/ruby/test_box.rb

@@ -567,7 +567,7 @@ def test_global_variables
     assert_equal nil, $,
 
     # used only in box
-    assert_not_include? global_variables, :$used_only_in_box
+    assert_not_include global_variables, :$used_only_in_box
     @box::UniqueGvar.write(123)
     assert_equal 123, @box::UniqueGvar.read
     assert_nil $used_only_in_box

thread_pthread.c

@@ -449,6 +449,19 @@ thread_sched_unlock_(struct rb_thread_sched *sched, rb_thread_t *th, const char
     rb_native_mutex_unlock(&sched->lock_);
 }
 
+#if USE_MN_THREADS
+// Like thread_sched_unlock(), but never dereferences th (the debug log above
+// reads th->serial).  For the MN termination epilogue, which unlocks after th
+// may already be collectable.  Keep in sync with thread_sched_unlock_.
+static void
+thread_sched_unlock_no_log(struct rb_thread_sched *sched, rb_thread_t *th)
+{
+    thread_sched_set_unlocked(sched, th); // pointer compare only
+
+    rb_native_mutex_unlock(&sched->lock_);
+}
+#endif
+
 static void
 ASSERT_thread_sched_locked(struct rb_thread_sched *sched, rb_thread_t *th)
 {

thread_pthread_mn.c

@@ -978,24 +978,23 @@ coroutine_thread_terminated(rb_thread_t *th)
     // GET_VM()). Make destruct wait until the reclaim finished. (Observed:
     // an assert_separately child exiting right after a Ractor finished
     // crashed at GET_VM()->default_params, offset 0x2600, on two arches.)
-    RUBY_ATOMIC_INC(th->vm->ractor.sched.winding_cnt);
+    rb_vm_t *const vm = th->vm; // survives th; the tail below must not read th
+    RUBY_ATOMIC_INC(vm->ractor.sched.winding_cnt);
 
     rb_thread_t *wake_th;
-
-    // Leave the living set BEFORE handing over the scheduler slot: the
-    // removal's VM-lock work (ractor_check_blocking, a barrier join) then
-    // runs as an ordinary counted running thread. Afterwards th may be
-    // unreachable, but no GC can complete while th still owns the slot
-    // (a barrier waits for it to join), so the handoff below may keep
-    // touching th/sched.
-    //
-    // The Ractor's last thread is the exception and keeps the reverse
-    // order (below): its removal unlinks the Ractor itself, after which
-    // r/sched must not be touched. That order is safe only for it: with
-    // no successor, sched->running stays NULL, so the removal's VM lock
-    // never joins a barrier (vm_need_barrier requires a running thread).
-    VM_ASSERT(sched->running == th); // th owns the slot through the removal
-    if (!last) rb_ractor_living_threads_remove(r, th);
+    bool wake_mn = false;
+
+    // Leave the living set here, while th is still barrier-registered and no
+    // successor can run: the GC's root scan walks r->threads.set without the
+    // Ractor lock, so the unlink must not race with it.  Off the set th would
+    // be unreachable although the handoff below keeps using it (and the GC can
+    // run: to_dead_common() deregisters th, so no barrier waits for it) --
+    // dying_th keeps it marked until its last use.
+    VM_ASSERT(sched->running == th); // th owns the slot through the handoff
+    if (!last) {
+        RUBY_ATOMIC_PTR_SET(r->threads.dying_th, th);
+        rb_ractor_living_threads_remove(r, th);
+    }
 
     thread_sched_lock(sched, th);
     {
@@ -1007,20 +1006,32 @@ coroutine_thread_terminated(rb_thread_t *th)
         // epilogue (below). If readyq was empty, running is now NULL and a
         // waker (e.g. the timer thread) that later installs a runnable
         // thread enqueues the Ractor itself -- enqueuing "whatever is
-        // running" at that point would duplicate its entry. While running
-        // is non-NULL, nobody else re-assigns it, so wake_th stays valid
-        // until we enqueue.
+        // running" at that point would duplicate its entry.
         wake_th = is_dnt ? NULL : sched->running;
+        // Read wake_th->nt under the lock: a dedicated successor was already
+        // woken by to_dead_common and may die (freeing wake_th) as soon as we
+        // unlock.  An M:N successor (nt == NULL) cannot run or be assigned an
+        // nt before our enqueue below, so the value cannot go stale.
+        wake_mn = (wake_th != NULL && wake_th->nt == NULL);
 
         tctx->nt = th->nt;        // stash the final transfer target for co_start
         native_thread_assign(NULL, th);
         th->sched.context = NULL; // the wrapper's dfree must not reclaim tctx
-    }
-    thread_sched_unlock(sched, th);
 
+        if (!last) {
+            // Still under the sched lock: a successor (even a dedicated one
+            // woken by to_dead_common) starts by taking it, so it cannot
+            // observe or overwrite these until we unlock.  th was last used
+            // above and running_ec no longer points into it; now it may be
+            // collected.
+            rb_ractor_set_current_ec(r, NULL); // r alive: it has other threads
+            VM_ASSERT(RUBY_ATOMIC_PTR_LOAD(r->threads.dying_th) == th);
+            RUBY_ATOMIC_PTR_SET(r->threads.dying_th, NULL);
+        }
+    }
     if (last) {
-        // The reverse order is safe only with no successor: running == NULL
-        // means the removal's VM lock cannot join a barrier (vm_need_barrier).
+        thread_sched_unlock(sched, th); // th is still on the living set here
+
         VM_ASSERT(sched->running == NULL);
         VM_ASSERT(wake_th == NULL);
         // Last access to th/r: the removal may unlink the Ractor, after
@@ -1029,13 +1040,15 @@ coroutine_thread_terminated(rb_thread_t *th)
         rb_current_ec_set(NULL); // TLS only; r may be collectable already
     }
     else {
-        rb_ractor_set_current_ec(r, NULL); // r alive: it has other threads
+        // th lost its root at the clear above; the plain unlock's debug log
+        // would read th->serial.
+        thread_sched_unlock_no_log(sched, th);
 
-        if (wake_th && wake_th->nt == NULL) {
+        if (wake_mn) {
             // enqueue the successor designated above -- exactly once per
             // "runnable but unserved" period, by its designator.
             thread_sched_lock(sched, NULL);
-            ractor_sched_enq(wake_th->vm, r);
+            ractor_sched_enq(vm, r);
             thread_sched_unlock(sched, NULL);
         }
     }
@@ -1109,6 +1122,7 @@ native_thread_create_shared(rb_thread_t *th)
     struct rb_thread_context *tctx = ruby_xmalloc(sizeof(struct rb_thread_context));
     tctx->stack = machine_stack;
     tctx->dead = false;
+    tctx->nt = NULL;
     th->sched.context = &tctx->co;
     coroutine_initialize(&tctx->co, co_start, machine_stack, machine_stack_size);
     tctx->co.argument = th;

tool/sync_default_gems.rb

@@ -6,6 +6,7 @@
 require "rbconfig"
 require "find"
 require "tempfile"
+require_relative "../lib/mkmf/depend"
 
 module SyncDefaultGems
   include FileUtils
@@ -46,7 +47,6 @@ def rewrite_for_ruby(path)
 
   def repo((upstream, branch), mappings, exclude: [])
     branch ||= CLASSICAL_DEFAULT_BRANCH
-    exclude += ["ext/**/depend"]
     Repository.new(upstream:, branch:, mappings:, exclude:)
   end
 
@@ -194,7 +194,6 @@ def lib((upstream, branch), gemspec_in_subdir: false)
       ["History.md", "ext/openssl/History.md"],
     ], exclude: [
       "test/openssl/envutil.rb",
-      "ext/openssl/depend",
     ]),
     optparse: lib("ruby/optparse", gemspec_in_subdir: true).tap {
       it.mappings << ["doc/optparse", "doc/optparse"]
@@ -395,6 +394,23 @@ def rubygems_do_fixup
     end
   end
 
+  def minimize_dependencies(gem)
+    files = REPOSITORIES[gem].mappings.flat_map do |_src, dst|
+      if File.file?(dst)
+        File.basename(dst) == "depend" ? [dst] : []
+      elsif File.directory?(dst)
+        Dir.glob("#{dst}/**/depend")
+      else
+        []
+      end
+    end.uniq
+    return if files.empty?
+
+    MakeMakefile::Depend.new(root: Dir.pwd).run(
+      files, mode: :inplace, sources: true,
+    )
+  end
+
   # We usually don't use this. Please consider using #sync_default_gems_with_commits instead.
   def sync_default_gems(gem)
     config = REPOSITORIES[gem]
@@ -438,6 +454,7 @@ def sync_default_gems(gem)
     if gem == "rubygems"
       rubygems_do_fixup
     end
+    minimize_dependencies(gem)
 
     check_prerelease_version(gem)
 
@@ -653,6 +670,7 @@ def fixup_commit(gem, commit)
       if gem == "rubygems"
         rubygems_do_fixup
       end
+      minimize_dependencies(gem)
       replace_rdoc_ref_all_full
     end
 

tool/test/test_mkdepend.rb

@@ -940,6 +940,7 @@ def test_update_extension_expands_make_variables_in_manual_rules
 
         #{MARK_START}
         example.o: example.c
+        example.o: local.h
         #{MARK_END}
       DEPEND
     end