From 419c8ebfb296df5450fc7fc89b8573f8461a15be Mon Sep 17 00:00:00 2001
From: "Yukihiro \"Matz\" Matsumoto" <matz@ruby.or.jp>
Date: Mon, 7 Jul 2025 13:23:00 +0900
Subject: [PATCH] hash.c: add recursion detection to prevent SystemStackError;
 fix #5531

Add generalized recursion detection system and integrate it into Hash#==
and Hash#eql? to prevent infinite recursion with mutually recursive hash
structures. Uses call stack inspection for minimal memory overhead.

Co-authored-by: Claude <noreply@anthropic.com>
---
 include/mruby.h | 12 ++++++++++++
 src/hash.c      | 11 +++++++++++
 src/kernel.c    | 24 ++++++++++++++++++++++++
 3 files changed, 47 insertions(+)

diff --git a/include/mruby.h b/include/mruby.h
index e2fcb53fd..1fd525a41 100644
--- a/include/mruby.h
+++ b/include/mruby.h
@@ -1303,6 +1303,18 @@ MRB_API mrb_bool mrb_eql(mrb_state *mrb, mrb_value obj1, mrb_value obj2);
 /* mrb_cmp(mrb, obj1, obj2): 1:0:-1; -2 for error */
 MRB_API mrb_int mrb_cmp(mrb_state *mrb, mrb_value obj1, mrb_value obj2);
 
+/* recursion detection */
+MRB_API mrb_bool mrb_recursive_method_p(mrb_state *mrb, mrb_sym mid, mrb_value obj1, mrb_value obj2);
+
+#define MRB_RECURSIVE_P(mrb, mid, obj1, obj2) \
+  mrb_recursive_method_p(mrb, mid, obj1, obj2)
+
+#define MRB_RECURSIVE_UNARY_P(mrb, mid, obj) \
+  mrb_recursive_method_p(mrb, mid, obj, mrb_nil_value())
+
+#define MRB_RECURSIVE_BINARY_P(mrb, mid, obj1, obj2) \
+  mrb_recursive_method_p(mrb, mid, obj1, obj2)
+
 #define mrb_gc_arena_save(mrb) ((mrb)->gc.arena_idx)
 #define mrb_gc_arena_restore(mrb, idx) ((mrb)->gc.arena_idx = (idx))
 
diff --git a/src/hash.c b/src/hash.c
index b22bd2232..8f7e57dc8 100644
--- a/src/hash.c
+++ b/src/hash.c
@@ -15,6 +15,7 @@
 #include <mruby/internal.h>
 #include <mruby/presym.h>
 
+
 /*
  * === Glossary
  *
@@ -2194,6 +2195,11 @@ mrb_hash_equal(mrb_state *mrb, mrb_value hash)
     return mrb_false_value();
   }
 
+  /* Check for recursion */
+  if (MRB_RECURSIVE_BINARY_P(mrb, MRB_OPSYM(eq), hash, hash2)) {
+    return mrb_false_value();
+  }
+
   struct RHash *h1 = mrb_hash_ptr(hash);
   struct RHash *h2 = mrb_hash_ptr(hash2);
 
@@ -2237,6 +2243,11 @@ mrb_hash_eql(mrb_state *mrb, mrb_value hash)
     return mrb_false_value();
   }
 
+  /* Check for recursion */
+  if (MRB_RECURSIVE_BINARY_P(mrb, MRB_SYM_Q(eql), hash, hash2)) {
+    return mrb_false_value();
+  }
+
   struct RHash *h1 = mrb_hash_ptr(hash);
   struct RHash *h2 = mrb_hash_ptr(hash2);
 
diff --git a/src/kernel.c b/src/kernel.c
index 2aba21977..7a6fda973 100644
--- a/src/kernel.c
+++ b/src/kernel.c
@@ -114,6 +114,30 @@ inspect_recursive_p(mrb_state *mrb, mrb_value obj, int n)
   return FALSE;
 }
 
+MRB_API mrb_bool
+mrb_recursive_method_p(mrb_state *mrb, mrb_sym mid, mrb_value obj1, mrb_value obj2)
+{
+  for (mrb_callinfo *ci=&mrb->c->ci[-1]; ci>=mrb->c->cibase; ci--) {
+    if (ci->mid == mid && mrb_obj_eq(mrb, obj1, ci->stack[0])) {
+      /* For unary methods, only check first argument */
+      if (mrb_nil_p(obj2)) return TRUE;
+
+      /* For binary methods, check both arguments */
+      if (mrb_obj_eq(mrb, obj2, ci->stack[1])) return TRUE;
+    }
+  }
+  return FALSE;
+}
+
+#define MRB_RECURSIVE_P(mrb, mid, obj1, obj2) \
+  mrb_recursive_method_p(mrb, mid, obj1, obj2)
+
+#define MRB_RECURSIVE_UNARY_P(mrb, mid, obj) \
+  mrb_recursive_method_p(mrb, mid, obj, mrb_nil_value())
+
+#define MRB_RECURSIVE_BINARY_P(mrb, mid, obj1, obj2) \
+  mrb_recursive_method_p(mrb, mid, obj1, obj2)
+
 mrb_bool
 mrb_inspect_recursive_p(mrb_state *mrb, mrb_value obj)
 {