hash.c: add recursion detection to prevent SystemStackError; fix #5531

Add generalized recursion detection system and integrate it into Hash#==
and Hash#eql? to prevent infinite recursion with mutually recursive hash
structures. Uses call stack inspection for minimal memory overhead.

Co-authored-by: Claude <noreply@anthropic.com>

src/hash.c

@@ -15,6 +15,7 @@
 #include <mruby/internal.h>
 #include <mruby/presym.h>
 
+
 /*
  * === Glossary
  *
@@ -2194,6 +2195,11 @@ mrb_hash_equal(mrb_state *mrb, mrb_value hash)
     return mrb_false_value();
   }
 
+  /* Check for recursion */
+  if (MRB_RECURSIVE_BINARY_P(mrb, MRB_OPSYM(eq), hash, hash2)) {
+    return mrb_false_value();
+  }
+
   struct RHash *h1 = mrb_hash_ptr(hash);
   struct RHash *h2 = mrb_hash_ptr(hash2);
 
@@ -2237,6 +2243,11 @@ mrb_hash_eql(mrb_state *mrb, mrb_value hash)
     return mrb_false_value();
   }
 
+  /* Check for recursion */
+  if (MRB_RECURSIVE_BINARY_P(mrb, MRB_SYM_Q(eql), hash, hash2)) {
+    return mrb_false_value();
+  }
+
   struct RHash *h1 = mrb_hash_ptr(hash);
   struct RHash *h2 = mrb_hash_ptr(hash2);
 

src/kernel.c

@@ -114,6 +114,30 @@ inspect_recursive_p(mrb_state *mrb, mrb_value obj, int n)
   return FALSE;
 }
 
+MRB_API mrb_bool
+mrb_recursive_method_p(mrb_state *mrb, mrb_sym mid, mrb_value obj1, mrb_value obj2)
+{
+  for (mrb_callinfo *ci=&mrb->c->ci[-1]; ci>=mrb->c->cibase; ci--) {
+    if (ci->mid == mid && mrb_obj_eq(mrb, obj1, ci->stack[0])) {
+      /* For unary methods, only check first argument */
+      if (mrb_nil_p(obj2)) return TRUE;
+
+      /* For binary methods, check both arguments */
+      if (mrb_obj_eq(mrb, obj2, ci->stack[1])) return TRUE;
+    }
+  }
+  return FALSE;
+}
+
+#define MRB_RECURSIVE_P(mrb, mid, obj1, obj2) \
+  mrb_recursive_method_p(mrb, mid, obj1, obj2)
+
+#define MRB_RECURSIVE_UNARY_P(mrb, mid, obj) \
+  mrb_recursive_method_p(mrb, mid, obj, mrb_nil_value())
+
+#define MRB_RECURSIVE_BINARY_P(mrb, mid, obj1, obj2) \
+  mrb_recursive_method_p(mrb, mid, obj1, obj2)
+
 mrb_bool
 mrb_inspect_recursive_p(mrb_state *mrb, mrb_value obj)
 {