Protect exception for mruby core initialization

src/error.c

@@ -490,6 +490,26 @@ mrb_no_method_error(mrb_state *mrb, mrb_sym id, mrb_value args, char const* fmt,
   mrb_exc_raise(mrb, exc);
 }
 
+int
+mrb_core_init_protect(mrb_state *mrb, void (*body)(mrb_state *, void *), void *opaque)
+{
+  struct mrb_jmpbuf *prev_jmp = mrb->jmp;
+  struct mrb_jmpbuf c_jmp;
+  int err = 1;
+
+  MRB_TRY(&c_jmp) {
+    mrb->jmp = &c_jmp;
+    body(mrb, opaque);
+    err = 0;
+  } MRB_CATCH(&c_jmp) {
+    mrb->exc = NULL;
+  } MRB_END_EXC(&c_jmp);
+
+  mrb->jmp = prev_jmp;
+
+  return err;
+}
+
 mrb_noreturn void
 mrb_core_init_abort(mrb_state *mrb)
 {

src/state.c

@@ -19,11 +19,25 @@ void mrb_init_mrbgems(mrb_state*);
 void mrb_gc_init(mrb_state*, mrb_gc *gc);
 void mrb_gc_destroy(mrb_state*, mrb_gc *gc);
 
+int mrb_core_init_protect(mrb_state *mrb, void (*body)(mrb_state *, void *), void *opaque);
+
+static void
+init_gc_and_core(mrb_state *mrb, void *opaque)
+{
+  static const struct mrb_context mrb_context_zero = { 0 };
+
+  mrb_gc_init(mrb, &mrb->gc);
+  mrb->c = (struct mrb_context*)mrb_malloc(mrb, sizeof(struct mrb_context));
+  *mrb->c = mrb_context_zero;
+  mrb->root_c = mrb->c;
+
+  mrb_init_core(mrb);
+}
+
 MRB_API mrb_state*
 mrb_open_core(mrb_allocf f, void *ud)
 {
   static const mrb_state mrb_state_zero = { 0 };
-  static const struct mrb_context mrb_context_zero = { 0 };
   mrb_state *mrb;
 
   mrb = (mrb_state *)(f)(NULL, NULL, sizeof(mrb_state), ud);
@@ -34,12 +48,10 @@ mrb_open_core(mrb_allocf f, void *ud)
   mrb->allocf = f;
   mrb->atexit_stack_len = 0;
 
-  mrb_gc_init(mrb, &mrb->gc);
-  mrb->c = (struct mrb_context*)mrb_malloc(mrb, sizeof(struct mrb_context));
-  *mrb->c = mrb_context_zero;
-  mrb->root_c = mrb->c;
-
-  mrb_init_core(mrb);
+  if (mrb_core_init_protect(mrb, init_gc_and_core, NULL)) {
+    mrb_close(mrb);
+    return NULL;
+  }
 
 #if !defined(MRB_DISABLE_STDIO) && defined(_MSC_VER) && _MSC_VER < 1900
   _set_output_format(_TWO_DIGIT_EXPONENT);
@@ -100,6 +112,12 @@ mrb_open(void)
   return mrb;
 }
 
+static void
+init_mrbgems(mrb_state *mrb, void *opaque)
+{
+  mrb_init_mrbgems(mrb);
+}
+
 MRB_API mrb_state*
 mrb_open_allocf(mrb_allocf f, void *ud)
 {
@@ -110,7 +128,10 @@ mrb_open_allocf(mrb_allocf f, void *ud)
   }
 
 #ifndef DISABLE_GEMS
-  mrb_init_mrbgems(mrb);
+  if (mrb_core_init_protect(mrb, init_mrbgems, NULL)) {
+    mrb_close(mrb);
+    return NULL;
+  }
   mrb_gc_arena_restore(mrb, 0);
 #endif
   return mrb;