add symbol table overflow check

Since raising an error might intern a few new strings, some symbols need
to be reserved. 8 should be sufficient.

If the real limit has been reached, mrb_bug() is called.

include/mruby.h

@@ -161,6 +161,7 @@ typedef struct mrb_state {
   mrb_bool gc_full:1;
   mrb_bool is_generational_gc_mode:1;
   mrb_bool out_of_memory:1;
+  mrb_bool symbol_table_overflow:1;
   size_t majorgc_old_threshold;
   struct alloca_header *mems;
 

src/symbol.c

@@ -5,6 +5,7 @@
 */
 
 #include <ctype.h>
+#include <limits.h>
 #include <string.h>
 #include "mruby.h"
 #include "mruby/khash.h"
@@ -34,6 +35,15 @@ sym_hash_func(mrb_state *mrb, const symbol_name s)
 KHASH_DECLARE(n2s, symbol_name, mrb_sym, TRUE)
 KHASH_DEFINE (n2s, symbol_name, mrb_sym, TRUE, sym_hash_func, sym_hash_equal)
 /* ------------------------------------------------------ */
+
+#define MRB_SYM_MAX SHRT_MAX
+
+static mrb_value
+sym_tbl_overflow_new_str(mrb_state *mrb, const char *name, size_t len)
+{
+  return mrb_str_inspect(mrb, mrb_str_new(mrb, name, len));
+}
+
 static mrb_sym
 sym_intern(mrb_state *mrb, const char *name, size_t len, mrb_bool lit)
 {
@@ -53,6 +63,19 @@ sym_intern(mrb_state *mrb, const char *name, size_t len, mrb_bool lit)
   if (k != kh_end(h))
     return kh_value(h, k);
 
+  if (mrb->symbol_table_overflow) {
+    if (mrb->symidx == MRB_SYM_MAX) {
+      mrb_bug(mrb, "symbol table overflow (symbol %S)", sym_tbl_overflow_new_str(mrb, name, len));
+    }
+  }
+  else {
+    if (mrb->symidx >= MRB_SYM_MAX - 8) { /* raising might intern a few new strings */
+      mrb->symbol_table_overflow = TRUE;
+      mrb_raisef(mrb, E_RUNTIME_ERROR, "symbol table overflow (symbol %S)",
+        sym_tbl_overflow_new_str(mrb, name, len));
+    }
+  }
+
   sym = ++mrb->symidx;
   if (lit) {
     sname.name = name;