diff --git a/mrbgems/mruby-numeric-ext/src/numeric_ext.c b/mrbgems/mruby-numeric-ext/src/numeric_ext.c index 1c80a5055..13159b75d 100644 --- a/mrbgems/mruby-numeric-ext/src/numeric_ext.c +++ b/mrbgems/mruby-numeric-ext/src/numeric_ext.c @@ -84,16 +84,20 @@ mrb_value mrb_int_pow(mrb_state *mrb, mrb_value x, mrb_value y); static mrb_int mrb_int_gcd(mrb_int x, mrb_int y) { - if (x < 0) x = -x; - if (y < 0) y = -y; + /* Negate via unsigned so MRB_INT_MIN doesn't overflow. + The cast back at the end produces MRB_INT_MIN only when the + true result is 2^63 (gcd of MRB_INT_MIN with itself or 0); + callers detect that case from the negative return value. */ + mrb_uint ux = (x < 0) ? -(mrb_uint)x : (mrb_uint)x; + mrb_uint uy = (y < 0) ? -(mrb_uint)y : (mrb_uint)y; - while (y != 0) { - mrb_int temp = y; - y = x % y; - x = temp; + while (uy != 0) { + mrb_uint temp = uy; + uy = ux % uy; + ux = temp; } - return x; + return (mrb_int)ux; } /* @@ -122,7 +126,11 @@ int_gcd(mrb_state *mrb, mrb_value x) if (!mrb_integer_p(y)) { mrb_raisef(mrb, E_TYPE_ERROR, "can't convert %Y into Integer", y); } - return mrb_int_value(mrb, mrb_int_gcd(mrb_integer(x), mrb_integer(y))); + mrb_int g = mrb_int_gcd(mrb_integer(x), mrb_integer(y)); + /* g < 0 only when the mathematical result is 2^63 (= |MRB_INT_MIN|), + which does not fit in mrb_int. */ + if (g < 0) mrb_int_overflow(mrb, "gcd"); + return mrb_int_value(mrb, g); } /* @@ -158,6 +166,10 @@ int_lcm(mrb_state *mrb, mrb_value x) if (a == 0 || b == 0) return mrb_int_value(mrb, 0); + /* Negation of MRB_INT_MIN is UB and the lcm with any non-zero + operand would not fit in mrb_int anyway. */ + if (a == MRB_INT_MIN || b == MRB_INT_MIN) mrb_int_overflow(mrb, "lcm"); + gcd_val = mrb_int_gcd(a, b); if (a < 0) a = -a; if (b < 0) b = -b;