rational.c: fix undefined behavior from large shift exponents

In rational_new_f(), the code performed ((mrb_int)1)<<exp without
checking if exp >= MRB_INT_BIT. Shifting by a value >= bit width
is undefined behavior in C.

Also fixed the negative exponent case which incorrectly used
deno >>= exp (right-shift by negative is UB). The correct logic
is deno <<= -exp to multiply denominator by 2^(-exp).

Both cases now check for overflow before shifting and fall back
to bigint operations when necessary.

Discovered via ClusterFuzz with input "92r**11".

Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Yukihiro "Matz" Matsumoto
2025-12-30 14:28:44 +09:00
parent ff0e20453f
commit 5eca2fae1e
+13 -3
View File
@@ -342,7 +342,8 @@ rational_new_f(mrb_state *mrb, mrb_float f)
if (exp > 0) {
mrb_int temp;
if (mrb_int_mul_overflow(nume, ((mrb_int)1)<<exp, &temp)) {
/* Check exp < MRB_INT_BIT to avoid undefined behavior from shifting */
if (exp >= MRB_INT_BIT || mrb_int_mul_overflow(nume, ((mrb_int)1)<<exp, &temp)) {
#ifndef RAT_BIGINT
rat_overflow(mrb);
#else
@@ -352,8 +353,17 @@ rational_new_f(mrb_state *mrb, mrb_float f)
}
nume = temp;
}
else {
deno >>= exp;
else if (exp < 0) {
/* exp is negative, so we need to multiply denominator by 2^(-exp) */
int neg_exp = -exp;
if (neg_exp >= MRB_INT_BIT || mrb_int_mul_overflow(deno, ((mrb_int)1)<<neg_exp, &deno)) {
#ifndef RAT_BIGINT
rat_overflow(mrb);
#else
mrb_value d = mrb_bint_lshift(mrb, mrb_bint_new_int(mrb, deno), neg_exp);
return rational_new_b(mrb, mrb_int_value(mrb, nume), d);
#endif
}
}
return rational_new_i(mrb, nume, deno);
}