numeric.h: fix integer multiplication overflow check

The previous implementation of mrb_int_mul_overflow performed
the multiplication before checking for overflow. This is undefined
behavior for signed integers and can lead to incorrect results on
some compilers (e.g., MSVC).

The implementation has been changed to perform the overflow checks
before the multiplication.

Co-authored-by: Gemini <gemini@google.com>
This commit is contained in:
Yukihiro "Matz" Matsumoto
2025-08-23 07:39:45 +09:00
parent 164a7302b1
commit 9a7211bb25
+1 -1
View File
@@ -119,12 +119,12 @@ mrb_int_mul_overflow(mrb_int a, mrb_int b, mrb_int *c)
*c = (mrb_int)n;
return n > MRB_INT_MAX || n < MRB_INT_MIN;
#else /* MRB_INT64 */
*c = a * b;
if (a > 0 && b > 0 && a > MRB_INT_MAX / b) return TRUE;
if (a < 0 && b > 0 && a < MRB_INT_MIN / b) return TRUE;
if (a > 0 && b < 0 && b < MRB_INT_MIN / a) return TRUE;
if (a < 0 && b < 0 && (a <= MRB_INT_MIN || b <= MRB_INT_MIN || -a > MRB_INT_MAX / -b))
return TRUE;
*c = a * b;
return FALSE;
#endif
}