bigint.c: fix memory leak in powm with oversized modulus

Barrett and Montgomery reduction compute 2^(2k) internally where k
is the modulus bit length. When this exceeds MRB_BIGINT_BIT_LIMIT,
mrb_raise() via longjmp skips cleanup of allocated temporaries.
Add early modulus size check before any heap allocation.

Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Yukihiro "Matz" Matsumoto
2026-02-04 23:38:36 +09:00
parent edce0a338f
commit 06d6d0b0a5
+20
View File
@@ -4514,6 +4514,15 @@ mpz_powm(mpz_ctx_t *ctx, mpz_t *zz, mpz_t *x, mpz_t *ex, mpz_t *n)
return;
}
/* Check modulus size before allocating large temporaries. */
{
size_t mod_bits = (size_t)n->sz * DIG_SIZE;
if (mod_bits > MRB_BIGINT_BIT_LIMIT / 2) {
mrb_state *mrb = MPZ_MRB(ctx);
mrb_raise(mrb, E_RANGE_ERROR, "modulus too large");
}
}
/*
* Use Montgomery reduction for odd moduli >= 4 limbs.
* Montgomery is faster because it replaces division with multiplication.
@@ -4592,6 +4601,17 @@ mpz_powm_i(mpz_ctx_t *ctx, mpz_t *zz, mpz_t *x, mrb_int ex, mpz_t *n)
return;
}
/* Check modulus size before allocating large temporaries.
* Both Barrett and Montgomery need 2^(2k) internally,
* which would exceed MRB_BIGINT_BIT_LIMIT for large moduli. */
{
size_t mod_bits = (size_t)n->sz * DIG_SIZE;
if (mod_bits > MRB_BIGINT_BIT_LIMIT / 2) {
mrb_state *mrb = MPZ_MRB(ctx);
mrb_raise(mrb, E_RANGE_ERROR, "modulus too large");
}
}
/*
* Use Montgomery reduction for odd moduli (common in cryptography).
* Convert integer exponent to mpz_t and use the main Montgomery implementation.