From 06d6d0b0a5cbfa4ef6e1ec9665fcdb033c271a48 Mon Sep 17 00:00:00 2001 From: "Yukihiro \"Matz\" Matsumoto" Date: Wed, 4 Feb 2026 23:38:36 +0900 Subject: [PATCH] bigint.c: fix memory leak in powm with oversized modulus Barrett and Montgomery reduction compute 2^(2k) internally where k is the modulus bit length. When this exceeds MRB_BIGINT_BIT_LIMIT, mrb_raise() via longjmp skips cleanup of allocated temporaries. Add early modulus size check before any heap allocation. Co-authored-by: Claude --- mrbgems/mruby-bigint/core/bigint.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/mrbgems/mruby-bigint/core/bigint.c b/mrbgems/mruby-bigint/core/bigint.c index 57b7cd7a4..4ef5319f6 100644 --- a/mrbgems/mruby-bigint/core/bigint.c +++ b/mrbgems/mruby-bigint/core/bigint.c @@ -4514,6 +4514,15 @@ mpz_powm(mpz_ctx_t *ctx, mpz_t *zz, mpz_t *x, mpz_t *ex, mpz_t *n) return; } + /* Check modulus size before allocating large temporaries. */ + { + size_t mod_bits = (size_t)n->sz * DIG_SIZE; + if (mod_bits > MRB_BIGINT_BIT_LIMIT / 2) { + mrb_state *mrb = MPZ_MRB(ctx); + mrb_raise(mrb, E_RANGE_ERROR, "modulus too large"); + } + } + /* * Use Montgomery reduction for odd moduli >= 4 limbs. * Montgomery is faster because it replaces division with multiplication. @@ -4592,6 +4601,17 @@ mpz_powm_i(mpz_ctx_t *ctx, mpz_t *zz, mpz_t *x, mrb_int ex, mpz_t *n) return; } + /* Check modulus size before allocating large temporaries. + * Both Barrett and Montgomery need 2^(2k) internally, + * which would exceed MRB_BIGINT_BIT_LIMIT for large moduli. */ + { + size_t mod_bits = (size_t)n->sz * DIG_SIZE; + if (mod_bits > MRB_BIGINT_BIT_LIMIT / 2) { + mrb_state *mrb = MPZ_MRB(ctx); + mrb_raise(mrb, E_RANGE_ERROR, "modulus too large"); + } + } + /* * Use Montgomery reduction for odd moduli (common in cryptography). * Convert integer exponent to mpz_t and use the main Montgomery implementation.