From 6ac2c3dc56f3c420e56d40e6060b89cc4cf3b95c Mon Sep 17 00:00:00 2001 From: "Yukihiro \"Matz\" Matsumoto" Date: Thu, 21 May 2026 17:04:08 +0900 Subject: [PATCH] mruby-bigint: maintain canonical sn=0 when trim reduces sz to 0 mpz_t treats sn (sign) as the canonical "is zero" flag (zero_p(x) := (x)->sn == 0). When an arithmetic operation produces a value whose limbs trim to zero size, sn must be reset to 0 to preserve the invariant. Several call sites already enforced this locally (e.g. mpz_sub line 616); make trim() responsible so every caller benefits. Without this, an inconsistent zero bignum (sn!=0, sz=0) can flow into mpz_sqr, miss the zero_p guard, and reach mpz_init_heap with hint=0 where mpn_zero(NULL, 0) invokes UB (memset() declares its first argument nonnull). The trip survives at runtime on glibc but is formally undefined behavior, flagged by UBSan via Integer#pow(b,e,m) with specific operands. close #6849 Co-authored-by: Claude --- mrbgems/mruby-bigint/core/bigint.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mrbgems/mruby-bigint/core/bigint.c b/mrbgems/mruby-bigint/core/bigint.c index 549b9df19..5c64c7d87 100644 --- a/mrbgems/mruby-bigint/core/bigint.c +++ b/mrbgems/mruby-bigint/core/bigint.c @@ -373,6 +373,8 @@ trim(mpz_t *x) while (x->sz && x->p[x->sz-1] == 0) { x->sz--; } + /* Maintain invariant: sz == 0 implies sn == 0 (zero is canonical). */ + if (x->sz == 0) x->sn = 0; } /* z = x + y, without regard for sign */