string.c: guard memcmp() call to avoid undefined behavior with NULL pointers

passing NULL pointers to memcmp() is undefined behavior per C standard,
even when size is 0. memcmp() is declared with nonnull attributes,
and ASAN can detect this violation.

in mrb_str_cmp(), when comparing two empty strings or when the minimum
length is 0, we now skip the memcmp() call and directly set retval to 0.
this avoids the undefined behavior while maintaining correct comparison
semantics.

Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Yukihiro "Matz" Matsumoto
2025-10-22 22:47:39 +09:00
parent db3d754261
commit 6122fdfee9
+10 -1
View File
@@ -1200,7 +1200,16 @@ mrb_str_cmp(mrb_state *mrb, mrb_value str1, mrb_value str2)
mrb_int len1 = RSTR_LEN(s1);
mrb_int len2 = RSTR_LEN(s2);
mrb_int len = lesser(len1, len2);
mrb_int retval = memcmp(RSTR_PTR(s1), RSTR_PTR(s2), len);
mrb_int retval;
/* avoid UB: memcmp requires non-NULL pointers even when size is 0 */
if (len == 0) {
retval = 0;
}
else {
retval = memcmp(RSTR_PTR(s1), RSTR_PTR(s2), len);
}
if (retval == 0) {
if (len1 == len2) return 0;
if (len1 > len2) return 1;