mirror of
https://github.com/mruby/mruby
synced 2026-06-08 16:11:16 +00:00
ccb62ceb57
Replaces each maximal run of invalid UTF-8 bytes with a replacement string (U+FFFD by default), returning a valid UTF-8 copy. Mirrors CRuby's String#scrub (Feature #6752) -- the recovery counterpart to the existing String#valid_encoding? detection API. Validation matches utf8code() in src/string.c after the RFC 3629 / Unicode D93b conformance fixup (#2708): overlong encodings, UTF-16 surrogates, and codepoints above U+10FFFF are all treated as invalid. This is stricter than the existing mrb_utf8len()-based check used by valid_encoding?, so a string can report valid_encoding? = true and still get scrubbed; aligning valid_encoding? is a follow-up. The block form lives in mrblib on top of two C primitives -- __scrub and __scrub_chunks -- to avoid VM re-entry from C per CLAUDE.md. Non-String block return values are coerced via to_s (CRuby raises TypeError instead; the choice is locked in by test). Closes #6859. Co-authored-by: Claude <noreply@anthropic.com>
197 lines
4.9 KiB
Ruby
197 lines
4.9 KiB
Ruby
class String
|
|
|
|
##
|
|
# Call the given block for each character of
|
|
# `self`.
|
|
def each_char(&block)
|
|
return to_enum(:each_char) unless block
|
|
pos = 0
|
|
while pos < self.size
|
|
block.call(self[pos])
|
|
pos += 1
|
|
end
|
|
self
|
|
end
|
|
|
|
##
|
|
# call-seq:
|
|
# str.chars -> array
|
|
# str.chars {|char| block } -> str
|
|
#
|
|
# Returns an array of characters in str when called without a block.
|
|
# When called with a block, passes each character to the block.
|
|
#
|
|
# "hello".chars #=> ["h", "e", "l", "l", "o"]
|
|
# "hello".chars {|c| print c } #=> "hello"
|
|
#
|
|
def chars(&block)
|
|
if block_given?
|
|
__chars.each(&block)
|
|
self
|
|
else
|
|
__chars
|
|
end
|
|
end
|
|
|
|
def codepoints(&block)
|
|
cp = __codepoints()
|
|
if block_given?
|
|
cp.each do|x|
|
|
block.call(x)
|
|
end
|
|
self
|
|
else
|
|
cp
|
|
end
|
|
end
|
|
alias each_codepoint codepoints
|
|
|
|
##
|
|
# call-seq:
|
|
# string.lines -> array of string
|
|
# string.lines {|s| block} -> array of string
|
|
#
|
|
# Returns strings per line;
|
|
#
|
|
# a = "abc\ndef"
|
|
# a.lines #=> ["abc\n", "def"]
|
|
#
|
|
# If a block is given, it works the same as `each_line`.
|
|
def lines(&blk)
|
|
lines = self.__lines
|
|
if blk
|
|
lines.each do |line|
|
|
blk.call(line)
|
|
end
|
|
self
|
|
else
|
|
lines
|
|
end
|
|
end
|
|
|
|
##
|
|
# call-seq:
|
|
# str.upto(other_str, exclusive=false) {|s| block } -> str
|
|
# str.upto(other_str, exclusive=false) -> an_enumerator
|
|
#
|
|
# Iterates through successive values, starting at *str* and
|
|
# ending at *other_str* inclusive, passing each value in turn to
|
|
# the block. The `String#succ` method is used to generate
|
|
# each value. If optional second argument exclusive is omitted or is false,
|
|
# the last value will be included; otherwise it will be excluded.
|
|
#
|
|
# If no block is given, an enumerator is returned instead.
|
|
#
|
|
# "a8".upto("b6") {|s| print s, ' ' }
|
|
# for s in "a8".."b6"
|
|
# print s, ' '
|
|
# end
|
|
#
|
|
# <em>produces:</em>
|
|
#
|
|
# a8 a9 b0 b1 b2 b3 b4 b5 b6
|
|
# a8 a9 b0 b1 b2 b3 b4 b5 b6
|
|
#
|
|
# If *str* and *other_str* contains only ascii numeric characters,
|
|
# both are recognized as decimal numbers. In addition, the width of
|
|
# string (e.g. leading zeros) is handled appropriately.
|
|
#
|
|
# "9".upto("11").to_a #=> ["9", "10", "11"]
|
|
# "25".upto("5").to_a #=> []
|
|
# "07".upto("11").to_a #=> ["07", "08", "09", "10", "11"]
|
|
def upto(max, exclusive=false, &block)
|
|
return to_enum(:upto, max, exclusive) unless block
|
|
raise TypeError, "no implicit conversion of #{max.class} into String" unless max.kind_of? String
|
|
|
|
len = self.length
|
|
maxlen = max.length
|
|
# single character
|
|
if len == 1 and maxlen == 1
|
|
c = self.ord
|
|
e = max.ord
|
|
while c <= e
|
|
break if exclusive and c == e
|
|
yield c.chr(__ENCODING__)
|
|
c += 1
|
|
end
|
|
return self
|
|
end
|
|
# both edges are all digits
|
|
bi = self.to_i(10)
|
|
ei = max.to_i(10)
|
|
if (bi > 0 or bi == "0"*len) and (ei > 0 or ei == "0"*maxlen)
|
|
while bi <= ei
|
|
break if exclusive and bi == ei
|
|
s = bi.to_s
|
|
s = s.rjust(len, "0") if s.length < len
|
|
yield s
|
|
bi += 1
|
|
end
|
|
return self
|
|
end
|
|
bs = self
|
|
while true
|
|
n = (bs <=> max)
|
|
break if n > 0
|
|
break if exclusive and n == 0
|
|
yield bs
|
|
break if n == 0
|
|
bsiz = bs.size
|
|
break if bsiz > max.size || bsiz == 0
|
|
bs = bs.succ
|
|
end
|
|
self
|
|
end
|
|
|
|
def __upto_endless(&block)
|
|
len = self.length
|
|
# both edges are all digits
|
|
bi = self.to_i(10)
|
|
if bi > 0 or bi == "0"*len
|
|
while true
|
|
s = bi.to_s
|
|
s = s.rjust(len, "0") if s.length < len
|
|
yield s
|
|
bi += 1
|
|
end
|
|
return self
|
|
end
|
|
bs = self
|
|
while true
|
|
yield bs
|
|
bs = bs.succ
|
|
end
|
|
self
|
|
end
|
|
|
|
##
|
|
# call-seq:
|
|
# str.scrub -> new_str
|
|
# str.scrub(repl) -> new_str
|
|
# str.scrub {|bytes| block } -> new_str
|
|
#
|
|
# Returns a copy of +self+ with each maximal run of invalid UTF-8 bytes
|
|
# replaced by +repl+ (U+FFFD if +repl+ is omitted), or by the value
|
|
# returned from the block when one is given. The block receives the
|
|
# invalid bytes as a String.
|
|
#
|
|
# "abc\x80def".scrub #=> "abc\u{FFFD}def"
|
|
# "abc\x80def".scrub("?") #=> "abc?def"
|
|
# "\xE3\x81".scrub #=> "\u{FFFD}"
|
|
# "\x80\x81".scrub { |b| b.bytes.map { |c| "<%02X>" % c }.join }
|
|
# #=> "<80><81>"
|
|
def scrub(repl = nil, &block)
|
|
return __scrub(repl) unless block
|
|
chunks = __scrub_chunks
|
|
return chunks[0] if chunks.length == 1
|
|
result = chunks[0].dup
|
|
i = 1
|
|
while i < chunks.length
|
|
result << yield(chunks[i]).to_s
|
|
result << chunks[i + 1] if i + 1 < chunks.length
|
|
i += 2
|
|
end
|
|
result
|
|
end
|
|
end
|