mirror of
https://github.com/yaml/libyaml
synced 2026-06-08 18:28:36 +00:00
Complete buffering and encoding code.
This commit is contained in:
Kirill Simonov
@@ -300,6 +300,9 @@ typedef struct {
|
||||
/** The byte about which the problem occured. */
|
||||
size_t problem_offset;
|
||||
|
||||
/** The problematic value (@c -1 is none). */
|
||||
int problem_value;
|
||||
|
||||
/**
|
||||
* @}
|
||||
*/
|
||||
|
||||
+78
-59
@@ -12,11 +12,13 @@
|
||||
*/
|
||||
|
||||
int
|
||||
yaml_parser_set_reader_error(yaml_parser_t *parser, const char *problem)
|
||||
yaml_parser_set_reader_error(yaml_parser_t *parser, const char *problem,
|
||||
size_t offset, int value)
|
||||
{
|
||||
parser->error = YAML_READER_ERROR;
|
||||
parser->problem = problem;
|
||||
parser->problem_offset = parser->offset;
|
||||
parser->problem_offset = offset;
|
||||
parser->problem_value = value;
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -32,9 +34,9 @@ yaml_parser_set_reader_error(yaml_parser_t *parser, const char *problem)
|
||||
int
|
||||
yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
|
||||
{
|
||||
/* If the EOF flag is set, do nothing. */
|
||||
/* If the EOF flag is set and the raw buffer is empty, do nothing. */
|
||||
|
||||
if (parser->eof)
|
||||
if (parser->eof && !parser->raw_unread)
|
||||
return 1;
|
||||
|
||||
/* Return if the buffer contains enough characters. */
|
||||
@@ -71,18 +73,14 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
|
||||
|
||||
if (!yaml_parser_update_raw_buffer(parser)) return 0;
|
||||
|
||||
/* If the raw buffer is empty, it is EOF. */
|
||||
|
||||
if (!parser->raw_unread) return 1;
|
||||
|
||||
/* Decode the raw buffer. */
|
||||
|
||||
while (parser->raw_unread)
|
||||
{
|
||||
unsigned int value, value2;
|
||||
int incomplete = 0;
|
||||
unsigned char utf8_octet;
|
||||
unsigned int utf8_length;
|
||||
unsigned char octet;
|
||||
unsigned int width;
|
||||
int k, low, high;
|
||||
|
||||
/* Decode the next character. */
|
||||
@@ -113,25 +111,26 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
|
||||
|
||||
/* Determine the length of the UTF-8 sequence. */
|
||||
|
||||
utf8_octet = parser->raw_pointer[0];
|
||||
utf8_length = (
|
||||
(utf8_octet & 0x80) == 0x00 ? 1 :
|
||||
(utf8_octet & 0xE0) == 0xC0 ? 2 :
|
||||
(utf8_octet & 0xF0) == 0xE0 ? 3 :
|
||||
(utf8_octet & 0xF8) == 0xF0 ? 4 : 0);
|
||||
octet = parser->raw_pointer[0];
|
||||
width = (octet & 0x80) == 0x00 ? 1 :
|
||||
(octet & 0xE0) == 0xC0 ? 2 :
|
||||
(octet & 0xF0) == 0xE0 ? 3 :
|
||||
(octet & 0xF8) == 0xF0 ? 4 : 0;
|
||||
|
||||
/* Check if the leading octet is valid. */
|
||||
|
||||
if (!utf8_length)
|
||||
if (!width)
|
||||
return yaml_parser_set_reader_error(parser,
|
||||
"Invalid leading UTF-8 octet");
|
||||
"Invalid leading UTF-8 octet",
|
||||
parser->offset, octet);
|
||||
|
||||
/* Check if the raw buffer contains an incomplete character. */
|
||||
|
||||
if (utf8_length > parser->raw_unread) {
|
||||
if (width > parser->raw_unread) {
|
||||
if (parser->eof) {
|
||||
return yaml_parser_set_reader_error(parser,
|
||||
"Incomplete UTF-8 octet sequence");
|
||||
"Incomplete UTF-8 octet sequence",
|
||||
parser->offset, -1);
|
||||
}
|
||||
incomplete = 1;
|
||||
break;
|
||||
@@ -139,47 +138,45 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
|
||||
|
||||
/* Decode the leading octet. */
|
||||
|
||||
value = (
|
||||
(utf8_octet & 0x80) == 0x00 ? utf8_octet & 0x7F :
|
||||
(utf8_octet & 0xE0) == 0xC0 ? utf8_octet & 0x1F :
|
||||
(utf8_octet & 0xF0) == 0xE0 ? utf8_octet & 0x0F :
|
||||
(utf8_octet & 0xF8) == 0xF0 ? utf8_octet & 0x07 : 0);
|
||||
value = (octet & 0x80) == 0x00 ? octet & 0x7F :
|
||||
(octet & 0xE0) == 0xC0 ? octet & 0x1F :
|
||||
(octet & 0xF0) == 0xE0 ? octet & 0x0F :
|
||||
(octet & 0xF8) == 0xF0 ? octet & 0x07 : 0;
|
||||
|
||||
/* Check and decode the trailing octets. */
|
||||
|
||||
for (k = 1; k < utf8_length; k ++)
|
||||
for (k = 1; k < width; k ++)
|
||||
{
|
||||
utf8_octet = parser->raw_pointer[k];
|
||||
octet = parser->raw_pointer[k];
|
||||
|
||||
/* Check if the octet is valid. */
|
||||
|
||||
if ((utf8_octet & 0xC0) != 0x80)
|
||||
if ((octet & 0xC0) != 0x80)
|
||||
return yaml_parser_set_reader_error(parser,
|
||||
"Invalid trailing UTF-8 octet");
|
||||
"Invalid trailing UTF-8 octet",
|
||||
parser->offset+k, octet);
|
||||
|
||||
/* Decode the octet. */
|
||||
|
||||
value = (value << 6) + (utf8_octet & 0x3F);
|
||||
value = (value << 6) + (octet & 0x3F);
|
||||
}
|
||||
|
||||
/* Check the length of the sequence against the value. */
|
||||
|
||||
if (!((utf8_length == 1) ||
|
||||
(utf8_length == 2 && value >= 0x80) ||
|
||||
(utf8_length == 3 && value >= 0x800) ||
|
||||
(utf8_length == 4 && value >= 0x10000)))
|
||||
if (!((width == 1) ||
|
||||
(width == 2 && value >= 0x80) ||
|
||||
(width == 3 && value >= 0x800) ||
|
||||
(width == 4 && value >= 0x10000)))
|
||||
return yaml_parser_set_reader_error(parser,
|
||||
"Invalid length of a UTF-8 sequence");
|
||||
"Invalid length of a UTF-8 sequence",
|
||||
parser->offset, -1);
|
||||
|
||||
/* Check the range of the value. */
|
||||
|
||||
if ((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF)
|
||||
return yaml_parser_set_reader_error(parser,
|
||||
"Invalid Unicode character");
|
||||
|
||||
parser->raw_pointer += utf8_length;
|
||||
parser->raw_unread -= utf8_length;
|
||||
parser->offset += utf8_length;
|
||||
"Invalid Unicode character",
|
||||
parser->offset, value);
|
||||
|
||||
break;
|
||||
|
||||
@@ -220,7 +217,8 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
|
||||
if (parser->raw_unread < 2) {
|
||||
if (parser->eof) {
|
||||
return yaml_parser_set_reader_error(parser,
|
||||
"Incomplete UTF-16 character");
|
||||
"Incomplete UTF-16 character",
|
||||
parser->offset, -1);
|
||||
}
|
||||
incomplete = 1;
|
||||
break;
|
||||
@@ -235,18 +233,22 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
|
||||
|
||||
if ((value & 0xFC00) == 0xDC00)
|
||||
return yaml_parser_set_reader_error(parser,
|
||||
"Unexpected low surrogate area");
|
||||
"Unexpected low surrogate area",
|
||||
parser->offset, value);
|
||||
|
||||
/* Check for a high surrogate area. */
|
||||
|
||||
if ((value & 0xFC00) == 0xD800) {
|
||||
|
||||
width = 4;
|
||||
|
||||
/* Check for incomplete surrogate pair. */
|
||||
|
||||
if (parser->raw_unread < 4) {
|
||||
if (parser->eof) {
|
||||
return yaml_parser_set_reader_error(parser,
|
||||
"Incomplete UTF-16 surrogate pair");
|
||||
"Incomplete UTF-16 surrogate pair",
|
||||
parser->offset, -1);
|
||||
}
|
||||
incomplete = 1;
|
||||
break;
|
||||
@@ -261,21 +263,16 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
|
||||
|
||||
if ((value2 & 0xFC00) != 0xDC00)
|
||||
return yaml_parser_set_reader_error(parser,
|
||||
"Expected low surrogate area");
|
||||
"Expected low surrogate area",
|
||||
parser->offset+2, value2);
|
||||
|
||||
/* Generate the value of the surrogate pair. */
|
||||
|
||||
value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF);
|
||||
|
||||
parser->raw_pointer += 4;
|
||||
parser->raw_unread -= 4;
|
||||
parser->offset += 4;
|
||||
}
|
||||
|
||||
else {
|
||||
parser->raw_pointer += 2;
|
||||
parser->raw_unread -= 2;
|
||||
parser->offset += 4;
|
||||
width = 2;
|
||||
}
|
||||
|
||||
break;
|
||||
@@ -298,7 +295,14 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
|
||||
|| (value >= 0xE000 && value <= 0xFFFD)
|
||||
|| (value >= 0x10000 && value <= 0x10FFFF)))
|
||||
return yaml_parser_set_reader_error(parser,
|
||||
"Control characters are not allowed");
|
||||
"Control characters are not allowed",
|
||||
parser->offset, value);
|
||||
|
||||
/* Move the raw pointers. */
|
||||
|
||||
parser->raw_pointer += width;
|
||||
parser->raw_unread -= width;
|
||||
parser->offset += width;
|
||||
|
||||
/* Finally put the character into the buffer. */
|
||||
|
||||
@@ -309,22 +313,33 @@ yaml_parser_update_buffer(yaml_parser_t *parser, size_t length)
|
||||
/* 0000 0080-0000 07FF -> 110xxxxx 10xxxxxx */
|
||||
else if (value <= 0x7FF) {
|
||||
*(parser->buffer_end++) = 0xC0 + (value >> 6);
|
||||
*(parser->buffer_end++) = 0x80 + value & 0x3F;
|
||||
*(parser->buffer_end++) = 0x80 + (value & 0x3F);
|
||||
}
|
||||
/* 0000 0800-0000 FFFF -> 1110xxxx 10xxxxxx 10xxxxxx */
|
||||
else if (value <= 0xFFFF) {
|
||||
*(parser->buffer_end++) = 0xE0 + (value >> 12);
|
||||
*(parser->buffer_end++) = 0x80 + (value >> 6) & 0x3F;
|
||||
*(parser->buffer_end++) = 0x80 + value & 0x3F;
|
||||
*(parser->buffer_end++) = 0x80 + ((value >> 6) & 0x3F);
|
||||
*(parser->buffer_end++) = 0x80 + (value & 0x3F);
|
||||
}
|
||||
/* 0001 0000-0010 FFFF -> 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */
|
||||
else {
|
||||
*(parser->buffer_end++) = 0xF0 + (value >> 18);
|
||||
*(parser->buffer_end++) = 0x80 + (value >> 12) & 0x3F;
|
||||
*(parser->buffer_end++) = 0x80 + (value >> 6) & 0x3F;
|
||||
*(parser->buffer_end++) = 0x80 + value & 0x3F;
|
||||
*(parser->buffer_end++) = 0x80 + ((value >> 12) & 0x3F);
|
||||
*(parser->buffer_end++) = 0x80 + ((value >> 6) & 0x3F);
|
||||
*(parser->buffer_end++) = 0x80 + (value & 0x3F);
|
||||
}
|
||||
|
||||
parser->unread ++;
|
||||
}
|
||||
|
||||
/* On EOF, put NUL into the buffer and return. */
|
||||
|
||||
if (parser->eof) {
|
||||
*(parser->buffer_end++) = '\0';
|
||||
parser->unread ++;
|
||||
return 1;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return 1;
|
||||
@@ -357,18 +372,21 @@ yaml_parser_determine_encoding(yaml_parser_t *parser)
|
||||
parser->encoding = YAML_UTF16LE_ENCODING;
|
||||
parser->raw_pointer += 2;
|
||||
parser->raw_unread -= 2;
|
||||
parser->offset += 2;
|
||||
}
|
||||
else if (parser->raw_unread >= 2
|
||||
&& !memcmp(parser->raw_pointer, BOM_UTF16BE, 2)) {
|
||||
parser->encoding = YAML_UTF16BE_ENCODING;
|
||||
parser->raw_pointer += 2;
|
||||
parser->raw_unread -= 2;
|
||||
parser->offset += 2;
|
||||
}
|
||||
else if (parser->raw_unread >= 3
|
||||
&& !memcmp(parser->raw_pointer, BOM_UTF8, 3)) {
|
||||
parser->encoding = YAML_UTF8_ENCODING;
|
||||
parser->raw_pointer += 3;
|
||||
parser->raw_unread -= 3;
|
||||
parser->offset += 3;
|
||||
}
|
||||
else {
|
||||
parser->encoding = YAML_UTF8_ENCODING;
|
||||
@@ -407,7 +425,8 @@ yaml_parser_update_raw_buffer(yaml_parser_t *parser)
|
||||
parser->raw_buffer + parser->raw_unread,
|
||||
YAML_RAW_BUFFER_SIZE - parser->raw_unread,
|
||||
&size_read)) {
|
||||
return yaml_parser_set_reader_error(parser, "Input error");
|
||||
return yaml_parser_set_reader_error(parser, "Input error",
|
||||
parser->offset, -1);
|
||||
}
|
||||
parser->raw_unread += size_read;
|
||||
if (!size_read) {
|
||||
|
||||
+268
-62
@@ -16,84 +16,96 @@ typedef struct {
|
||||
} test_case;
|
||||
|
||||
test_case utf8_sequences[] = {
|
||||
/* {"title", "test 1|test 2|...|test N!", (0 or 1)}, */
|
||||
/* {"title", "test 1|test 2|...|test N!", (0 or 1)}, */
|
||||
|
||||
{"a simple test", "'test' is '\xd0\xbf\xd1\x80\xd0\xbe\xd0\xb2\xd0\xb5\xd1\x80\xd0\xba\xd0\xb0' in Russian!", 1},
|
||||
{"an empty line", "!", 1},
|
||||
{"a simple test", "'test' is '\xd0\xbf\xd1\x80\xd0\xbe\xd0\xb2\xd0\xb5\xd1\x80\xd0\xba\xd0\xb0' in Russian!", 1},
|
||||
{"an empty line", "!", 1},
|
||||
|
||||
{"u-0 is a control character", "\x00!", 0},
|
||||
{"u-80 is a control character", "\xc2\x80!", 0},
|
||||
{"u-800 is valid", "\xe0\xa0\x80!", 1},
|
||||
{"u-10000 is valid", "\xf0\x90\x80\x80!", 1},
|
||||
{"5 bytes sequences are not allowed", "\xf8\x88\x80\x80\x80!", 0},
|
||||
{"6 bytes sequences are not allowed", "\xfc\x84\x80\x80\x80\x80!", 0},
|
||||
{"u-0 is a control character", "\x00!", 0},
|
||||
{"u-80 is a control character", "\xc2\x80!", 0},
|
||||
{"u-800 is valid", "\xe0\xa0\x80!", 1},
|
||||
{"u-10000 is valid", "\xf0\x90\x80\x80!", 1},
|
||||
{"5 bytes sequences are not allowed", "\xf8\x88\x80\x80\x80!", 0},
|
||||
{"6 bytes sequences are not allowed", "\xfc\x84\x80\x80\x80\x80!", 0},
|
||||
|
||||
{"u-7f is a control character", "\x7f!", 0},
|
||||
{"u-7FF is valid", "\xdf\xbf!", 1},
|
||||
{"u-FFFF is a control character", "\xef\xbf\xbf!", 0},
|
||||
{"u-1FFFFF is too large", "\xf7\xbf\xbf\xbf!", 0},
|
||||
{"u-3FFFFFF is 5 bytes", "\xfb\xbf\xbf\xbf\xbf!", 0},
|
||||
{"u-7FFFFFFF is 6 bytes", "\xfd\xbf\xbf\xbf\xbf\xbf!", 0},
|
||||
{"u-7f is a control character", "\x7f!", 0},
|
||||
{"u-7FF is valid", "\xdf\xbf!", 1},
|
||||
{"u-FFFF is a control character", "\xef\xbf\xbf!", 0},
|
||||
{"u-1FFFFF is too large", "\xf7\xbf\xbf\xbf!", 0},
|
||||
{"u-3FFFFFF is 5 bytes", "\xfb\xbf\xbf\xbf\xbf!", 0},
|
||||
{"u-7FFFFFFF is 6 bytes", "\xfd\xbf\xbf\xbf\xbf\xbf!", 0},
|
||||
|
||||
{"u-D7FF", "\xed\x9f\xbf!", 1},
|
||||
{"u-E000", "\xee\x80\x80!", 1},
|
||||
{"u-FFFD", "\xef\xbf\xbd!", 1},
|
||||
{"u-10FFFF", "\xf4\x8f\xbf\xbf!", 1},
|
||||
{"u-110000", "\xf4\x90\x80\x80!", 0},
|
||||
{"u-D7FF", "\xed\x9f\xbf!", 1},
|
||||
{"u-E000", "\xee\x80\x80!", 1},
|
||||
{"u-FFFD", "\xef\xbf\xbd!", 1},
|
||||
{"u-10FFFF", "\xf4\x8f\xbf\xbf!", 1},
|
||||
{"u-110000", "\xf4\x90\x80\x80!", 0},
|
||||
|
||||
{"first continuation byte", "\x80!", 0},
|
||||
{"last continuation byte", "\xbf!", 0},
|
||||
{"first continuation byte", "\x80!", 0},
|
||||
{"last continuation byte", "\xbf!", 0},
|
||||
|
||||
{"2 continuation bytes", "\x80\xbf!", 0},
|
||||
{"3 continuation bytes", "\x80\xbf\x80!", 0},
|
||||
{"4 continuation bytes", "\x80\xbf\x80\xbf!", 0},
|
||||
{"5 continuation bytes", "\x80\xbf\x80\xbf\x80!", 0},
|
||||
{"6 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf!", 0},
|
||||
{"7 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf\x80!", 0},
|
||||
{"2 continuation bytes", "\x80\xbf!", 0},
|
||||
{"3 continuation bytes", "\x80\xbf\x80!", 0},
|
||||
{"4 continuation bytes", "\x80\xbf\x80\xbf!", 0},
|
||||
{"5 continuation bytes", "\x80\xbf\x80\xbf\x80!", 0},
|
||||
{"6 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf!", 0},
|
||||
{"7 continuation bytes", "\x80\xbf\x80\xbf\x80\xbf\x80!", 0},
|
||||
|
||||
{"sequence of all 64 possible continuation bytes",
|
||||
"\x80|\x81|\x82|\x83|\x84|\x85|\x86|\x87|\x88|\x89|\x8a|\x8b|\x8c|\x8d|\x8e|\x8f|"
|
||||
"\x90|\x91|\x92|\x93|\x94|\x95|\x96|\x97|\x98|\x99|\x9a|\x9b|\x9c|\x9d|\x9e|\x9f|"
|
||||
"\xa0|\xa1|\xa2|\xa3|\xa4|\xa5|\xa6|\xa7|\xa8|\xa9|\xaa|\xab|\xac|\xad|\xae|\xaf|"
|
||||
"\xb0|\xb1|\xb2|\xb3|\xb4|\xb5|\xb6|\xb7|\xb8|\xb9|\xba|\xbb|\xbc|\xbd|\xbe|\xbf!", 0},
|
||||
{"32 first bytes of 2-byte sequences {0xc0-0xdf}",
|
||||
"\xc0 |\xc1 |\xc2 |\xc3 |\xc4 |\xc5 |\xc6 |\xc7 |\xc8 |\xc9 |\xca |\xcb |\xcc |\xcd |\xce |\xcf |"
|
||||
"\xd0 |\xd1 |\xd2 |\xd3 |\xd4 |\xd5 |\xd6 |\xd7 |\xd8 |\xd9 |\xda |\xdb |\xdc |\xdd |\xde |\xdf !", 0},
|
||||
{"16 first bytes of 3-byte sequences {0xe0-0xef}",
|
||||
"\xe0 |\xe1 |\xe2 |\xe3 |\xe4 |\xe5 |\xe6 |\xe7 |\xe8 |\xe9 |\xea |\xeb |\xec |\xed |\xee |\xef !", 0},
|
||||
{"8 first bytes of 4-byte sequences {0xf0-0xf7}", "\xf0 |\xf1 |\xf2 |\xf3 |\xf4 |\xf5 |\xf6 |\xf7 !", 0},
|
||||
{"4 first bytes of 5-byte sequences {0xf8-0xfb}", "\xf8 |\xf9 |\xfa |\xfb !", 0},
|
||||
{"2 first bytes of 6-byte sequences {0xfc-0xfd}", "\xfc |\xfd !", 0},
|
||||
{"sequence of all 64 possible continuation bytes",
|
||||
"\x80|\x81|\x82|\x83|\x84|\x85|\x86|\x87|\x88|\x89|\x8a|\x8b|\x8c|\x8d|\x8e|\x8f|"
|
||||
"\x90|\x91|\x92|\x93|\x94|\x95|\x96|\x97|\x98|\x99|\x9a|\x9b|\x9c|\x9d|\x9e|\x9f|"
|
||||
"\xa0|\xa1|\xa2|\xa3|\xa4|\xa5|\xa6|\xa7|\xa8|\xa9|\xaa|\xab|\xac|\xad|\xae|\xaf|"
|
||||
"\xb0|\xb1|\xb2|\xb3|\xb4|\xb5|\xb6|\xb7|\xb8|\xb9|\xba|\xbb|\xbc|\xbd|\xbe|\xbf!", 0},
|
||||
{"32 first bytes of 2-byte sequences {0xc0-0xdf}",
|
||||
"\xc0 |\xc1 |\xc2 |\xc3 |\xc4 |\xc5 |\xc6 |\xc7 |\xc8 |\xc9 |\xca |\xcb |\xcc |\xcd |\xce |\xcf |"
|
||||
"\xd0 |\xd1 |\xd2 |\xd3 |\xd4 |\xd5 |\xd6 |\xd7 |\xd8 |\xd9 |\xda |\xdb |\xdc |\xdd |\xde |\xdf !", 0},
|
||||
{"16 first bytes of 3-byte sequences {0xe0-0xef}",
|
||||
"\xe0 |\xe1 |\xe2 |\xe3 |\xe4 |\xe5 |\xe6 |\xe7 |\xe8 |\xe9 |\xea |\xeb |\xec |\xed |\xee |\xef !", 0},
|
||||
{"8 first bytes of 4-byte sequences {0xf0-0xf7}", "\xf0 |\xf1 |\xf2 |\xf3 |\xf4 |\xf5 |\xf6 |\xf7 !", 0},
|
||||
{"4 first bytes of 5-byte sequences {0xf8-0xfb}", "\xf8 |\xf9 |\xfa |\xfb !", 0},
|
||||
{"2 first bytes of 6-byte sequences {0xfc-0xfd}", "\xfc |\xfd !", 0},
|
||||
|
||||
{"sequences with last byte missing {u-0}",
|
||||
"\xc0|\xe0\x80|\xf0\x80\x80|\xf8\x80\x80\x80|\xfc\x80\x80\x80\x80!", 0},
|
||||
{"sequences with last byte missing {u-...FF}",
|
||||
"\xdf|\xef\xbf|\xf7\xbf\xbf|\xfb\xbf\xbf\xbf|\xfd\xbf\xbf\xbf\xbf!", 0},
|
||||
{"sequences with last byte missing {u-0}",
|
||||
"\xc0|\xe0\x80|\xf0\x80\x80|\xf8\x80\x80\x80|\xfc\x80\x80\x80\x80!", 0},
|
||||
{"sequences with last byte missing {u-...FF}",
|
||||
"\xdf|\xef\xbf|\xf7\xbf\xbf|\xfb\xbf\xbf\xbf|\xfd\xbf\xbf\xbf\xbf!", 0},
|
||||
|
||||
{"impossible bytes", "\xfe|\xff|\xfe\xfe\xff\xff!", 0},
|
||||
{"impossible bytes", "\xfe|\xff|\xfe\xfe\xff\xff!", 0},
|
||||
|
||||
{"overlong sequences {u-2f}",
|
||||
"\xc0\xaf|\xe0\x80\xaf|\xf0\x80\x80\xaf|\xf8\x80\x80\x80\xaf|\xfc\x80\x80\x80\x80\xaf!", 0},
|
||||
{"overlong sequences {u-2f}",
|
||||
"\xc0\xaf|\xe0\x80\xaf|\xf0\x80\x80\xaf|\xf8\x80\x80\x80\xaf|\xfc\x80\x80\x80\x80\xaf!", 0},
|
||||
|
||||
{"maximum overlong sequences",
|
||||
"\xc1\xbf|\xe0\x9f\xbf|\xf0\x8f\xbf\xbf|\xf8\x87\xbf\xbf\xbf|\xfc\x83\xbf\xbf\xbf\xbf!", 0},
|
||||
{"maximum overlong sequences",
|
||||
"\xc1\xbf|\xe0\x9f\xbf|\xf0\x8f\xbf\xbf|\xf8\x87\xbf\xbf\xbf|\xfc\x83\xbf\xbf\xbf\xbf!", 0},
|
||||
|
||||
{"overlong representation of the NUL character",
|
||||
"\xc0\x80|\xe0\x80\x80|\xf0\x80\x80\x80|\xf8\x80\x80\x80\x80|\xfc\x80\x80\x80\x80\x80!", 0},
|
||||
{"overlong representation of the NUL character",
|
||||
"\xc0\x80|\xe0\x80\x80|\xf0\x80\x80\x80|\xf8\x80\x80\x80\x80|\xfc\x80\x80\x80\x80\x80!", 0},
|
||||
|
||||
{"single UTF-16 surrogates",
|
||||
"\xed\xa0\x80|\xed\xad\xbf|\xed\xae\x80|\xed\xaf\xbf|\xed\xb0\x80|\xed\xbe\x80|\xed\xbf\xbf!", 0},
|
||||
{"single UTF-16 surrogates",
|
||||
"\xed\xa0\x80|\xed\xad\xbf|\xed\xae\x80|\xed\xaf\xbf|\xed\xb0\x80|\xed\xbe\x80|\xed\xbf\xbf!", 0},
|
||||
|
||||
{"paired UTF-16 surrogates",
|
||||
"\xed\xa0\x80\xed\xb0\x80|\xed\xa0\x80\xed\xbf\xbf|\xed\xad\xbf\xed\xb0\x80|"
|
||||
"\xed\xad\xbf\xed\xbf\xbf|\xed\xae\x80\xed\xb0\x80|\xed\xae\x80\xed\xbf\xbf|"
|
||||
"\xed\xaf\xbf\xed\xb0\x80|\xed\xaf\xbf\xed\xbf\xbf!", 0},
|
||||
{"paired UTF-16 surrogates",
|
||||
"\xed\xa0\x80\xed\xb0\x80|\xed\xa0\x80\xed\xbf\xbf|\xed\xad\xbf\xed\xb0\x80|"
|
||||
"\xed\xad\xbf\xed\xbf\xbf|\xed\xae\x80\xed\xb0\x80|\xed\xae\x80\xed\xbf\xbf|"
|
||||
"\xed\xaf\xbf\xed\xb0\x80|\xed\xaf\xbf\xed\xbf\xbf!", 0},
|
||||
|
||||
{"other illegal code positions", "\xef\xbf\xbe|\xef\xbf\xbf!", 0},
|
||||
{"other illegal code positions", "\xef\xbf\xbe|\xef\xbf\xbf!", 0},
|
||||
|
||||
{NULL, NULL, 0}
|
||||
{NULL, NULL, 0}
|
||||
};
|
||||
|
||||
test_case boms[] = {
|
||||
|
||||
/* {"title", "test!", lenth}, */
|
||||
|
||||
{"no bom (utf-8)", "Hi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82!", 13},
|
||||
{"bom (utf-8)", "\xef\xbb\xbfHi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82!", 13},
|
||||
{"bom (utf-16-le)", "\xff\xfeH\x00i\x00 \x00i\x00s\x00 \x00\x1f\x04@\x04""8\x04""2\x04""5\x04""B\x04!", 13},
|
||||
{"bom (utf-16-be)", "\xfe\xff\x00H\x00i\x00 \x00i\x00s\x00 \x04\x1f\x04@\x04""8\x04""2\x04""5\x04""B!", 13}
|
||||
};
|
||||
|
||||
char *bom_original = "Hi is \xd0\x9f\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb5\xd1\x82";
|
||||
|
||||
int check_utf8_sequences(void)
|
||||
{
|
||||
yaml_parser_t *parser;
|
||||
@@ -124,7 +136,14 @@ int check_utf8_sequences(void)
|
||||
printf("(no error)\n");
|
||||
}
|
||||
else if (parser->error == YAML_READER_ERROR) {
|
||||
printf("(reader error: %s at %d)\n", parser->problem, parser->problem_offset);
|
||||
if (parser->problem_value != -1) {
|
||||
printf("(reader error: %s: #%X at %d)\n",
|
||||
parser->problem, parser->problem_value, parser->problem_offset);
|
||||
}
|
||||
else {
|
||||
printf("(reader error: %s at %d)\n",
|
||||
parser->problem, parser->problem_offset);
|
||||
}
|
||||
}
|
||||
if (*end == '!') break;
|
||||
start = ++end;
|
||||
@@ -136,9 +155,196 @@ int check_utf8_sequences(void)
|
||||
return failed;
|
||||
}
|
||||
|
||||
int check_boms(void)
|
||||
{
|
||||
yaml_parser_t *parser;
|
||||
int failed = 0;
|
||||
int k;
|
||||
printf("checking boms...\n");
|
||||
for (k = 0; boms[k].test; k++) {
|
||||
char *title = boms[k].title;
|
||||
int check = boms[k].result;
|
||||
int result;
|
||||
char *start = boms[k].test;
|
||||
char *end = start;
|
||||
while (*end != '!') end++;
|
||||
printf("\t%s: ", title);
|
||||
parser = yaml_parser_new();
|
||||
assert(parser);
|
||||
yaml_parser_set_input_string(parser, (unsigned char *)start, end-start);
|
||||
result = yaml_parser_update_buffer(parser, end-start);
|
||||
if (!result) {
|
||||
printf("- (reader error: %s at %d)\n", parser->problem, parser->problem_offset);
|
||||
failed++;
|
||||
}
|
||||
else {
|
||||
if (parser->unread != check) {
|
||||
printf("- (length=%d while expected length=%d)\n", parser->unread, check);
|
||||
failed++;
|
||||
}
|
||||
else if (memcmp(parser->buffer, bom_original, check) != 0) {
|
||||
printf("- (value '%s' does not equal to the original value '%s')\n", parser->buffer, bom_original);
|
||||
failed++;
|
||||
}
|
||||
else {
|
||||
printf("+\n");
|
||||
}
|
||||
}
|
||||
yaml_parser_delete(parser);
|
||||
}
|
||||
printf("checking boms: %d fail(s)\n", failed);
|
||||
return failed;
|
||||
}
|
||||
|
||||
#define LONG 100000
|
||||
|
||||
int check_long_utf8(void)
|
||||
{
|
||||
yaml_parser_t *parser;
|
||||
int k = 0;
|
||||
int j;
|
||||
int failed = 0;
|
||||
unsigned char ch0, ch1;
|
||||
unsigned char *buffer = malloc(3+LONG*2);
|
||||
assert(buffer);
|
||||
printf("checking a long utf8 sequence...\n");
|
||||
buffer[k++] = '\xef';
|
||||
buffer[k++] = '\xbb';
|
||||
buffer[k++] = '\xbf';
|
||||
for (j = 0; j < LONG; j ++) {
|
||||
if (j % 2) {
|
||||
buffer[k++] = '\xd0';
|
||||
buffer[k++] = '\x90';
|
||||
}
|
||||
else {
|
||||
buffer[k++] = '\xd0';
|
||||
buffer[k++] = '\xaf';
|
||||
}
|
||||
}
|
||||
parser = yaml_parser_new();
|
||||
assert(parser);
|
||||
yaml_parser_set_input_string(parser, buffer, 3+LONG*2);
|
||||
for (k = 0; k < LONG; k++) {
|
||||
if (!parser->unread) {
|
||||
if (!yaml_parser_update_buffer(parser, 1)) {
|
||||
printf("\treader error: %s at %d\n", parser->problem, parser->problem_offset);
|
||||
failed = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!parser->unread) {
|
||||
printf("\tnot enough characters at %d\n", k);
|
||||
failed = 1;
|
||||
break;
|
||||
}
|
||||
if (k % 2) {
|
||||
ch0 = '\xd0';
|
||||
ch1 = '\x90';
|
||||
}
|
||||
else {
|
||||
ch0 = '\xd0';
|
||||
ch1 = '\xaf';
|
||||
}
|
||||
if (parser->pointer[0] != ch0 || parser->pointer[1] != ch1) {
|
||||
printf("\tincorrect UTF-8 sequence: %X %X instead of %X %X\n",
|
||||
(int)parser->pointer[0], (int)parser->pointer[1],
|
||||
(int)ch0, (int)ch1);
|
||||
failed = 1;
|
||||
break;
|
||||
}
|
||||
parser->pointer += 2;
|
||||
parser->unread -= 1;
|
||||
}
|
||||
if (!failed) {
|
||||
if (!yaml_parser_update_buffer(parser, 1)) {
|
||||
printf("\treader error: %s at %d\n", parser->problem, parser->problem_offset);
|
||||
failed = 1;
|
||||
}
|
||||
else if (parser->pointer[0] != '\0') {
|
||||
printf("\texpected NUL, found %X (eof=%d, unread=%d)\n", (int)parser->pointer[0], parser->eof, parser->unread);
|
||||
failed = 1;
|
||||
}
|
||||
}
|
||||
yaml_parser_delete(parser);
|
||||
free(buffer);
|
||||
printf("checking a long utf8 sequence: %d fail(s)\n", failed);
|
||||
return failed;
|
||||
}
|
||||
|
||||
int check_long_utf16(void)
|
||||
{
|
||||
yaml_parser_t *parser;
|
||||
int k = 0;
|
||||
int j;
|
||||
int failed = 0;
|
||||
unsigned char ch0, ch1;
|
||||
unsigned char *buffer = malloc(2+LONG*2);
|
||||
assert(buffer);
|
||||
printf("checking a long utf16 sequence...\n");
|
||||
buffer[k++] = '\xff';
|
||||
buffer[k++] = '\xfe';
|
||||
for (j = 0; j < LONG; j ++) {
|
||||
if (j % 2) {
|
||||
buffer[k++] = '\x10';
|
||||
buffer[k++] = '\x04';
|
||||
}
|
||||
else {
|
||||
buffer[k++] = '/';
|
||||
buffer[k++] = '\x04';
|
||||
}
|
||||
}
|
||||
parser = yaml_parser_new();
|
||||
assert(parser);
|
||||
yaml_parser_set_input_string(parser, buffer, 2+LONG*2);
|
||||
for (k = 0; k < LONG; k++) {
|
||||
if (!parser->unread) {
|
||||
if (!yaml_parser_update_buffer(parser, 1)) {
|
||||
printf("\treader error: %s at %d\n", parser->problem, parser->problem_offset);
|
||||
failed = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!parser->unread) {
|
||||
printf("\tnot enough characters at %d\n", k);
|
||||
failed = 1;
|
||||
break;
|
||||
}
|
||||
if (k % 2) {
|
||||
ch0 = '\xd0';
|
||||
ch1 = '\x90';
|
||||
}
|
||||
else {
|
||||
ch0 = '\xd0';
|
||||
ch1 = '\xaf';
|
||||
}
|
||||
if (parser->pointer[0] != ch0 || parser->pointer[1] != ch1) {
|
||||
printf("\tincorrect UTF-8 sequence: %X %X instead of %X %X\n",
|
||||
(int)parser->pointer[0], (int)parser->pointer[1],
|
||||
(int)ch0, (int)ch1);
|
||||
failed = 1;
|
||||
break;
|
||||
}
|
||||
parser->pointer += 2;
|
||||
parser->unread -= 1;
|
||||
}
|
||||
if (!failed) {
|
||||
if (!yaml_parser_update_buffer(parser, 1)) {
|
||||
printf("\treader error: %s at %d\n", parser->problem, parser->problem_offset);
|
||||
failed = 1;
|
||||
}
|
||||
else if (parser->pointer[0] != '\0') {
|
||||
printf("\texpected NUL, found %X (eof=%d, unread=%d)\n", (int)parser->pointer[0], parser->eof, parser->unread);
|
||||
failed = 1;
|
||||
}
|
||||
}
|
||||
yaml_parser_delete(parser);
|
||||
free(buffer);
|
||||
printf("checking a long utf16 sequence: %d fail(s)\n", failed);
|
||||
return failed;
|
||||
}
|
||||
|
||||
int
|
||||
main(void)
|
||||
{
|
||||
return check_utf8_sequences();
|
||||
return check_utf8_sequences() + check_boms() + check_long_utf8() + check_long_utf16();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user