From 8955c2c6d55eedfe05506fc7f9d85c72ca5b1ed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ja=C3=ABl=20Champagne=20Gareau?= Date: Sun, 5 Apr 2026 11:11:35 -0400 Subject: [PATCH] Fix failing `just_ascii` test (#2661) * fix just_ascii errors in test/benchmark sources * make just_ascii test run in CI --- .github/workflows/ubuntu24.yml | 2 +- CMakeLists.txt | 21 ++++++++++++------- benchmark/partial_tweets/tweet.h | 2 +- .../static_reflection_edge_cases_tests.cpp | 4 ++-- tests/dom/big_integer_tests.cpp | 2 +- tests/fractured_json_tests.cpp | 10 ++++----- 6 files changed, 24 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ubuntu24.yml b/.github/workflows/ubuntu24.yml index 163550400..3ca0c96c9 100644 --- a/.github/workflows/ubuntu24.yml +++ b/.github/workflows/ubuntu24.yml @@ -22,4 +22,4 @@ jobs: - name: Build run: cmake --build build -j=2 - name: Test - run: ctest --output-on-failure --test-dir build \ No newline at end of file + run: ctest --output-on-failure --test-dir build diff --git a/CMakeLists.txt b/CMakeLists.txt index 1363cdf03..c99bd7205 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -411,18 +411,25 @@ add_subdirectory(fuzz) # # Source files should be just ASCII # -find_program(FIND find) -find_program(FILE file) -find_program(GREP grep) -if(FIND AND FILE AND GREP) +find_program(FIND_CMD find) +find_program(FILE_CMD file) +find_program(GREP_CMD grep) +if(FIND_CMD AND FILE_CMD AND GREP_CMD) add_test( NAME just_ascii COMMAND sh -c "\ -${FIND} include src windows tools singleheader tests examples benchmark \ --path benchmark/checkperf-reference -prune -name '*.h' -o -name '*.cpp' \ --type f -exec ${FILE} '{}' \; | ${GREP} -qv ASCII || exit 0 && exit 1" +non_ascii=$(${FIND_CMD} include src windows tools singleheader tests examples benchmark \ + -path benchmark/checkperf-reference -prune -name '*.h' -o -name '*.cpp' \ + -type f -exec ${FILE_CMD} '{}' \; | ${GREP_CMD} -v ASCII); \ +if [ -n \"$non_ascii\" ]; then \ + echo 'The following files contain non-ASCII characters:'; \ + echo \"$non_ascii\"; \ + exit 1; \ +fi" WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" ) +else() + message(WARNING "just_ascii test disabled because required tools were not found: find='${FIND_CMD}', file='${FILE_CMD}', grep='${GREP_CMD}'") endif() ## diff --git a/benchmark/partial_tweets/tweet.h b/benchmark/partial_tweets/tweet.h index 3cfbbaa61..733207005 100644 --- a/benchmark/partial_tweets/tweet.h +++ b/benchmark/partial_tweets/tweet.h @@ -10,7 +10,7 @@ namespace partial_tweets { // { // "created_at": "Sun Aug 31 00:29:15 +0000 2014", // "id": 505874924095815700, -// "text": "@aym0566x \n\n名前:前田あゆみ\n第一印象:なんか怖っ！\n今の印象:とりあえずキモい。噛み合わない\n好きなところ:ぶすでキモいとこ😋✨✨\n思い出:んーーー、ありすぎ😊❤️\nLINE交換できる？:あぁ……ごめん✋\nトプ画をみて:照れますがな😘✨\n一言:お前は一生もんのダチ💖", +// "text": "@aym0566x ...", // "in_reply_to_status_id": null, // "user": { // "id": 1186275104, diff --git a/tests/builder/static_reflection_edge_cases_tests.cpp b/tests/builder/static_reflection_edge_cases_tests.cpp index 6adb65a37..ae22d7c7c 100644 --- a/tests/builder/static_reflection_edge_cases_tests.cpp +++ b/tests/builder/static_reflection_edge_cases_tests.cpp @@ -67,7 +67,7 @@ namespace builder_tests { test.quotes = "He said \"Hello\""; test.backslashes = "Path\\to\\file"; test.newlines = "Line1\nLine2\tTabbed"; - test.unicode = "Café résumé"; + test.unicode = "Caf\xc3\xa9 r\xc3\xa9sum\xc3\xa9"; test.null_char = '\0'; std::string json; @@ -213,4 +213,4 @@ namespace builder_tests { int main(int argc, char *argv[]) { return test_main(argc, argv, builder_tests::run); -} \ No newline at end of file +} diff --git a/tests/dom/big_integer_tests.cpp b/tests/dom/big_integer_tests.cpp index 98249a685..4da58260f 100644 --- a/tests/dom/big_integer_tests.cpp +++ b/tests/dom/big_integer_tests.cpp @@ -79,7 +79,7 @@ namespace big_integer_tests { std::string output = simdjson::to_string(doc); // Raw digits should appear ASSERT_TRUE(output.find("123456789012345678901") != std::string::npos); - // Must not be quoted — it's a number, not a string + // Must not be quoted - it's a number, not a string ASSERT_EQUAL(output.find("\"123456789012345678901\""), std::string::npos); TEST_SUCCEED(); } diff --git a/tests/fractured_json_tests.cpp b/tests/fractured_json_tests.cpp index 1ab2f56c2..371b3fb68 100644 --- a/tests/fractured_json_tests.cpp +++ b/tests/fractured_json_tests.cpp @@ -475,11 +475,11 @@ bool unicode_test() { std::cout << "Running " << __func__ << std::endl; const char* json = R"({ - "greeting": "Hello, 世界!", - "emoji": "🎉🚀✨", - "arabic": "مرحبا", - "russian": "Привет", - "mixed": "café résumé naïve" + "greeting": "Hello, \u4e16\u754c!", + "emoji": "\ud83c\udf89\ud83d\ude80\u2728", + "arabic": "\u0645\u0631\u062d\u0628\u0627", + "russian": "\u041f\u0440\u0438\u0432\u0435\u0442", + "mixed": "caf\u00e9 r\u00e9sum\u00e9 na\u00efve" })"; simdjson::dom::parser parser;