diff --git a/main.cpp b/main.cpp index 7b4051e74..48e6e0215 100644 --- a/main.cpp +++ b/main.cpp @@ -397,6 +397,9 @@ u32 tape_locs[MAX_DEPTH]; u8 string_buf[512*1024]; u8 * current_string_buf_loc; +u8 number_buf[512*1024]; // holds either doubles or longs, really +u8 * current_number_buf_loc; + // STATE MACHINE DECLARATIONS const u32 MAX_STATES = 16; @@ -461,6 +464,7 @@ never_inline bool ape_machine(const u8 * buf, UNUSED size_t len, ParsedJson & pj } current_string_buf_loc = string_buf; + current_number_buf_loc = number_buf; u32 error_sump = 0; u32 old_tape_loc = tape_locs[depth]; // need to initialize for first write @@ -675,6 +679,203 @@ really_inline bool parse_string(const u8 * buf, UNUSED size_t len, UNUSED Parsed return true; } +// put a parsed version of number (either as a double or a signed long) into the number buffer, +// put a 'tag' indicating which type and where it is back onto the tape at that location +// return false if we can't parse the number which means either +// (a) the number isn't valid, or (b) the number is followed by something that isn't whitespace, comma or a close }] character +// which are the only things that should follow a number at this stage +// bools to detect what we found in our initial character already here - we are already +// switching on 0 vs 1-9 vs - so we may as well keep separate paths where that's useful + +// TODO: see if we really need a separate number_buf or whether we should just +// have a generic scratch - would need to align before using for this +really_inline bool parse_number(const u8 * buf, UNUSED size_t len, UNUSED ParsedJson & pj, u32 tape_loc, UNUSED bool found_zero, bool found_minus) { + u32 offset = tape[tape_loc] & 0xffffff; + if (found_minus) { + offset++; + } + const u8 * src = &buf[offset]; + m256 v = _mm256_loadu_si256((const m256 *)(src)); + u64 error_sump = 0; +#ifdef DEBUG + for (u32 j = 0; j < 32; j++) { + char c = *(src+j); + if (isprint(c)) { + cout << c; + } else { + cout << '_'; + } + } + cout << "| ... number handling input\n"; +#endif + + // categories to extract + // Digits: + // 0 (0x30) - bucket 0 + // 1-9 (never any distinction except if we didn't get the free kick at 0 due to the leading minus) (0x31-0x39) - bucket 1 + // . (0x2e) - bucket 2 + // E or e - no distinction (0x45/0x65) - bucket 3 + // + (0x2b) - bucket 4 + // - (0x2d) - bucket 4 + // Terminators + // Whitespace: 0x20, 0x09, 0x0a, 0x0d - bucket 5+6 + // Comma and the closes: 0x2c is comma, } is 0x5d, ] is 0x7d - bucket 5+7 + + // Another shufti - also a bit hand-hacked. Need to make a better construction + const m256 low_nibble_mask = _mm256_setr_epi8( + // 0 1 2 3 4 5 6 7 8 9 a b c d e f + 33, 2, 2, 2, 2, 10, 2, 2, 2, 66, 64, 16, 32,208, 4, 0, + 33, 2, 2, 2, 2, 10, 2, 2, 2, 66, 64, 16, 32,208, 4, 0 + ); + const m256 high_nibble_mask = _mm256_setr_epi8( + // 0 1 2 3 4 5 6 7 8 9 a b c d e f + 64, 0, 52, 3, 8,128, 8,128, 0, 0, 0, 0, 0, 0, 0, 0, + 64, 0, 52, 3, 8,128, 8,128, 0, 0, 0, 0, 0, 0, 0, 0 + ); + + m256 tmp = _mm256_and_si256( + _mm256_shuffle_epi8(low_nibble_mask, v), + _mm256_shuffle_epi8(high_nibble_mask, + _mm256_and_si256(_mm256_srli_epi32(v, 4), _mm256_set1_epi8(0x7f)))); + + m256 enders_mask = _mm256_set1_epi8(0xe0); + m256 tmp_enders = _mm256_cmpeq_epi8(_mm256_and_si256(tmp, enders_mask), + _mm256_set1_epi8(0)); + u32 enders = ~(u32)_mm256_movemask_epi8(tmp_enders); + dumpbits32(enders, "ender characters"); + + if (enders == 0) { + // TODO: scream for help if enders == 0 which means we have + // a heroically long number string or some garbage + } + // TODO: make a mask that indicates where our digits are + u32 number_mask = ~enders & (enders-1); + dumpbits32(number_mask, "number mask"); + + m256 n_mask = _mm256_set1_epi8(0x1f); + m256 tmp_n = _mm256_cmpeq_epi8(_mm256_and_si256(tmp, n_mask), + _mm256_set1_epi8(0)); + u32 number_characters = ~(u32)_mm256_movemask_epi8(tmp_n); + + // put something into our error sump if we have something + // before our ending characters that isn't a valid character + // for the inside of our JSON + number_characters &= number_mask; + error_sump |= number_characters ^ number_mask; + dumpbits32(number_characters, "number characters"); + // TODO: prune by mask + // TODO: check that result of pruning by mask *is* the + // same as the mask + + m256 d_mask = _mm256_set1_epi8(0x03); + m256 tmp_d = _mm256_cmpeq_epi8(_mm256_and_si256(tmp, d_mask), + _mm256_set1_epi8(0)); + u32 digit_characters = ~(u32)_mm256_movemask_epi8(tmp_d); + digit_characters &= number_mask; + dumpbits32(digit_characters, "digit characters"); + + m256 p_mask = _mm256_set1_epi8(0x04); + m256 tmp_p = _mm256_cmpeq_epi8(_mm256_and_si256(tmp, p_mask), + _mm256_set1_epi8(0)); + u32 decimal_characters = ~(u32)_mm256_movemask_epi8(tmp_p); + decimal_characters &= number_mask; + dumpbits32(decimal_characters, "decimal characters"); + + m256 e_mask = _mm256_set1_epi8(0x08); + m256 tmp_e = _mm256_cmpeq_epi8(_mm256_and_si256(tmp, e_mask), + _mm256_set1_epi8(0)); + u32 exponent_characters = ~(u32)_mm256_movemask_epi8(tmp_e); + exponent_characters &= number_mask; + dumpbits32(exponent_characters, "exponent characters"); + + m256 s_mask = _mm256_set1_epi8(0x10); + m256 tmp_s = _mm256_cmpeq_epi8(_mm256_and_si256(tmp, s_mask), + _mm256_set1_epi8(0)); + u32 sign_characters = ~(u32)_mm256_movemask_epi8(tmp_s); + sign_characters &= number_mask; + dumpbits32(sign_characters, "sign characters"); + + u32 digit_edges = ~(digit_characters << 1) & digit_characters; + dumpbits32(digit_edges, "digit_edges"); + + // check that we have 1-3 'edges' only + u32 t = digit_edges; + t &= t-1; t &= t-1; t &= t-1; + error_sump |= t; + + // check that we start with a digit + error_sump |= ~digit_characters & 0x1; + + if (__builtin_popcount(digit_edges) == 1) { + // try a strtoll + char * end; + u64 result = strtoll((const char *)src, &end, 10); + if ((errno != 0) || (end == (const char *)src)) { + error_sump |= 1; + } +#ifdef DEBUG + cout << "Found number " << result << "\n"; +#endif + *((u64 *)current_number_buf_loc) = result; + tape[tape_loc] = ((u32)'l') << 24 | (current_number_buf_loc - number_buf); // assume 2^24 will hold all numbers for now + current_number_buf_loc += 8; + } else { + // try a strtod + char * end; + double result = strtod((const char *)src, &end); + if ((errno != 0) || (end == (const char *)src)) { + error_sump |= 1; + } +#ifdef DEBUG + cout << "Found number " << result << "\n"; +#endif + *((double *)current_number_buf_loc) = result; + tape[tape_loc] = ((u32)'d') << 24 | (current_number_buf_loc - number_buf); // assume 2^24 will hold all numbers for now + current_number_buf_loc += 8; + } + // TODO: check the MSB element is a digit + + // TODO: a whole bunch of checks + + // <=1 decimal point, eE mark, +- construct + + // first and last character in mask region must be + // digit + + // if it exists, + // Decimal point is after the first cluster of numbers only + // and before the second cluster of numbers only. It must + // be digit_or_zero . digit_or_zero strictly + + // eE mark and +- construct are adjacent with eE first + // eE mark preceeds final cluster of numbers only + // and immediately follows second-last cluster of numbers only (not + // necessarily second, as we may have 4e10). + // it may suffice to insist that eE is preceeded immediately + // by a digit of any kind and that it's followed locally by + // a digit immediately or a +- construct then a digit. + + // if we have both . and the eE mark then the . must + // precede the eE mark + + // if first character is a zero (we know in advance except for -0) + // second char must be . or eE. + + // if we have 1 cluster we have probably an integer, so try to strtol it. + // optimization: if it's a single digit just handle it here + + // otherwise we have a floating point value so strtod it. + + // return errors if strto* fail, otherwise fill in a code on the tape + // 'd' for floating point and 'l' for long and put a pointer to the + // spot in the buffer. + + // refer to this + if (error_sump) + return true; + return true; +} + never_inline bool shovel_machine(const u8 * buf, size_t len, ParsedJson & pj) { // fixup the mess made by the ape_machine // as such it does a bunch of miscellaneous things on the tapes @@ -697,7 +898,7 @@ never_inline bool shovel_machine(const u8 * buf, size_t len, ParsedJson & pj) { switch (tape[j]>>24) { case '{': case '[': { count_opens++; - // TODO: pivot our tapes + // pivot our tapes // point the enclosing structural char (}]) to the head marker ({[) and // put the end of the sequence on the tape at the head marker // we start with head marker pointing at the enclosing structural char @@ -715,20 +916,20 @@ never_inline bool shovel_machine(const u8 * buf, size_t len, ParsedJson & pj) { } case '"': { count_strings++; - parse_string(buf, len, pj, j); + error_sump |= !parse_string(buf, len, pj, j); break; } case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': count_non_zeros++; - // TODO: read in a number + error_sump |= !parse_number(buf, len, pj, j, false, false); break; case '0': count_leading_zeros++; - // TODO: read in a number. Must be float so we can skip some stuff. + error_sump |= !parse_number(buf, len, pj, j, true, false); break; case '-': count_minus++; - // TODO: read in a number + error_sump |= !parse_number(buf, len, pj, j, false, true); break; case 't': { count_true++;