admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Partial completion on numbers and string normalization.

Browse Source
This commit is contained in:
Geoff Langdale
2018-05-11 14:45:54 +10:00
parent 5835b77a00
commit 88b0edd60f
1 changed files with 206 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.cpp
+206 -5
View File
@@ -397,6 +397,9 @@ u32 tape_locs[MAX_DEPTH];
u8 string_buf[512*1024];
u8 * current_string_buf_loc;
u8 number_buf[512*1024]; // holds either doubles or longs, really
u8 * current_number_buf_loc;
// STATE MACHINE DECLARATIONS
const u32 MAX_STATES = 16;
@@ -461,6 +464,7 @@ never_inline bool ape_machine(const u8 * buf, UNUSED size_t len, ParsedJson & pj
}
current_string_buf_loc = string_buf;
current_number_buf_loc = number_buf;
u32 error_sump = 0;
u32 old_tape_loc = tape_locs[depth]; // need to initialize for first write
@@ -675,6 +679,203 @@ really_inline bool parse_string(const u8 * buf, UNUSED size_t len, UNUSED Parsed
return true;
}
// put a parsed version of number (either as a double or a signed long) into the number buffer,
// put a 'tag' indicating which type and where it is back onto the tape at that location
// return false if we can't parse the number which means either
// (a) the number isn't valid, or (b) the number is followed by something that isn't whitespace, comma or a close }] character
// which are the only things that should follow a number at this stage
// bools to detect what we found in our initial character already here - we are already
// switching on 0 vs 1-9 vs - so we may as well keep separate paths where that's useful
// TODO: see if we really need a separate number_buf or whether we should just
// have a generic scratch - would need to align before using for this
really_inline bool parse_number(const u8 * buf, UNUSED size_t len, UNUSED ParsedJson & pj, u32 tape_loc, UNUSED bool found_zero, bool found_minus) {
u32 offset = tape[tape_loc] & 0xffffff;
if (found_minus) {
offset++;
}
const u8 * src = &buf[offset];
m256 v = _mm256_loadu_si256((const m256 *)(src));
u64 error_sump = 0;
#ifdef DEBUG
for (u32 j = 0; j < 32; j++) {
char c = *(src+j);
if (isprint(c)) {
cout << c;
} else {
cout << '_';
}
}
cout << "| ... number handling input\n";
#endif
// categories to extract
// Digits:
// 0 (0x30) - bucket 0
// 1-9 (never any distinction except if we didn't get the free kick at 0 due to the leading minus) (0x31-0x39) - bucket 1
// . (0x2e) - bucket 2
// E or e - no distinction (0x45/0x65) - bucket 3
// + (0x2b) - bucket 4
// - (0x2d) - bucket 4
// Terminators
// Whitespace: 0x20, 0x09, 0x0a, 0x0d - bucket 5+6
// Comma and the closes: 0x2c is comma, } is 0x5d, ] is 0x7d - bucket 5+7
// Another shufti - also a bit hand-hacked. Need to make a better construction
const m256 low_nibble_mask = _mm256_setr_epi8(
// 0 1 2 3 4 5 6 7 8 9 a b c d e f
33, 2, 2, 2, 2, 10, 2, 2, 2, 66, 64, 16, 32,208, 4, 0,
33, 2, 2, 2, 2, 10, 2, 2, 2, 66, 64, 16, 32,208, 4, 0
);
const m256 high_nibble_mask = _mm256_setr_epi8(
// 0 1 2 3 4 5 6 7 8 9 a b c d e f
64, 0, 52, 3, 8,128, 8,128, 0, 0, 0, 0, 0, 0, 0, 0,
64, 0, 52, 3, 8,128, 8,128, 0, 0, 0, 0, 0, 0, 0, 0
);
m256 tmp = _mm256_and_si256(
_mm256_shuffle_epi8(low_nibble_mask, v),
_mm256_shuffle_epi8(high_nibble_mask,
_mm256_and_si256(_mm256_srli_epi32(v, 4), _mm256_set1_epi8(0x7f))));
m256 enders_mask = _mm256_set1_epi8(0xe0);
m256 tmp_enders = _mm256_cmpeq_epi8(_mm256_and_si256(tmp, enders_mask),
_mm256_set1_epi8(0));
u32 enders = ~(u32)_mm256_movemask_epi8(tmp_enders);
dumpbits32(enders, "ender characters");
if (enders == 0) {
// TODO: scream for help if enders == 0 which means we have
// a heroically long number string or some garbage
}
// TODO: make a mask that indicates where our digits are
u32 number_mask = ~enders & (enders-1);
dumpbits32(number_mask, "number mask");
m256 n_mask = _mm256_set1_epi8(0x1f);
m256 tmp_n = _mm256_cmpeq_epi8(_mm256_and_si256(tmp, n_mask),
_mm256_set1_epi8(0));
u32 number_characters = ~(u32)_mm256_movemask_epi8(tmp_n);
// put something into our error sump if we have something
// before our ending characters that isn't a valid character
// for the inside of our JSON
number_characters &= number_mask;
error_sump |= number_characters ^ number_mask;
dumpbits32(number_characters, "number characters");
// TODO: prune by mask
// TODO: check that result of pruning by mask *is* the
// same as the mask
m256 d_mask = _mm256_set1_epi8(0x03);
m256 tmp_d = _mm256_cmpeq_epi8(_mm256_and_si256(tmp, d_mask),
_mm256_set1_epi8(0));
u32 digit_characters = ~(u32)_mm256_movemask_epi8(tmp_d);
digit_characters &= number_mask;
dumpbits32(digit_characters, "digit characters");
m256 p_mask = _mm256_set1_epi8(0x04);
m256 tmp_p = _mm256_cmpeq_epi8(_mm256_and_si256(tmp, p_mask),
_mm256_set1_epi8(0));
u32 decimal_characters = ~(u32)_mm256_movemask_epi8(tmp_p);
decimal_characters &= number_mask;
dumpbits32(decimal_characters, "decimal characters");
m256 e_mask = _mm256_set1_epi8(0x08);
m256 tmp_e = _mm256_cmpeq_epi8(_mm256_and_si256(tmp, e_mask),
_mm256_set1_epi8(0));
u32 exponent_characters = ~(u32)_mm256_movemask_epi8(tmp_e);
exponent_characters &= number_mask;
dumpbits32(exponent_characters, "exponent characters");
m256 s_mask = _mm256_set1_epi8(0x10);
m256 tmp_s = _mm256_cmpeq_epi8(_mm256_and_si256(tmp, s_mask),
_mm256_set1_epi8(0));
u32 sign_characters = ~(u32)_mm256_movemask_epi8(tmp_s);
sign_characters &= number_mask;
dumpbits32(sign_characters, "sign characters");
u32 digit_edges = ~(digit_characters << 1) & digit_characters;
dumpbits32(digit_edges, "digit_edges");
// check that we have 1-3 'edges' only
u32 t = digit_edges;
t &= t-1; t &= t-1; t &= t-1;
error_sump |= t;
// check that we start with a digit
error_sump |= ~digit_characters & 0x1;
if (__builtin_popcount(digit_edges) == 1) {
// try a strtoll
char * end;
u64 result = strtoll((const char *)src, &end, 10);
if ((errno != 0) || (end == (const char *)src)) {
error_sump |= 1;
}
#ifdef DEBUG
cout << "Found number " << result << "\n";
#endif
*((u64 *)current_number_buf_loc) = result;
tape[tape_loc] = ((u32)'l') << 24 | (current_number_buf_loc - number_buf); // assume 2^24 will hold all numbers for now
current_number_buf_loc += 8;
} else {
// try a strtod
char * end;
double result = strtod((const char *)src, &end);
if ((errno != 0) || (end == (const char *)src)) {
error_sump |= 1;
}
#ifdef DEBUG
cout << "Found number " << result << "\n";
#endif
*((double *)current_number_buf_loc) = result;
tape[tape_loc] = ((u32)'d') << 24 | (current_number_buf_loc - number_buf); // assume 2^24 will hold all numbers for now
current_number_buf_loc += 8;
}
// TODO: check the MSB element is a digit
// TODO: a whole bunch of checks
// <=1 decimal point, eE mark, +- construct
// first and last character in mask region must be
// digit
// if it exists,
// Decimal point is after the first cluster of numbers only
// and before the second cluster of numbers only. It must
// be digit_or_zero . digit_or_zero strictly
// eE mark and +- construct are adjacent with eE first
// eE mark preceeds final cluster of numbers only
// and immediately follows second-last cluster of numbers only (not
// necessarily second, as we may have 4e10).
// it may suffice to insist that eE is preceeded immediately
// by a digit of any kind and that it's followed locally by
// a digit immediately or a +- construct then a digit.
// if we have both . and the eE mark then the . must
// precede the eE mark
// if first character is a zero (we know in advance except for -0)
// second char must be . or eE.
// if we have 1 cluster we have probably an integer, so try to strtol it.
// optimization: if it's a single digit just handle it here
// otherwise we have a floating point value so strtod it.
// return errors if strto* fail, otherwise fill in a code on the tape
// 'd' for floating point and 'l' for long and put a pointer to the
// spot in the buffer.
// refer to this
if (error_sump)
return true;
return true;
}
never_inline bool shovel_machine(const u8 * buf, size_t len, ParsedJson & pj) {
// fixup the mess made by the ape_machine
// as such it does a bunch of miscellaneous things on the tapes
@@ -697,7 +898,7 @@ never_inline bool shovel_machine(const u8 * buf, size_t len, ParsedJson & pj) {
switch (tape[j]>>24) {
case '{': case '[': {
count_opens++;
// TODO: pivot our tapes
// pivot our tapes
// point the enclosing structural char (}]) to the head marker ({[) and
// put the end of the sequence on the tape at the head marker
// we start with head marker pointing at the enclosing structural char
@@ -715,20 +916,20 @@ never_inline bool shovel_machine(const u8 * buf, size_t len, ParsedJson & pj) {
}
case '"': {
count_strings++;
parse_string(buf, len, pj, j);
error_sump |= !parse_string(buf, len, pj, j);
break;
}
case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9':
count_non_zeros++;
// TODO: read in a number
error_sump |= !parse_number(buf, len, pj, j, false, false);
break;
case '0':
count_leading_zeros++;
// TODO: read in a number. Must be float so we can skip some stuff.
error_sump |= !parse_number(buf, len, pj, j, true, false);
break;
case '-':
count_minus++;
// TODO: read in a number
error_sump |= !parse_number(buf, len, pj, j, false, true);
break;
case 't': {
count_true++;