From d9a77e220a08441bbf3a85952ca407421851ace3 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 31 Aug 2016 16:49:14 -0600 Subject: [PATCH 1/3] unpack: new msgpack_unpacker_next_with_size() function This new function is an extension of the original msgpack_unpacker_next() where it now adds third argument to store the number of parsed bytes for the returned buffer upon a MSGPACK_UNPACK_SUCCESS case. This is useful for cases where the caller needs to optimize memory usage in the original buffer,s so upon success retrieval of the object, it can later deprecate the already 'parsed' bytes. For more details about the origins of this function please refer to the following issue on github: https://github.com/msgpack/msgpack-c/issues/514 Signed-off-by: Eduardo Silva --- include/msgpack/unpack.h | 13 ++++++++++++- src/unpack.c | 31 +++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/include/msgpack/unpack.h b/include/msgpack/unpack.h index 0f9aede2..036d575e 100644 --- a/include/msgpack/unpack.h +++ b/include/msgpack/unpack.h @@ -146,6 +146,18 @@ static inline void msgpack_unpacker_buffer_consumed(msgpack_unpacker* mpac, si MSGPACK_DLLEXPORT msgpack_unpack_return msgpack_unpacker_next(msgpack_unpacker* mpac, msgpack_unpacked* pac); +/** + * Deserializes one object and set the number of parsed bytes involved. + * Returns true if it successes. Otherwise false is returned. + * @param mpac pointer to an initialized msgpack_unpacker object. + * @param result pointer to an initialized msgpack_unpacked object. + * @param p_bytes pointer to variable that will be set with the number of parsed bytes. + */ +MSGPACK_DLLEXPORT +msgpack_unpack_return msgpack_unpacker_next_with_size(msgpack_unpacker* mpac, + msgpack_unpacked* result, + size_t *p_bytes); + /** * Initializes a msgpack_unpacked object. * The initialized object must be destroyed by msgpack_unpacked_destroy(msgpack_unpacker*). @@ -267,4 +279,3 @@ static inline msgpack_zone* msgpack_unpacked_release_zone(msgpack_unpacked* resu #endif #endif /* msgpack/unpack.h */ - diff --git a/src/unpack.c b/src/unpack.c index 1bfcebbc..882b0b2c 100644 --- a/src/unpack.c +++ b/src/unpack.c @@ -510,7 +510,8 @@ void msgpack_unpacker_reset(msgpack_unpacker* mpac) mpac->parsed = 0; } -msgpack_unpack_return msgpack_unpacker_next(msgpack_unpacker* mpac, msgpack_unpacked* result) +static inline msgpack_unpack_return unpacker_next(msgpack_unpacker* mpac, + msgpack_unpacked* result) { int ret; @@ -529,11 +530,37 @@ msgpack_unpack_return msgpack_unpacker_next(msgpack_unpacker* mpac, msgpack_unpa } result->zone = msgpack_unpacker_release_zone(mpac); result->data = msgpack_unpacker_data(mpac); - msgpack_unpacker_reset(mpac); return MSGPACK_UNPACK_SUCCESS; } +msgpack_unpack_return msgpack_unpacker_next(msgpack_unpacker* mpac, + msgpack_unpacked* result) +{ + int ret; + + ret = unpacker_next(mpac, result); + if (ret == MSGPACK_UNPACK_SUCCESS) { + msgpack_unpacker_reset(mpac); + } + + return ret; +} + +msgpack_unpack_return +msgpack_unpacker_next_with_size(msgpack_unpacker* mpac, + msgpack_unpacked* result, size_t *p_bytes) +{ + int ret; + + ret = unpacker_next(mpac, result); + if (ret == MSGPACK_UNPACK_SUCCESS) { + *p_bytes = mpac->parsed; + msgpack_unpacker_reset(mpac); + } + + return ret; +} msgpack_unpack_return msgpack_unpack(const char* data, size_t len, size_t* off, From b90bcf3c110631cf616b4f71c70fdf78f737d54e Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 6 Sep 2016 10:46:30 -0600 Subject: [PATCH 2/3] unpack: unpacker_next_with_size() update parsed bytes on success or continue This patch makes unpacker_next_with_size(...), update p_bytes when unpacker_next() returns MSGPACK_UNPACK_SUCCESS or MSGPACK_UNPACK_CONTINUE. Signed-off-by: Eduardo Silva --- src/unpack.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/unpack.c b/src/unpack.c index 882b0b2c..a6b29f1a 100644 --- a/src/unpack.c +++ b/src/unpack.c @@ -554,8 +554,11 @@ msgpack_unpacker_next_with_size(msgpack_unpacker* mpac, int ret; ret = unpacker_next(mpac, result); - if (ret == MSGPACK_UNPACK_SUCCESS) { + if (ret == MSGPACK_UNPACK_SUCCESS || ret == MSGPACK_UNPACK_CONTINUE) { *p_bytes = mpac->parsed; + } + + if (ret == MSGPACK_UNPACK_SUCCESS) { msgpack_unpacker_reset(mpac); } From da46fb1ef73b5e4457367ebe3f7a98d695b76c2b Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 7 Sep 2016 16:24:52 -0600 Subject: [PATCH 3/3] test: c: add test for new msgpack_unpacker_next_with_size() Signed-off-by: Eduardo Silva --- test/streaming_c.cpp | 61 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/test/streaming_c.cpp b/test/streaming_c.cpp index 2fb4fe67..5d85da2c 100644 --- a/test/streaming_c.cpp +++ b/test/streaming_c.cpp @@ -120,3 +120,64 @@ TEST(streaming, basic) msgpack_unpacked_destroy(&result); msgpack_sbuffer_free(buffer); } + +TEST(streaming, basic_with_size) +{ + int ret; + size_t bytes; + size_t parsed = 0; + msgpack_sbuffer* buffer = msgpack_sbuffer_new(); + msgpack_packer* pk = msgpack_packer_new(buffer, msgpack_sbuffer_write); + msgpack_unpacked result; + msgpack_unpacker *unp; + + // 1, 2, 3, "str", ["str_data"], "bin", ["bin_data"], {0.3: 0.4} + msgpack_pack_int(pk, 1); + msgpack_pack_int(pk, 2); + msgpack_pack_int(pk, 3); + msgpack_pack_str(pk, 3); + msgpack_pack_str_body(pk, "str", 3); + msgpack_pack_array(pk, 1); + msgpack_pack_str(pk, 8); + msgpack_pack_str_body(pk, "str_data", 8); + msgpack_pack_bin(pk, 3); + msgpack_pack_bin_body(pk, "bin", 3); + msgpack_pack_array(pk, 1); + msgpack_pack_bin(pk, 8); + msgpack_pack_bin_body(pk, "bin_data", 8); + msgpack_pack_map(pk, 1); + msgpack_pack_float(pk, 0.4f); + msgpack_pack_double(pk, 0.8); + msgpack_packer_free(pk); + + unp = msgpack_unpacker_new(32 * 1024); + msgpack_unpacked_init(&result); + + const char* input = buffer->data; + + while (parsed < buffer->size) { + memcpy(msgpack_unpacker_buffer(unp), input, 1); + msgpack_unpacker_buffer_consumed(unp, 1); + input += 1; + + bytes = 0; + ret = msgpack_unpacker_next_with_size(unp, &result, &bytes); + if (ret == MSGPACK_UNPACK_CONTINUE) { + EXPECT_GT(bytes, 0); + continue; + } + + while (ret == MSGPACK_UNPACK_SUCCESS) { + EXPECT_GT(bytes, 0); + parsed += bytes; + ret = msgpack_unpacker_next_with_size(unp, &result, &bytes); + } + + } + + EXPECT_EQ(parsed, buffer->size); + + msgpack_unpacked_destroy(&result); + msgpack_unpacker_free(unp); + msgpack_sbuffer_free(buffer); +}