From 1f6d83ed56df2fafc013b9a73a6d6c8759a8462f Mon Sep 17 00:00:00 2001 From: Joao Paulo Magalhaes Date: Sat, 27 Jan 2024 19:36:54 +0100 Subject: [PATCH] all tests ok --- src/c4/yml/filter_processor.hpp | 48 ++++++++++++++++-- src/c4/yml/parse.cpp | 14 ++--- test/test_block_folded.cpp | 32 ++++++++++-- test/test_block_literal.cpp | 25 +++++++++ test/test_filter.cpp | 90 ++++++++++++++++----------------- test/test_plain_scalar.cpp | 10 +++- test/test_single_quoted.cpp | 10 +++- 7 files changed, 166 insertions(+), 63 deletions(-) diff --git a/src/c4/yml/filter_processor.hpp b/src/c4/yml/filter_processor.hpp index 74cd72451..4e4ba4955 100644 --- a/src/c4/yml/filter_processor.hpp +++ b/src/c4/yml/filter_processor.hpp @@ -101,7 +101,7 @@ struct FilterProcessorSrcDst ++wpos; rpos += 2; } - C4_ALWAYS_INLINE void translate_esc(const char *C4_RESTRICT s, size_t nw, size_t nr) noexcept + C4_ALWAYS_INLINE void translate_esc_bulk(const char *C4_RESTRICT s, size_t nw, size_t nr) noexcept { RYML_ASSERT(nw > 0); RYML_ASSERT(nr > 0); @@ -111,6 +111,10 @@ struct FilterProcessorSrcDst wpos += nw; rpos += 1 + nr; } + C4_ALWAYS_INLINE void translate_esc_extending(const char *C4_RESTRICT s, size_t nw, size_t nr) noexcept + { + translate_esc_bulk(s, nw, nr); + } }; @@ -226,7 +230,7 @@ struct FilterProcessorInplaceEndExtending ++wpos; } - C4_NO_INLINE void translate_esc(const char *C4_RESTRICT s, size_t nw, size_t nr) noexcept + void translate_esc_bulk(const char *C4_RESTRICT s, size_t nw, size_t nr) noexcept { RYML_ASSERT(nw > 0); RYML_ASSERT(nr > 0); @@ -241,9 +245,25 @@ struct FilterProcessorInplaceEndExtending rpos = rpos_next; wpos = wpos_next; } + + C4_ALWAYS_INLINE void translate_esc_extending(const char *C4_RESTRICT s, size_t nw, size_t nr) noexcept + { + translate_esc_bulk(s, nw, nr); + } }; +/** Filters in place. The result may be larger than the source, and + * extending may happen anywhere. As a result some characters may be + * left unfiltered when there is no slack in the buffer and the + * write-position would overlap the read-position. Consequently, it's + * possible for characters to be left unfiltered. In YAML, this + * happens only with double-quoted strings, and only with a small + * number of escape sequences such as \L which is substituted by three + * bytes. These escape sequences cause a call to translate_esc_extending() + * which is the only entry point to this unfiltered situation. + * + * @see FilterProcessorInplaceMidExtending */ struct FilterProcessorInplaceMidExtending { substr src; ///< the subject string @@ -388,7 +408,29 @@ struct FilterProcessorInplaceMidExtending maxcap = wpos > maxcap ? wpos : maxcap; } - C4_NO_INLINE void translate_esc(const char *C4_RESTRICT s, size_t nw, size_t nr) noexcept + C4_NO_INLINE void translate_esc_bulk(const char *C4_RESTRICT s, size_t nw, size_t nr) noexcept + { + RYML_ASSERT(nw > 0); + RYML_ASSERT(nr > 0); + RYML_ASSERT(nr+1u >= nw); + const size_t wpos_next = wpos + nw; + const size_t rpos_next = rpos + nr + 1u; // add 1u to account for the escape character + if(wpos_next <= wcap) // respect write-capacity + { + if((wpos <= rpos) && !unfiltered_chars) // write only if wpos is behind rpos + memcpy(src.str + wpos, s, nw); + } + else + { + _c4dbgip("inplace: add unwritten {}->{} (wpos={}!=rpos={})={} (wpos={}{}!", unfiltered_chars, true, wpos, rpos, wpos!=rpos, wpos, wcap, wpos maxcap ? wpos : maxcap; + } + + C4_NO_INLINE void translate_esc_extending(const char *C4_RESTRICT s, size_t nw, size_t nr) noexcept { RYML_ASSERT(nw > 0); RYML_ASSERT(nr > 0); diff --git a/src/c4/yml/parse.cpp b/src/c4/yml/parse.cpp index 3ee081766..cbf8d06bb 100644 --- a/src/c4/yml/parse.cpp +++ b/src/c4/yml/parse.cpp @@ -4896,7 +4896,7 @@ void Parser::_filter_dquoted_backslash(FilterProcessor &C4_RESTRICT proc) uint8_t byteval = {}; if(C4_UNLIKELY(!read_hex(codepoint, &byteval))) _c4err("failed to read \\x codepoint. scalar pos={}", proc.rpos); - proc.translate_esc((const char*)&byteval, 1u, /*nread*/3u); + proc.translate_esc_bulk((const char*)&byteval, 1u, /*nread*/3u); _c4dbgfdq("utf8 after rpos={} rem=~~~{}~~~", proc.rpos, proc.src.sub(proc.rpos)); } else if(next == 'u') // UTF16 @@ -4910,7 +4910,7 @@ void Parser::_filter_dquoted_backslash(FilterProcessor &C4_RESTRICT proc) _c4err("failed to parse \\u codepoint. scalar pos={}", proc.rpos); size_t numbytes = decode_code_point((uint8_t*)readbuf, sizeof(readbuf), codepoint_val); C4_ASSERT(numbytes <= 4); - proc.translate_esc(readbuf, numbytes, /*nread*/5u); + proc.translate_esc_bulk(readbuf, numbytes, /*nread*/5u); } else if(next == 'U') // UTF32 { @@ -4923,7 +4923,7 @@ void Parser::_filter_dquoted_backslash(FilterProcessor &C4_RESTRICT proc) _c4err("failed to parse \\U codepoint. scalar pos={}", proc.rpos); size_t numbytes = decode_code_point((uint8_t*)readbuf, sizeof(readbuf), codepoint_val); C4_ASSERT(numbytes <= 4); - proc.translate_esc(readbuf, numbytes, /*nread*/9u); + proc.translate_esc_bulk(readbuf, numbytes, /*nread*/9u); } // https://yaml.org/spec/1.2.2/#rule-c-ns-esc-char else if(next == '0') @@ -4957,7 +4957,7 @@ void Parser::_filter_dquoted_backslash(FilterProcessor &C4_RESTRICT proc) _RYML_CHCONST(-0x3e, 0xc2), _RYML_CHCONST(-0x60, 0xa0), }; - proc.translate_esc(payload, /*nwrite*/2, /*nread*/1); + proc.translate_esc_bulk(payload, /*nwrite*/2, /*nread*/1); } else if(next == 'N') // unicode next line \u0085 { @@ -4966,7 +4966,7 @@ void Parser::_filter_dquoted_backslash(FilterProcessor &C4_RESTRICT proc) _RYML_CHCONST(-0x3e, 0xc2), _RYML_CHCONST(-0x7b, 0x85), }; - proc.translate_esc(payload, /*nwrite*/2, /*nread*/1); + proc.translate_esc_bulk(payload, /*nwrite*/2, /*nread*/1); } else if(next == 'L') // unicode line separator \u2028 { @@ -4976,7 +4976,7 @@ void Parser::_filter_dquoted_backslash(FilterProcessor &C4_RESTRICT proc) _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x58, 0xa8), }; - proc.translate_esc(payload, /*nwrite*/3, /*nread*/1); + proc.translate_esc_extending(payload, /*nwrite*/3, /*nread*/1); } else if(next == 'P') // unicode paragraph separator \u2029 { @@ -4986,7 +4986,7 @@ void Parser::_filter_dquoted_backslash(FilterProcessor &C4_RESTRICT proc) _RYML_CHCONST(-0x80, 0x80), _RYML_CHCONST(-0x57, 0xa9), }; - proc.translate_esc(payload, /*nwrite*/3, /*nread*/1); + proc.translate_esc_extending(payload, /*nwrite*/3, /*nread*/1); } else if(next == '\0') { diff --git a/test/test_block_folded.cpp b/test/test_block_folded.cpp index e184f68d1..043134f45 100644 --- a/test/test_block_folded.cpp +++ b/test/test_block_folded.cpp @@ -35,10 +35,16 @@ void test_filter_inplace(blockfolded_case const& blcase) if(blcase.input.len >= blcase.expected.len) { std::string subject_(blcase.input.str, blcase.input.len); + std::string subject_2 = subject_; c4::substr dst = to_substr(subject_); - Parser proc = {}; - FilterResult result = proc.filter_scalar_block_folded_in_place(dst, subject_.size(), blcase.indentation, blcase.chomp); + Parser parser1 = {}; + FilterResult result = parser1.filter_scalar_block_folded_in_place(dst, subject_.size(), blcase.indentation, blcase.chomp); ASSERT_TRUE(result.valid()); + Parser parser2 = {}; + Tree tree = parser2.parse_in_arena("file", "# set the tree in the parser"); + csubstr sresult = parser2._filter_scalar_block_folded(to_substr(subject_2), blcase.chomp, blcase.indentation); + EXPECT_GE(result.required_len(), blcase.expected.len); + EXPECT_EQ(sresult.len, result.str.len); const csubstr out = result.get(); if(blcase.chomp != CHOMP_CLIP) { @@ -54,13 +60,19 @@ void test_filter_inplace(blockfolded_case const& blcase) { SCOPED_TRACE("spare size"); std::string subject_(blcase.input.str, blcase.input.len); + std::string subject_2 = subject_; subject_.resize(blcase.expected.len + 30); c4::substr dst = to_substr(subject_).first(blcase.input.len); c4::substr rem = to_substr(subject_).sub(blcase.expected.len); rem.fill('^'); - Parser proc = {}; - FilterResult result = proc.filter_scalar_block_folded_in_place(dst, subject_.size(), blcase.indentation, blcase.chomp); + Parser parser1 = {}; + FilterResult result = parser1.filter_scalar_block_folded_in_place(dst, subject_.size(), blcase.indentation, blcase.chomp); ASSERT_TRUE(result.valid()); + Parser parser2 = {}; + Tree tree = parser2.parse_in_arena("file", "# set the tree in the parser"); + csubstr sresult = parser2._filter_scalar_block_folded(to_substr(subject_2), blcase.chomp, blcase.indentation); + EXPECT_GE(result.required_len(), blcase.expected.len); + EXPECT_EQ(sresult.len, result.str.len); const csubstr out = result.get(); if(blcase.chomp != CHOMP_CLIP) { @@ -75,11 +87,17 @@ void test_filter_inplace(blockfolded_case const& blcase) { SCOPED_TRACE("trimmed size"); std::string subject_(blcase.input.str, blcase.input.len); + std::string subject_2 = subject_; subject_.resize(blcase.expected.len); c4::substr dst = to_substr(subject_).first(blcase.input.len); Parser proc = {}; FilterResult result = proc.filter_scalar_block_folded_in_place(dst, subject_.size(), blcase.indentation, blcase.chomp); ASSERT_TRUE(result.valid()); + Parser parser2 = {}; + Tree tree = parser2.parse_in_arena("file", "# set the tree in the parser"); + csubstr sresult = parser2._filter_scalar_block_folded(to_substr(subject_2), blcase.chomp, blcase.indentation); + EXPECT_GE(result.required_len(), blcase.expected.len); + EXPECT_EQ(sresult.len, result.str.len); const csubstr out = result.get(); if(blcase.chomp != CHOMP_CLIP) { @@ -93,9 +111,15 @@ void test_filter_inplace(blockfolded_case const& blcase) { SCOPED_TRACE("insufficient size"); std::string subject_(blcase.input.str, blcase.input.len); + std::string subject_2 = subject_; c4::substr dst = to_substr(subject_); Parser proc = {}; FilterResult result = proc.filter_scalar_block_folded_in_place(dst, subject_.size(), blcase.indentation, blcase.chomp); + Parser parser2 = {}; + Tree tree = parser2.parse_in_arena("file", "# set the tree in the parser"); + csubstr sresult = parser2._filter_scalar_block_folded(to_substr(subject_2), blcase.chomp, blcase.indentation); + EXPECT_GE(result.required_len(), blcase.expected.len); + EXPECT_EQ(sresult.len, result.str.len); if(blcase.chomp != CHOMP_CLIP) { EXPECT_EQ(result.required_len(), blcase.expected.len); diff --git a/test/test_block_literal.cpp b/test/test_block_literal.cpp index b03b65903..89e66aeb7 100644 --- a/test/test_block_literal.cpp +++ b/test/test_block_literal.cpp @@ -16,6 +16,7 @@ void test_filter_src_dst(blocklit_case const& blcase) RYML_TRACE_FMT("\nstr=[{}]~~~{}~~~\nexp=[{}]~~~{}~~~", blcase.input.len, blcase.input, blcase.expected.len, blcase.expected); std::string subject_; subject_.resize(2 * blcase.input.size()); + std::string subject_2 = subject_; c4::substr dst = to_substr(subject_); Parser proc = {}; FilterResult result = proc.filter_scalar_block_literal(blcase.input, dst, blcase.indentation, blcase.chomp); @@ -36,10 +37,16 @@ void test_filter_inplace(blocklit_case const& blcase) if(blcase.input.len >= blcase.expected.len) { std::string subject_(blcase.input.str, blcase.input.len); + std::string subject_2 = subject_; c4::substr dst = to_substr(subject_); Parser proc = {}; FilterResult result = proc.filter_scalar_block_literal_in_place(dst, subject_.size(), blcase.indentation, blcase.chomp); ASSERT_TRUE(result.valid()); + Parser parser2 = {}; + Tree tree = parser2.parse_in_arena("file", "# set the tree in the parser"); + csubstr sresult = parser2._filter_scalar_block_literal(to_substr(subject_2), blcase.chomp, blcase.indentation); + EXPECT_GE(result.required_len(), blcase.expected.len); + EXPECT_EQ(sresult.len, result.str.len); const csubstr out = result.get(); if(blcase.chomp != CHOMP_CLIP) { @@ -55,6 +62,7 @@ void test_filter_inplace(blocklit_case const& blcase) { SCOPED_TRACE("spare size"); std::string subject_(blcase.input.str, blcase.input.len); + std::string subject_2 = subject_; subject_.resize(blcase.expected.len + 30); c4::substr dst = to_substr(subject_).first(blcase.input.len); c4::substr rem = to_substr(subject_).sub(blcase.expected.len); @@ -62,6 +70,11 @@ void test_filter_inplace(blocklit_case const& blcase) Parser proc = {}; FilterResult result = proc.filter_scalar_block_literal_in_place(dst, subject_.size(), blcase.indentation, blcase.chomp); ASSERT_TRUE(result.valid()); + Parser parser2 = {}; + Tree tree = parser2.parse_in_arena("file", "# set the tree in the parser"); + csubstr sresult = parser2._filter_scalar_block_literal(to_substr(subject_2), blcase.chomp, blcase.indentation); + EXPECT_GE(result.required_len(), blcase.expected.len); + EXPECT_EQ(sresult.len, result.str.len); const csubstr out = result.get(); if(blcase.chomp != CHOMP_CLIP) { @@ -76,11 +89,17 @@ void test_filter_inplace(blocklit_case const& blcase) { SCOPED_TRACE("trimmed size"); std::string subject_(blcase.input.str, blcase.input.len); + std::string subject_2 = subject_; subject_.resize(blcase.expected.len); c4::substr dst = to_substr(subject_).first(blcase.input.len); Parser proc = {}; FilterResult result = proc.filter_scalar_block_literal_in_place(dst, subject_.size(), blcase.indentation, blcase.chomp); ASSERT_TRUE(result.valid()); + Parser parser2 = {}; + Tree tree = parser2.parse_in_arena("file", "# set the tree in the parser"); + csubstr sresult = parser2._filter_scalar_block_literal(to_substr(subject_2), blcase.chomp, blcase.indentation); + EXPECT_GE(result.required_len(), blcase.expected.len); + EXPECT_EQ(sresult.len, result.str.len); const csubstr out = result.get(); if(blcase.chomp != CHOMP_CLIP) { @@ -94,10 +113,16 @@ void test_filter_inplace(blocklit_case const& blcase) { SCOPED_TRACE("insufficient size"); std::string subject_(blcase.input.str, blcase.input.len); + std::string subject_2 = subject_; c4::substr dst = to_substr(subject_); Parser proc = {}; FilterResult result = proc.filter_scalar_block_literal_in_place(dst, subject_.size(), blcase.indentation, blcase.chomp); ASSERT_FALSE(result.valid()); + Parser parser2 = {}; + Tree tree = parser2.parse_in_arena("file", "# set the tree in the parser"); + csubstr sresult = parser2._filter_scalar_block_literal(to_substr(subject_2), blcase.chomp, blcase.indentation); + EXPECT_GE(result.required_len(), blcase.expected.len); + EXPECT_EQ(sresult.len, result.str.len); if(blcase.chomp != CHOMP_CLIP) { EXPECT_EQ(result.required_len(), blcase.expected.len); diff --git a/test/test_filter.cpp b/test/test_filter.cpp index 502ad5a35..94c35c72c 100644 --- a/test/test_filter.cpp +++ b/test/test_filter.cpp @@ -795,35 +795,35 @@ TEST(FilterProcessorInplaceMid, translate_esc_bulk) EXPECT_EQ(t.proc.wpos, 0); EXPECT_FALSE(t.proc.unfiltered_chars); EXPECT_EQ(t.proc.sofar(), ""); - t.proc.translate_esc("aa", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("aa", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 2); EXPECT_EQ(t.proc.wpos, 2); EXPECT_EQ(t.proc.sofar(), "aa"); EXPECT_EQ(t.proc.result().get(), "aa"); EXPECT_EQ(t.subject, "aa11223344"); EXPECT_FALSE(t.proc.unfiltered_chars); - t.proc.translate_esc("bb", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("bb", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 4); EXPECT_EQ(t.proc.wpos, 4); EXPECT_EQ(t.proc.sofar(), "aabb"); EXPECT_EQ(t.proc.result().get(), "aabb"); EXPECT_EQ(t.subject, "aabb223344"); EXPECT_FALSE(t.proc.unfiltered_chars); - t.proc.translate_esc("cc", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("cc", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 6); EXPECT_EQ(t.proc.wpos, 6); EXPECT_EQ(t.proc.sofar(), "aabbcc"); EXPECT_EQ(t.proc.result().get(), "aabbcc"); EXPECT_EQ(t.subject, "aabbcc3344"); EXPECT_FALSE(t.proc.unfiltered_chars); - t.proc.translate_esc("dd", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("dd", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 8); EXPECT_EQ(t.proc.wpos, 8); EXPECT_EQ(t.proc.sofar(), "aabbccdd"); EXPECT_EQ(t.proc.result().get(), "aabbccdd"); EXPECT_EQ(t.subject, "aabbccdd44"); EXPECT_FALSE(t.proc.unfiltered_chars); - t.proc.translate_esc("ee", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("ee", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 10); EXPECT_EQ(t.proc.wpos, 10); EXPECT_EQ(t.proc.sofar(), "aabbccddee"); @@ -840,31 +840,31 @@ TEST(FilterProcessorInplaceEnd, translate_esc_bulk) EXPECT_EQ(t.proc.rpos, 0); EXPECT_EQ(t.proc.wpos, 0); EXPECT_EQ(t.proc.sofar(), ""); - t.proc.translate_esc("aa", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("aa", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 2); EXPECT_EQ(t.proc.wpos, 2); EXPECT_EQ(t.proc.sofar(), "aa"); EXPECT_EQ(t.proc.result().get(), "aa"); EXPECT_EQ(t.subject, "aa11223344"); - t.proc.translate_esc("bb", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("bb", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 4); EXPECT_EQ(t.proc.wpos, 4); EXPECT_EQ(t.proc.sofar(), "aabb"); EXPECT_EQ(t.proc.result().get(), "aabb"); EXPECT_EQ(t.subject, "aabb223344"); - t.proc.translate_esc("cc", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("cc", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 6); EXPECT_EQ(t.proc.wpos, 6); EXPECT_EQ(t.proc.sofar(), "aabbcc"); EXPECT_EQ(t.proc.result().get(), "aabbcc"); EXPECT_EQ(t.subject, "aabbcc3344"); - t.proc.translate_esc("dd", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("dd", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 8); EXPECT_EQ(t.proc.wpos, 8); EXPECT_EQ(t.proc.sofar(), "aabbccdd"); EXPECT_EQ(t.proc.result().get(), "aabbccdd"); EXPECT_EQ(t.subject, "aabbccdd44"); - t.proc.translate_esc("ee", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("ee", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 10); EXPECT_EQ(t.proc.wpos, 10); EXPECT_EQ(t.proc.sofar(), "aabbccddee"); @@ -878,27 +878,27 @@ TEST(FilterProcessorSrcDst, translate_esc_bulk) EXPECT_EQ(t.proc.rpos, 0); EXPECT_EQ(t.proc.wpos, 0); EXPECT_EQ(t.proc.sofar(), ""); - t.proc.translate_esc("aa", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("aa", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 2); EXPECT_EQ(t.proc.wpos, 2); EXPECT_EQ(t.proc.sofar(), "aa"); EXPECT_EQ(t.proc.result().get(), "aa"); - t.proc.translate_esc("bb", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("bb", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 4); EXPECT_EQ(t.proc.wpos, 4); EXPECT_EQ(t.proc.sofar(), "aabb"); EXPECT_EQ(t.proc.result().get(), "aabb"); - t.proc.translate_esc("cc", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("cc", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 6); EXPECT_EQ(t.proc.wpos, 6); EXPECT_EQ(t.proc.sofar(), "aabbcc"); EXPECT_EQ(t.proc.result().get(), "aabbcc"); - t.proc.translate_esc("dd", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("dd", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 8); EXPECT_EQ(t.proc.wpos, 8); EXPECT_EQ(t.proc.sofar(), "aabbccdd"); EXPECT_EQ(t.proc.result().get(), "aabbccdd"); - t.proc.translate_esc("ee", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("ee", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 10); EXPECT_EQ(t.proc.wpos, 10); EXPECT_EQ(t.proc.sofar(), "aabbccddee"); @@ -908,7 +908,7 @@ TEST(FilterProcessorSrcDst, translate_esc_bulk) //----------------------------------------------------------------------------- -TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__trimmed_capacity) +TEST(FilterProcessorInplaceMid, translate_esc_extending_bulk_excess__trimmed_capacity) { TesterInplaceMid t("0011223344"); t.trim_capacity(); @@ -918,7 +918,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__trimmed_capacity) EXPECT_EQ(t.proc.src.len, 10); EXPECT_FALSE(t.proc.unfiltered_chars); EXPECT_EQ(t.proc.sofar(), ""); - t.proc.translate_esc("aaa", /*nw*/3, /*nr*/1); + t.proc.translate_esc_extending("aaa", /*nw*/3, /*nr*/1); EXPECT_EQ(t.proc.rpos, 2); EXPECT_EQ(t.proc.wpos, 3); EXPECT_EQ(t.proc.src.len, 10); @@ -926,7 +926,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__trimmed_capacity) EXPECT_EQ(t.proc.result().str, nullptr); EXPECT_EQ(t.subject, "0011223344"); EXPECT_TRUE(t.proc.unfiltered_chars); - t.proc.translate_esc("bbb", /*nw*/3, /*nr*/1); + t.proc.translate_esc_extending("bbb", /*nw*/3, /*nr*/1); EXPECT_EQ(t.proc.rpos, 4); EXPECT_EQ(t.proc.wpos, 6); EXPECT_EQ(t.proc.src.len, 10); @@ -934,7 +934,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__trimmed_capacity) EXPECT_EQ(t.proc.result().str, nullptr); EXPECT_EQ(t.subject, "0011223344"); EXPECT_TRUE(t.proc.unfiltered_chars); - t.proc.translate_esc("ccc", /*nw*/3, /*nr*/1); + t.proc.translate_esc_extending("ccc", /*nw*/3, /*nr*/1); EXPECT_EQ(t.proc.rpos, 6); EXPECT_EQ(t.proc.wpos, 9); EXPECT_EQ(t.proc.src.len, 10); @@ -942,7 +942,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__trimmed_capacity) EXPECT_EQ(t.proc.result().str, nullptr); EXPECT_EQ(t.subject, "0011223344"); EXPECT_TRUE(t.proc.unfiltered_chars); - t.proc.translate_esc("ddd", /*nw*/3, /*nr*/1); + t.proc.translate_esc_extending("ddd", /*nw*/3, /*nr*/1); EXPECT_EQ(t.proc.rpos, 8); EXPECT_EQ(t.proc.wpos, 12); EXPECT_EQ(t.proc.src.len, 10); @@ -951,7 +951,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__trimmed_capacity) EXPECT_EQ(t.subject, "0011223344"); EXPECT_TRUE(t.proc.unfiltered_chars); // write 4 characters - t.proc.translate_esc("cccc", /*nw*/4, /*nr*/1); + t.proc.translate_esc_extending("cccc", /*nw*/4, /*nr*/1); EXPECT_EQ(t.proc.rpos, 10); EXPECT_EQ(t.proc.wpos, 16); EXPECT_EQ(t.proc.src.len, 10); @@ -961,7 +961,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__trimmed_capacity) EXPECT_TRUE(t.proc.unfiltered_chars); } -TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__spare_capacity) +TEST(FilterProcessorInplaceMid, translate_esc_extending_bulk_excess__spare_capacity) { TesterInplaceMid t("0011223344"); const size_t needed_capacity = 16u; @@ -978,7 +978,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__spare_capacity) EXPECT_EQ(full_subject, "0011223344^^^^"); EXPECT_FALSE(t.proc.unfiltered_chars); EXPECT_EQ(t.proc.sofar(), ""); - t.proc.translate_esc("aaa", /*nw*/3, /*nr*/1); + t.proc.translate_esc_extending("aaa", /*nw*/3, /*nr*/1); EXPECT_EQ(t.proc.rpos, 3); EXPECT_EQ(t.proc.wpos, 3); EXPECT_EQ(t.proc.src.len, 11); @@ -986,7 +986,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__spare_capacity) EXPECT_EQ(t.proc.result().get(), "aaa"); EXPECT_EQ(full_subject, "aaa11223344^^^"); EXPECT_FALSE(t.proc.unfiltered_chars); - t.proc.translate_esc("bbb", /*nw*/3, /*nr*/1); + t.proc.translate_esc_extending("bbb", /*nw*/3, /*nr*/1); EXPECT_EQ(t.proc.rpos, 6); EXPECT_EQ(t.proc.wpos, 6); EXPECT_EQ(t.proc.src.len, 12); @@ -994,7 +994,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__spare_capacity) EXPECT_EQ(t.proc.result().get(), "aaabbb"); EXPECT_EQ(full_subject, "aaabbb223344^^"); EXPECT_FALSE(t.proc.unfiltered_chars); - t.proc.translate_esc("ccc", /*nw*/3, /*nr*/1); + t.proc.translate_esc_extending("ccc", /*nw*/3, /*nr*/1); EXPECT_EQ(t.proc.rpos, 9); EXPECT_EQ(t.proc.wpos, 9); EXPECT_EQ(t.proc.src.len, 13); @@ -1002,7 +1002,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__spare_capacity) EXPECT_EQ(t.proc.result().get(), "aaabbbccc"); EXPECT_EQ(full_subject, "aaabbbccc3344^"); EXPECT_FALSE(t.proc.unfiltered_chars); - t.proc.translate_esc("ddd", /*nw*/3, /*nr*/1); + t.proc.translate_esc_extending("ddd", /*nw*/3, /*nr*/1); EXPECT_EQ(t.proc.rpos, 12); EXPECT_EQ(t.proc.wpos, 12); EXPECT_EQ(t.proc.src.len, 14); @@ -1010,7 +1010,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__spare_capacity) EXPECT_EQ(t.proc.result().get(), "aaabbbcccddd"); EXPECT_EQ(full_subject, "aaabbbcccddd44"); EXPECT_FALSE(t.proc.unfiltered_chars); - t.proc.translate_esc("eeee", /*nw*/4, /*nr*/1); + t.proc.translate_esc_extending("eeee", /*nw*/4, /*nr*/1); EXPECT_EQ(t.proc.rpos, 14); EXPECT_EQ(t.proc.wpos, 16); EXPECT_EQ(t.proc.src.len, smaller_capacity); // not 16! limited @@ -1020,7 +1020,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_bulk_excess__spare_capacity) EXPECT_TRUE(t.proc.unfiltered_chars); } -TEST(FilterProcessorInplaceMid, copy_after_translate_esc_bulk_excess__trimmed_capacity) +TEST(FilterProcessorInplaceMid, copy_after_translate_esc_extending_bulk_excess__trimmed_capacity) { { TesterInplaceMid t("0011223344"); @@ -1031,7 +1031,7 @@ TEST(FilterProcessorInplaceMid, copy_after_translate_esc_bulk_excess__trimmed_ca EXPECT_EQ(t.proc.src.len, 10); EXPECT_FALSE(t.proc.unfiltered_chars); EXPECT_EQ(t.proc.sofar(), ""); - t.proc.translate_esc("aaaa", /*nw*/4, /*nr*/1); + t.proc.translate_esc_extending("aaaa", /*nw*/4, /*nr*/1); EXPECT_EQ(t.proc.rpos, 2); EXPECT_EQ(t.proc.wpos, 4); EXPECT_EQ(t.proc.src.len, 10); @@ -1057,7 +1057,7 @@ TEST(FilterProcessorInplaceMid, copy_after_translate_esc_bulk_excess__trimmed_ca EXPECT_EQ(t.proc.src.len, 10); EXPECT_FALSE(t.proc.unfiltered_chars); EXPECT_EQ(t.proc.sofar(), ""); - t.proc.translate_esc("aaaa", /*nw*/4, /*nr*/1); + t.proc.translate_esc_extending("aaaa", /*nw*/4, /*nr*/1); EXPECT_EQ(t.proc.rpos, 2); EXPECT_EQ(t.proc.wpos, 4); EXPECT_EQ(t.proc.src.len, 10); @@ -1076,7 +1076,7 @@ TEST(FilterProcessorInplaceMid, copy_after_translate_esc_bulk_excess__trimmed_ca } } -TEST(FilterProcessorInplaceMid, set_after_translate_esc_bulk_excess__trimmed_capacity) +TEST(FilterProcessorInplaceMid, set_after_translate_esc_extending_bulk_excess__trimmed_capacity) { { TesterInplaceMid t("0011223344"); @@ -1087,7 +1087,7 @@ TEST(FilterProcessorInplaceMid, set_after_translate_esc_bulk_excess__trimmed_cap EXPECT_EQ(t.proc.src.len, 10); EXPECT_FALSE(t.proc.unfiltered_chars); EXPECT_EQ(t.proc.sofar(), ""); - t.proc.translate_esc("aaaa", /*nw*/4, /*nr*/1); + t.proc.translate_esc_extending("aaaa", /*nw*/4, /*nr*/1); EXPECT_EQ(t.proc.rpos, 2); EXPECT_EQ(t.proc.wpos, 4); EXPECT_EQ(t.proc.src.len, 10); @@ -1113,7 +1113,7 @@ TEST(FilterProcessorInplaceMid, set_after_translate_esc_bulk_excess__trimmed_cap EXPECT_EQ(t.proc.src.len, 10); EXPECT_FALSE(t.proc.unfiltered_chars); EXPECT_EQ(t.proc.sofar(), ""); - t.proc.translate_esc("aaaa", /*nw*/4, /*nr*/1); + t.proc.translate_esc_extending("aaaa", /*nw*/4, /*nr*/1); EXPECT_EQ(t.proc.rpos, 2); EXPECT_EQ(t.proc.wpos, 4); EXPECT_EQ(t.proc.src.len, 10); @@ -1132,7 +1132,7 @@ TEST(FilterProcessorInplaceMid, set_after_translate_esc_bulk_excess__trimmed_cap } } -TEST(FilterProcessorInplaceMid, translate_esc_with_temporary_excess_requirement__trimmed_capacity) +TEST(FilterProcessorInplaceMid, translate_esc_extending_with_temporary_excess_requirement__trimmed_capacity) { TesterInplaceMid t("00112233445566"); t.trim_capacity(); @@ -1146,7 +1146,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_with_temporary_excess_requirement_ // 00112233445566 // ^ (rpos) // ^ (wpos) - t.proc.translate_esc("aaaa", /*nw*/4, /*nr*/1); + t.proc.translate_esc_extending("aaaa", /*nw*/4, /*nr*/1); EXPECT_EQ(t.proc.rpos, 2); EXPECT_EQ(t.proc.wpos, 4); EXPECT_EQ(t.proc.src.len, 14); @@ -1341,7 +1341,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_with_temporary_excess_requirement_ EXPECT_TRUE(t.proc.unfiltered_chars); } -TEST(FilterProcessorInplaceMid, translate_esc_after_translate_esc_bulk_excess__trimmed_capacity) +TEST(FilterProcessorInplaceMid, translate_esc_extending_after_translate_esc_extending_bulk_excess__trimmed_capacity) { { TesterInplaceMid t("0011223344"); @@ -1352,7 +1352,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_after_translate_esc_bulk_excess__t EXPECT_EQ(t.proc.src.len, 10); EXPECT_FALSE(t.proc.unfiltered_chars); EXPECT_EQ(t.proc.sofar(), ""); - t.proc.translate_esc("aaaa", /*nw*/4, /*nr*/1); + t.proc.translate_esc_extending("aaaa", /*nw*/4, /*nr*/1); EXPECT_EQ(t.proc.rpos, 2); EXPECT_EQ(t.proc.wpos, 4); EXPECT_EQ(t.proc.src.len, 10); @@ -1378,7 +1378,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_after_translate_esc_bulk_excess__t EXPECT_EQ(t.proc.src.len, 10); EXPECT_FALSE(t.proc.unfiltered_chars); EXPECT_EQ(t.proc.sofar(), ""); - t.proc.translate_esc("aaaa", /*nw*/4, /*nr*/1); + t.proc.translate_esc_extending("aaaa", /*nw*/4, /*nr*/1); EXPECT_EQ(t.proc.rpos, 2); EXPECT_EQ(t.proc.wpos, 4); EXPECT_EQ(t.proc.src.len, 10); @@ -1386,7 +1386,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_after_translate_esc_bulk_excess__t EXPECT_EQ(t.proc.result().str, nullptr); EXPECT_EQ(t.subject, "0011223344"); EXPECT_TRUE(t.proc.unfiltered_chars); - t.proc.translate_esc("!!", /*nw*/2, /*nr*/1); + t.proc.translate_esc_bulk("!!", /*nw*/2, /*nr*/1); EXPECT_EQ(t.proc.rpos, 4); EXPECT_EQ(t.proc.wpos, 6); EXPECT_EQ(t.proc.src.len, 10); @@ -1400,7 +1400,7 @@ TEST(FilterProcessorInplaceMid, translate_esc_after_translate_esc_bulk_excess__t //----------------------------------------------------------------------------- -TEST(FilterProcessorSrcDst, translate_esc_bulk_excess) +TEST(FilterProcessorSrcDst, translate_esc_extending_bulk_excess) { TesterSrcDst t("0011223344"); const size_t needed_size = 16u; @@ -1413,31 +1413,31 @@ TEST(FilterProcessorSrcDst, translate_esc_bulk_excess) EXPECT_EQ(t.proc.wpos, 0); EXPECT_EQ(t.proc.sofar(), ""); EXPECT_EQ(t.dst, csubstr("^^^^^^^^^^^^^^")); - t.proc.translate_esc("aaa", /*nw*/3, /*nr*/1); + t.proc.translate_esc_extending("aaa", /*nw*/3, /*nr*/1); EXPECT_EQ(t.proc.rpos, 2); EXPECT_EQ(t.proc.wpos, 3); EXPECT_EQ(t.proc.sofar(), "aaa"); EXPECT_EQ(t.proc.result().get(), "aaa"); EXPECT_EQ(t.dst, "aaa^^^^^^^^^^^"); - t.proc.translate_esc("bbb", /*nw*/3, /*nr*/1); + t.proc.translate_esc_extending("bbb", /*nw*/3, /*nr*/1); EXPECT_EQ(t.proc.rpos, 4); EXPECT_EQ(t.proc.wpos, 6); EXPECT_EQ(t.proc.sofar(), "aaabbb"); EXPECT_EQ(t.proc.result().get(), "aaabbb"); EXPECT_EQ(t.dst, "aaabbb^^^^^^^^"); - t.proc.translate_esc("ccc", /*nw*/3, /*nr*/1); + t.proc.translate_esc_extending("ccc", /*nw*/3, /*nr*/1); EXPECT_EQ(t.proc.rpos, 6); EXPECT_EQ(t.proc.wpos, 9); EXPECT_EQ(t.proc.sofar(), "aaabbbccc"); EXPECT_EQ(t.proc.result().get(), "aaabbbccc"); EXPECT_EQ(t.dst, "aaabbbccc^^^^^"); - t.proc.translate_esc("ddd", /*nw*/3, /*nr*/1); + t.proc.translate_esc_extending("ddd", /*nw*/3, /*nr*/1); EXPECT_EQ(t.proc.rpos, 8); EXPECT_EQ(t.proc.wpos, 12); EXPECT_EQ(t.proc.sofar(), "aaabbbcccddd"); EXPECT_EQ(t.proc.result().get(), "aaabbbcccddd"); EXPECT_EQ(t.dst, "aaabbbcccddd^^"); - t.proc.translate_esc("eeee", /*nw*/4, /*nr*/1); + t.proc.translate_esc_extending("eeee", /*nw*/4, /*nr*/1); EXPECT_EQ(t.proc.rpos, 10); EXPECT_EQ(t.proc.wpos, 16); EXPECT_EQ(t.proc.sofar(), "aaabbbcccddd^^"); diff --git a/test/test_plain_scalar.cpp b/test/test_plain_scalar.cpp index 4433553da..a5baadf09 100644 --- a/test/test_plain_scalar.cpp +++ b/test/test_plain_scalar.cpp @@ -60,6 +60,7 @@ void test_filter_inplace(csubstr input, csubstr expected, size_t indentation) auto run = [&](size_t cap){ // create the string std::string subject_(input.str, input.len); + std::string subject_2 = subject_; subject_.resize(full_sz); // fill the canary region const char refchar = '`'; @@ -67,10 +68,15 @@ void test_filter_inplace(csubstr input, csubstr expected, size_t indentation) full.sub(max_sz).fill(refchar); substr dst = full.first(input.len); // filter now - Parser proc = {}; - FilterResult result = proc.filter_scalar_plain_in_place(dst, cap, indentation); + Parser parser1 = {}; + FilterResult result = parser1.filter_scalar_plain_in_place(dst, cap, indentation); EXPECT_EQ(result.get().len, expected.len); EXPECT_EQ(result.required_len(), expected.len); + Parser parser2 = {}; + Tree tree = parser2.parse_in_arena("file", "# set the tree in the parser"); + csubstr sresult = parser2._filter_scalar_plain(to_substr(subject_2), indentation); + EXPECT_GE(result.required_len(), expected.len); + EXPECT_EQ(sresult.len, result.str.len); if(result.valid()) { const csubstr out = result.get(); diff --git a/test/test_single_quoted.cpp b/test/test_single_quoted.cpp index 4a459603d..7dfc1c967 100644 --- a/test/test_single_quoted.cpp +++ b/test/test_single_quoted.cpp @@ -33,9 +33,15 @@ void test_filter_inplace(csubstr input, csubstr expected) RYML_TRACE_FMT("\nstr=[{}]~~~{}~~~\nexp=[{}]~~~{}~~~", input.len, input, expected.len, expected); ASSERT_LE(expected.len, input.len); std::string subject_(input.str, input.len); + std::string subject_2 = subject_; c4::substr dst = to_substr(subject_); - Parser proc = {}; - FilterResult result = proc.filter_scalar_squoted_in_place(dst, subject_.size()); + Parser parser1 = {}; + FilterResult result = parser1.filter_scalar_squoted_in_place(dst, subject_.size()); + Parser parser2 = {}; + Tree tree = parser2.parse_in_arena("file", "# set the tree in the parser"); + csubstr sresult = parser2._filter_scalar_squot(to_substr(subject_2)); + EXPECT_GE(result.required_len(), expected.len); + EXPECT_EQ(sresult.len, result.str.len); ASSERT_TRUE(result.valid()); csubstr out = result.get(); ASSERT_TRUE(out.str);