Draft: new scalar filtering

.gitignore

@@ -33,6 +33,7 @@ build/
 install/
 .python-version
 compile_commands.json
+launch.json
 
 # test files
 /Testing/

CMakeLists.txt

@@ -36,9 +36,11 @@ c4_add_library(ryml
         c4/yml/common.cpp
         c4/yml/emit.def.hpp
         c4/yml/emit.hpp
+        c4/yml/filter_processor.hpp
         c4/yml/export.hpp
         c4/yml/node.hpp
         c4/yml/node.cpp
+        c4/yml/node_type.hpp
         c4/yml/parse.hpp
         c4/yml/parse.cpp
         c4/yml/preprocess.hpp

samples/quickstart.cpp

@@ -138,26 +138,32 @@ namespace sample {
 bool report_check(int line, const char *predicate, bool result);
 #ifdef __GNUC__
 #if __GNUC__ == 4 && __GNUC_MINOR__ >= 8
-struct CheckPredicate {
+#define CHECK CheckPredicate{__FILE__, __LINE__}
+struct CheckPredicate
+{
     const char *file;
     const int line;
-
     void operator() (bool predicate) const
     {
         if (!report_check(line, nullptr, predicate))
         {
+#ifdef RYML_DBG
             RYML_DEBUG_BREAK();
+#endif
         }
     }
 };
-#define CHECK CheckPredicate{__FILE__, __LINE__}
 #endif
 #endif
 
-#if !defined(CHECK)
+#ifndef CHECK
+#ifndef RYML_DBG
 /// a quick'n'dirty assertion to verify a predicate
+#define CHECK(predicate) report_check(__LINE__, #predicate, (predicate))
+#else
 #define CHECK(predicate) do { if(!report_check(__LINE__, #predicate, (predicate))) { RYML_DEBUG_BREAK(); } } while(0)
 #endif
+#endif
 
 
 //-----------------------------------------------------------------------------
@@ -3877,19 +3883,28 @@ void sample_error_handler()
 struct GlobalAllocatorExample
 {
     std::vector<char> memory_pool = std::vector<char>(10u * 1024u); // 10KB
-    size_t num_allocs = 0, alloc_size = 0;
+    size_t num_allocs = 0, alloc_size = 0, corr_size = 0;
     size_t num_deallocs = 0, dealloc_size = 0;
 
     void *allocate(size_t len)
     {
         void *ptr = &memory_pool[alloc_size];
         alloc_size += len;
         ++num_allocs;
-        if(C4_UNLIKELY(alloc_size > memory_pool.size()))
+        // ensure the ptr is aligned
+        uintptr_t uptr = (uintptr_t)ptr;
+        const uintptr_t align = alignof(max_align_t);
+        if (uptr % align)
         {
-            std::cerr << "out of memory! requested=" << alloc_size << " vs " << memory_pool.size() << " available" << std::endl;
-            std::abort();
+            uintptr_t prev = uptr - (uptr % align);
+            uintptr_t next = prev + align;
+            uintptr_t corr = next - uptr;
+            ptr = (void*)(((char*)ptr) + corr);
+            corr_size += corr;
         }
+        C4_CHECK_MSG(alloc_size + corr_size <= memory_pool.size(),
+                     "out of memory! requested=%zu+%zu available=%zu\n",
+                     alloc_size, corr_size, memory_pool.size());
         return ptr;
     }
 

src/c4/yml/common.cpp

@@ -89,9 +89,9 @@ Callbacks::Callbacks(void *user_data, pfn_allocate alloc_, pfn_free free_, pfn_e
     m_error(error_)
     #endif
 {
-    C4_CHECK(m_allocate);
-    C4_CHECK(m_free);
-    C4_CHECK(m_error);
+    RYML_CHECK(m_allocate);
+    RYML_CHECK(m_free);
+    RYML_CHECK(m_error);
 }
 
 

src/c4/yml/common.hpp

@@ -6,6 +6,17 @@
 #include <c4/yml/export.hpp>
 
 
+
+
+#include <cstdio>
+
+
+#ifndef RYML_ERRMSG_SIZE
+/// size for the error message buffer
+    #define RYML_ERRMSG_SIZE 1024
+#endif
+
+
 #ifndef RYML_USE_ASSERT
 #   define RYML_USE_ASSERT C4_USE_ASSERT
 #endif
@@ -20,24 +31,26 @@
 #endif
 
 
-#if defined(NDEBUG) || defined(C4_NO_DEBUG_BREAK)
+#ifndef RYML_DBG
+#   define RYML_DEBUG_BREAK()
+#elif (defined(NDEBUG) || defined(C4_NO_DEBUG_BREAK))
 #   define RYML_DEBUG_BREAK()
 #else
 #   define RYML_DEBUG_BREAK()                               \
-    {                                                       \
+    do {                                                    \
         if(c4::get_error_flags() & c4::ON_ERROR_DEBUGBREAK) \
         {                                                   \
             C4_DEBUG_BREAK();                               \
         }                                                   \
-    }
+    } while(0)
 #endif
 
 
 #define RYML_CHECK(cond)                                                \
     do {                                                                \
         if(!(cond))                                                     \
         {                                                               \
-            RYML_DEBUG_BREAK()                                          \
+            RYML_DEBUG_BREAK();                                         \
             c4::yml::error("check failed: " #cond, c4::yml::Location(__FILE__, __LINE__, 0)); \
         }                                                               \
     } while(0)
@@ -47,7 +60,7 @@
     {                                                                   \
         if(!(cond))                                                     \
         {                                                               \
-            RYML_DEBUG_BREAK()                                          \
+            RYML_DEBUG_BREAK();                                         \
             c4::yml::error(msg ": check failed: " #cond, c4::yml::Location(__FILE__, __LINE__, 0)); \
         }                                                               \
     } while(0)
@@ -197,7 +210,7 @@ RYML_EXPORT void reset_callbacks();
 do                                                                      \
 {                                                                       \
     const char msg[] = msg_literal;                                     \
-    RYML_DEBUG_BREAK()                                                  \
+    RYML_DEBUG_BREAK();                                                 \
     (cb).m_error(msg, sizeof(msg), c4::yml::Location(__FILE__, 0, __LINE__, 0), (cb).m_user_data); \
 } while(0)
 #define _RYML_CB_CHECK(cb, cond)                                        \
@@ -206,7 +219,7 @@ do                                                                      \
         if(!(cond))                                                     \
         {                                                               \
             const char msg[] = "check failed: " #cond;                  \
-            RYML_DEBUG_BREAK()                                          \
+            RYML_DEBUG_BREAK();                                         \
             (cb).m_error(msg, sizeof(msg), c4::yml::Location(__FILE__, 0, __LINE__, 0), (cb).m_user_data); \
         }                                                               \
     } while(0)
@@ -236,27 +249,61 @@ struct _charconstant_t
 } // namespace detail
 
 
+typedef enum {
+    BLOCK_LITERAL, //!< keep newlines (|)
+    BLOCK_FOLD     //!< replace newline with single space (>)
+} BlockStyle_e;
+
+typedef enum {
+    CHOMP_CLIP,    //!< single newline at end (default)
+    CHOMP_STRIP,   //!< no newline at end     (-)
+    CHOMP_KEEP     //!< all newlines from end (+)
+} BlockChomp_e;
+
+
+/** Abstracts the fact that a filter result may not fit in the intended memory. */
+struct FilterResult
+{
+    C4_ALWAYS_INLINE bool valid() const noexcept { return str.str != nullptr; }
+    C4_ALWAYS_INLINE size_t required_len() const noexcept { return str.len; }
+    C4_ALWAYS_INLINE csubstr get() { RYML_ASSERT(valid()); return str; }
+    csubstr str;
+};
+/** Abstracts the fact that a filter result may not fit in the intended memory. */
+struct FilterResultInPlace
+{
+    C4_ALWAYS_INLINE bool valid() const noexcept { return str.str != nullptr; }
+    C4_ALWAYS_INLINE size_t required_len() const noexcept { return reqlen; }
+    C4_ALWAYS_INLINE csubstr get() { RYML_ASSERT(valid()); return str; }
+    csubstr str;
+    size_t reqlen;
+};
+
+
 namespace detail {
 struct _SubstrWriter
 {
     substr buf;
     size_t pos;
-    _SubstrWriter(substr buf_, size_t pos_=0) : buf(buf_), pos(pos_) {}
+    _SubstrWriter(substr buf_, size_t pos_=0) : buf(buf_), pos(pos_) { C4_ASSERT(buf.str); }
     void append(csubstr s)
     {
         C4_ASSERT(!s.overlaps(buf));
-        if(pos + s.len <= buf.len)
+        C4_ASSERT(s.str || !s.len);
+        if(s.len && pos + s.len <= buf.len)
             memcpy(buf.str + pos, s.str, s.len);
         pos += s.len;
     }
     void append(char c)
     {
+        C4_ASSERT(buf.str);
         if(pos < buf.len)
             buf.str[pos] = c;
         ++pos;
     }
     void append_n(char c, size_t numtimes)
     {
+        C4_ASSERT(buf.str);
         if(pos + numtimes < buf.len)
             memset(buf.str + pos, c, numtimes);
         pos += numtimes;
@@ -274,6 +321,43 @@ struct _SubstrWriter
 
 /// @endcond
 
+
+//-----------------------------------------------------------------------------
+//-----------------------------------------------------------------------------
+//-----------------------------------------------------------------------------
+
+template<class DumpFn, class ...Args>
+C4_NO_INLINE void _parse_dump(DumpFn dumpfn, csubstr fmt, Args&& ...args)
+{
+    char writebuf[256];
+    auto results = format_dump_resume(dumpfn, writebuf, fmt, std::forward<Args>(args)...);
+    // resume writing if the results failed to fit the buffer
+    if(C4_UNLIKELY(results.bufsize > sizeof(writebuf))) // bufsize will be that of the largest element serialized. Eg int(1), will require 1 byte.
+    {
+        results = format_dump_resume(dumpfn, results, writebuf, fmt, std::forward<Args>(args)...);
+        if(C4_UNLIKELY(results.bufsize > sizeof(writebuf)))
+        {
+            results = format_dump_resume(dumpfn, results, writebuf, fmt, std::forward<Args>(args)...);
+            C4_CHECK(results.bufsize <= sizeof(writebuf));
+        }
+    }
+}
+template<class ...Args>
+void _report_err(Callbacks const& C4_RESTRICT callbacks, Location const& C4_RESTRICT loc, csubstr fmt, Args const& C4_RESTRICT ...args)
+{
+    char errmsg[RYML_ERRMSG_SIZE];
+    detail::_SubstrWriter writer(errmsg);
+    auto dumpfn = [&writer](csubstr s){ writer.append(s); };
+    _parse_dump(dumpfn, fmt, args...);
+    writer.append('\n');
+    if(loc.name.len)
+        _parse_dump(dumpfn, "{}:", loc.name);
+    _parse_dump(dumpfn, "{}:{}: ", loc.line, loc.col);
+    size_t len = writer.pos < RYML_ERRMSG_SIZE ? writer.pos : RYML_ERRMSG_SIZE;
+    printf("AQUI 0\n");
+    callbacks.m_error(errmsg, len, loc, callbacks.m_user_data);
+}
+
 C4_SUPPRESS_WARNING_GCC_CLANG_POP
 
 } // namespace yml

src/c4/yml/detail/parser_dbg.hpp

@@ -24,10 +24,14 @@
 #pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments"
 
 // some debugging scaffolds
-#ifdef RYML_DBG
 #include <c4/dump.hpp>
+#ifdef RYML_DBG
 namespace c4 {
-inline void _dbg_dumper(csubstr s) { fwrite(s.str, 1, s.len, stdout); };
+inline void _dbg_dumper(csubstr s)
+{
+    if(s.str)
+        fwrite(s.str, 1, s.len, stdout);
+}
 template<class ...Args>
 void _dbg_printf(c4::csubstr fmt, Args&& ...args)
 {
@@ -50,21 +54,24 @@ void _dbg_printf(c4::csubstr fmt, Args&& ...args)
 #   define _c4dbgp(msg)        _dbg_printf("{}:{}: "   msg "\n", __FILE__, __LINE__                )
 #   define _c4dbgq(msg)        _dbg_printf(msg "\n")
 #   define _c4err(fmt, ...)   \
-    do { if(c4::is_debugger_attached()) { C4_DEBUG_BREAK(); } \
-         this->_err("ERROR:\n" "{}:{}: " fmt, __FILE__, __LINE__, ## __VA_ARGS__); } while(0)
+    do { RYML_DEBUG_BREAK(); this->_err("ERROR:\n" "{}:{}: " fmt, __FILE__, __LINE__, ## __VA_ARGS__); } while(0)
 #else
 #   define _c4dbgt(fmt, ...)
 #   define _c4dbgpf(fmt, ...)
 #   define _c4dbgp(msg)
 #   define _c4dbgq(msg)
 #   define _c4err(fmt, ...)   \
-    do { if(c4::is_debugger_attached()) { C4_DEBUG_BREAK(); } \
-         this->_err("ERROR: " fmt, ## __VA_ARGS__); } while(0)
+    this->_err("ERROR: " fmt, ## __VA_ARGS__)
 #endif
 
+
+//-----------------------------------------------------------------------------
+//-----------------------------------------------------------------------------
+//-----------------------------------------------------------------------------
+
 #define _c4prsp(sp) sp
 #define _c4presc(s) __c4presc(s.str, s.len)
-inline c4::csubstr _c4prc(const char &C4_RESTRICT c)
+inline c4::csubstr _c4prc(const char &C4_RESTRICT c) // pass by reference!
 {
     switch(c)
     {
@@ -81,6 +88,7 @@ inline c4::csubstr _c4prc(const char &C4_RESTRICT c)
 }
 inline void __c4presc(const char *s, size_t len)
 {
+    RYML_ASSERT(s || !len);
     size_t prev = 0;
     for(size_t i = 0; i < len; ++i)
     {
@@ -123,7 +131,8 @@ inline void __c4presc(const char *s, size_t len)
             }
         }
     }
-    fwrite(s + prev, 1, len - prev, stdout);
+    if(len > prev)
+        fwrite(s + prev, 1, len - prev, stdout);
 }
 
 #pragma clang diagnostic pop

src/c4/yml/detail/stack.hpp

@@ -177,6 +177,7 @@ void stack<T, N>::reserve(size_t sz)
         return;
     }
     T *buf = (T*) m_callbacks.m_allocate(sz * sizeof(T), m_stack, m_callbacks.m_user_data);
+    RYML_ASSERT(((uintptr_t)buf % alignof(T)) == 0u);
     memcpy(buf, m_stack, m_size * sizeof(T));
     if(m_stack != m_buf)
     {

src/c4/yml/emit.hpp

@@ -16,11 +16,11 @@
 
 #define RYML_DEPRECATE_EMIT                                             \
     RYML_DEPRECATED("use emit_yaml() instead. See https://github.com/biojppm/rapidyaml/issues/120")
+#define RYML_DEPRECATE_EMITRS                                           \
+    RYML_DEPRECATED("use emitrs_yaml() instead. See https://github.com/biojppm/rapidyaml/issues/120")
 #ifdef emit
 #error "emit is defined, likely from a Qt include. This will cause a compilation error. See https://github.com/biojppm/rapidyaml/issues/120"
 #endif
-#define RYML_DEPRECATE_EMITRS                                           \
-    RYML_DEPRECATED("use emitrs_yaml() instead. See https://github.com/biojppm/rapidyaml/issues/120")
 
 
 //-----------------------------------------------------------------------------