rapidyaml 0.16.0
parse and emit YAML, and do it fast
Loading...
Searching...
No Matches
parse_engine.def.hpp
Go to the documentation of this file.
1#ifndef C4_YML_PARSE_ENGINE_DEF_HPP_
2#define C4_YML_PARSE_ENGINE_DEF_HPP_
3
4#ifndef C4_YML_PARSE_ENGINE_HPP_
5#include "c4/yml/parse_engine.hpp"
6#endif
7#ifndef C4_CHARCONV_HPP_
8#include "c4/charconv.hpp"
9#endif
10#ifndef C4_UTF_HPP_
11#include "c4/utf.hpp"
12#endif
13#ifndef C4_YML_FILTER_PROCESSOR_HPP_
14#include "c4/yml/filter_processor.hpp"
15#endif
16#ifndef C4_YML_TAG_HPP_
17#include "c4/yml/tag.hpp"
18#endif
19#ifndef C4_YML_NODE_TYPE_HPP_
20#include "c4/yml/node_type.hpp"
21#endif
22
23#ifndef C4_YML_DETAIL_DBGPRINT_HPP_
24#include "c4/yml/detail/dbgprint.hpp"
25#endif
26
27#ifdef RYML_DBG
28#ifndef C4_DUMP_HPP_
29#include <c4/dump.hpp>
30#endif
31#define _c4err(...) \
32 do { RYML_DEBUG_BREAK(); this->_err(RYML_LOC_HERE(), __VA_ARGS__); } while(0)
33#else
34#define _c4err(...) \
35 this->_err(RYML_LOC_HERE(), __VA_ARGS__)
36#endif
37#define _c4assert(...) \
38 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, __VA_ARGS__, m_evt_handler->m_curr->pos)
39
40
41#if defined(RYML_WITH_TAB_TOKENS)
42#define RYML_WITH_TAB_TOKENS_(...) __VA_ARGS__
43#define RYML_WITHOUT_TAB_TOKENS_(...)
44#define RYML_WITH_OR_WITHOUT_TAB_TOKENS_(with, without) with
45#else
46#define RYML_WITH_TAB_TOKENS_(...)
47#define RYML_WITHOUT_TAB_TOKENS_(...) __VA_ARGS__
48#define RYML_WITH_OR_WITHOUT_TAB_TOKENS_(with, without) without
49#endif
50
51
52// scaffold:
53#define _c4dbgnextline() \
54 do { \
55 _c4dbgq("\n-----------"); \
56 _c4dbgt("handling line={}, offset={}B", \
57 m_evt_handler->m_curr->pos.line, \
58 m_evt_handler->m_curr->pos.offset); \
59 } while(0)
60
61
62C4_SUPPRESS_WARNING_MSVC_PUSH
63C4_SUPPRESS_WARNING_MSVC(4296) // expression is always 'boolean_value'
64C4_SUPPRESS_WARNING_MSVC(4702) // unreachable code
65C4_SUPPRESS_WARNING_GCC_CLANG_PUSH
66C4_SUPPRESS_WARNING_GCC_CLANG("-Wtype-limits") // to remove a warning on an assertion that a size_t >= 0. Later on, this size_t will turn into a template argument, and then it can become < 0.
67C4_SUPPRESS_WARNING_GCC_CLANG("-Wformat-nonliteral")
68C4_SUPPRESS_WARNING_GCC_CLANG("-Wold-style-cast")
69#if defined(__GNUC__) && (__GNUC__ >= 6)
70C4_SUPPRESS_WARNING_GCC("-Wnull-dereference")
71#endif
72#if defined(__GNUC__) && (__GNUC__ >= 7)
73C4_SUPPRESS_WARNING_GCC("-Wduplicated-branches")
74#endif
75
76// NOLINTBEGIN(hicpp-signed-bitwise,cppcoreguidelines-avoid-goto,hicpp-avoid-goto,hicpp-multiway-paths-covered,modernize-avoid-c-style-cast)
77
78namespace c4 {
79namespace yml {
80
81namespace { // NOLINT
82
83C4_HOT C4_ALWAYS_INLINE void _set_first(substr &C4_RESTRICT subject, size_t pos) noexcept
84{
85 // avoids reassigning the ptr in substr
86 subject.len = pos != npos ? pos : subject.len;
87}
88C4_HOT C4_ALWAYS_INLINE void _set_first(csubstr &C4_RESTRICT subject, size_t pos) noexcept
89{
90 // avoids reassigning the ptr in substr
91 subject.len = pos != npos ? pos : subject.len;
92}
93C4_HOT C4_ALWAYS_INLINE void _set_first_strict(substr &C4_RESTRICT subject, size_t pos) RYML_NOEXCEPT
94{
95 // avoids reassigning the ptr in substr
96 RYML_ASSERT_BASIC_(pos != npos); // LCOV_EXCL_LINE
97 subject.len = pos;
98}
99C4_HOT C4_ALWAYS_INLINE void _set_first_strict(csubstr &C4_RESTRICT subject, size_t pos) RYML_NOEXCEPT
100{
101 // avoids reassigning the ptr in substr
102 RYML_ASSERT_BASIC_(pos != npos); // LCOV_EXCL_LINE
103 subject.len = pos;
104}
105
106C4_HOT C4_ALWAYS_INLINE bool _is_blck_token(csubstr s) RYML_NOEXCEPT
107{
108 RYML_ASSERT_BASIC_(s.len > 0);
109 RYML_ASSERT_BASIC_(s.str[0] == '-' || s.str[0] == ':' || s.str[0] == '?');
110 return ((s.len == 1) || ((s.str[1] == ' ') RYML_WITH_TAB_TOKENS_( || (s.str[1] == '\t'))));
111}
112
113C4_HOT C4_ALWAYS_INLINE bool _is_blck_seq_token_maybe(csubstr const& C4_RESTRICT s) noexcept
114{
115 return ((s.len >= 1) && (s.str[0] == '-') && ((s.len == 1) || ((s.str[1] == ' ') RYML_WITH_TAB_TOKENS_( || (s.str[1] == '\t')))));
116}
117
118inline bool _is_doc_begin_token(csubstr s) RYML_NOEXCEPT
119{
120 RYML_ASSERT_BASIC_(s.begins_with('-'));
121 RYML_ASSERT_BASIC_(!s.ends_with("\n"));
122 RYML_ASSERT_BASIC_(!s.ends_with("\r"));
123 return (s.len >= 3 && s.str[1] == '-' && s.str[2] == '-')
124 && (s.len == 3 || (s.str[3] == ' ' RYML_WITH_TAB_TOKENS_(|| s.str[3] == '\t')));
125}
126
127inline bool _is_doc_end_token(csubstr s) RYML_NOEXCEPT
128{
129 RYML_ASSERT_BASIC_(s.begins_with('.'));
130 RYML_ASSERT_BASIC_(!s.ends_with("\n"));
131 RYML_ASSERT_BASIC_(!s.ends_with("\r"));
132 return (s.len >= 3 && s.str[1] == '.' && s.str[2] == '.')
133 && (s.len == 3 || (s.str[3] == ' ' RYML_WITH_TAB_TOKENS_(|| s.str[3] == '\t')));
134}
135
136inline bool _is_doc_token(csubstr s) noexcept
137{
138 if(s.len >= 3)
139 {
140 switch(s.str[0])
141 {
142 case '-':
143 //return _is_doc_begin_token(s); // this was failing with gcc -O2
144 return (s.str[1] == '-' && s.str[2] == '-')
145 && (s.len == 3 || (s.str[3] == ' ' RYML_WITH_TAB_TOKENS_(|| s.str[3] == '\t')));
146 case '.':
147 //return _is_doc_end_token(s); // this was failing with gcc -O2
148 return (s.str[1] == '.' && s.str[2] == '.')
149 && (s.len == 3 || (s.str[3] == ' ' RYML_WITH_TAB_TOKENS_(|| s.str[3] == '\t')));
150 }
151 }
152 return false;
153}
154
155inline size_t _begins_with_special_json_scalar(csubstr s) RYML_NOEXCEPT
156{
157 RYML_ASSERT_BASIC_(s.len);
158 switch(s.str[0])
159 {
160 case 'f':
161 return s.begins_with("false") ? 5u : 0u;
162 case 't':
163 return s.begins_with("true") ? 4u : 0u;
164 case 'n':
165 return s.begins_with("null") ? 4u : 0u;
166 }
167 return 0u;
168}
169
170
171//-----------------------------------------------------------------------------
172
173C4_ALWAYS_INLINE size_t _extend_from_combined_newline(char nl, char following)
174{
175 return (nl == '\n' && following == '\r') || (nl == '\r' && following == '\n');
176}
177
178//! look for the next newline chars, and jump to the right of those
179inline substr _from_next_line(substr rem)
180{
181 size_t nlpos = rem.first_of("\r\n");
182 if(nlpos == csubstr::npos)
183 return {};
184 const char nl = rem[nlpos];
185 rem = rem.right_of(nlpos);
186 if(rem.empty())
187 return {};
188 if(_extend_from_combined_newline(nl, rem.front()))
189 rem = rem.sub(1);
190 return rem;
191}
192
193
194//-----------------------------------------------------------------------------
195
196inline size_t _count_following_newlines(csubstr r, size_t *C4_RESTRICT i)
197{
198 RYML_ASSERT_BASIC_(r[*i] == '\n');
199 size_t numnl_following = 0;
200 ++(*i);
201 for( ; *i < r.len; ++(*i))
202 {
203 if(r.str[*i] == '\n')
204 ++numnl_following;
205 // skip leading whitespace
206 else if(r.str[*i] == ' ' || r.str[*i] == '\t' || r.str[*i] == '\r')
207 ;
208 else
209 break;
210 }
211 return numnl_following;
212}
213
214/** @p i is set to the first non whitespace character after the line
215 * @return the number of empty lines after the initial position */
216inline size_t _count_following_newlines(csubstr r, size_t *C4_RESTRICT i, size_t indentation)
217{
218 RYML_ASSERT_BASIC_(r[*i] == '\n');
219 size_t numnl_following = 0;
220 ++(*i);
221 if(indentation == 0)
222 {
223 for( ; *i < r.len; ++(*i))
224 {
225 const char c = r.str[*i];
226 if(c == '\n')
227 ++numnl_following;
228 // skip leading whitespace
229 else if(c != ' ' && c != '\t' && c != '\r')
230 break;
231 }
232 }
233 else
234 {
235 for( ; *i < r.len; ++(*i))
236 {
237 char c = r.str[*i];
238 if(c == '\n')
239 {
240 ++numnl_following;
241 // skip the indentation after the newline
242 size_t stop = *i + indentation;
243 for( ; *i < r.len; ++(*i))
244 {
245 c = r.str[*i];
246 if(c != ' ' && c != '\r')
247 break;
248 RYML_ASSERT_BASIC_(*i < stop); // LCOV_EXCL_LINE
249 }
250 C4_UNUSED(stop);
251 }
252 // skip leading whitespace
253 else if(c != ' ' && c != '\t' && c != '\r')
254 {
255 break;
256 }
257 }
258 }
259 return numnl_following;
260}
261
262} // anon namespace
263
264
265//-----------------------------------------------------------------------------
266//-----------------------------------------------------------------------------
267//-----------------------------------------------------------------------------
268
269template<class EventHandler>
270ParseEngine<EventHandler>::~ParseEngine() noexcept
271{
272 _free();
273 _clr();
274}
275
276template<class EventHandler>
277ParseEngine<EventHandler>::ParseEngine(EventHandler *evt_handler, ParserOptions const& opts)
278 : m_options(opts)
279 , m_evt_handler(evt_handler)
280 , m_pending_anchors()
281 , m_pending_tags()
282 , m_has_directives_yaml(false)
283 , m_has_directives(false)
284 , m_doc_empty(true)
285 , m_prev_colon(npos)
286 , m_prev_val_end(npos)
287 , m_encoding(NOBOM)
288 , m_newline_offsets()
289 , m_newline_offsets_size(0)
290 , m_newline_offsets_capacity(0)
291{
292 RYML_CHECK_BASIC_(evt_handler);
293}
294
295template<class EventHandler>
296ParseEngine<EventHandler>::ParseEngine(ParseEngine &&that) noexcept
297 : m_options(that.m_options)
298 , m_evt_handler(that.m_evt_handler)
299 , m_pending_anchors(that.m_pending_anchors)
300 , m_pending_tags(that.m_pending_tags)
301 , m_has_directives_yaml(that.m_has_directives_yaml)
302 , m_has_directives(that.m_has_directives)
303 , m_doc_empty(that.m_doc_empty)
304 , m_prev_colon(npos)
305 , m_prev_val_end(npos)
306 , m_encoding(NOBOM)
307 , m_newline_offsets(that.m_newline_offsets)
308 , m_newline_offsets_size(that.m_newline_offsets_size)
309 , m_newline_offsets_capacity(that.m_newline_offsets_capacity)
310{
311 that._clr();
312}
313
314template<class EventHandler>
315ParseEngine<EventHandler>::ParseEngine(ParseEngine const& that)
316 : m_options(that.m_options)
317 , m_evt_handler(that.m_evt_handler)
318 , m_pending_anchors(that.m_pending_anchors)
319 , m_pending_tags(that.m_pending_tags)
320 , m_has_directives_yaml(that.m_has_directives_yaml)
321 , m_has_directives(that.m_has_directives)
322 , m_doc_empty(that.m_doc_empty)
323 , m_prev_colon(npos)
324 , m_prev_val_end(npos)
325 , m_encoding(NOBOM)
326 , m_newline_offsets()
327 , m_newline_offsets_size()
328 , m_newline_offsets_capacity()
329{
330 if(that.m_newline_offsets_capacity)
331 {
332 _resize_locations(that.m_newline_offsets_capacity);
333 RYML_CHECK_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets_capacity == that.m_newline_offsets_capacity);
334 memcpy(m_newline_offsets, that.m_newline_offsets, that.m_newline_offsets_size * sizeof(size_t));
335 m_newline_offsets_size = that.m_newline_offsets_size;
336 }
337}
338
339template<class EventHandler>
340ParseEngine<EventHandler>& ParseEngine<EventHandler>::operator=(ParseEngine &&that) noexcept
341{
342 _free();
343 m_options = (that.m_options);
344 m_evt_handler = that.m_evt_handler;
345 m_pending_anchors = that.m_pending_anchors;
346 m_pending_tags = that.m_pending_tags;
347 m_has_directives_yaml = that.m_has_directives_yaml;
348 m_has_directives = that.m_has_directives;
349 m_doc_empty = that.m_doc_empty;
350 m_prev_colon = that.m_prev_colon;
351 m_prev_val_end = that.m_prev_val_end;
352 m_encoding = that.m_encoding;
353 m_newline_offsets = (that.m_newline_offsets);
354 m_newline_offsets_size = (that.m_newline_offsets_size);
355 m_newline_offsets_capacity = (that.m_newline_offsets_capacity);
356 that._clr();
357 return *this;
358}
359
360template<class EventHandler>
361ParseEngine<EventHandler>& ParseEngine<EventHandler>::operator=(ParseEngine const& that)
362{
363 if(&that != this)
364 {
365 _free();
366 m_options = (that.m_options);
367 m_evt_handler = that.m_evt_handler;
368 m_pending_anchors = that.m_pending_anchors;
369 m_pending_tags = that.m_pending_tags;
370 m_has_directives_yaml = that.m_has_directives_yaml;
371 m_has_directives = that.m_has_directives;
372 m_doc_empty = that.m_doc_empty;
373 m_prev_colon = that.m_prev_colon;
374 m_prev_val_end = that.m_prev_val_end;
375 m_encoding = that.m_encoding;
376 if(that.m_newline_offsets_capacity > m_newline_offsets_capacity)
377 _resize_locations(that.m_newline_offsets_capacity);
378 RYML_CHECK_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets_capacity >= that.m_newline_offsets_capacity);
379 RYML_CHECK_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets_capacity >= that.m_newline_offsets_size);
380 memcpy(m_newline_offsets, that.m_newline_offsets, that.m_newline_offsets_size * sizeof(size_t));
381 m_newline_offsets_size = that.m_newline_offsets_size;
382 }
383 return *this;
384}
385
386template<class EventHandler>
387void ParseEngine<EventHandler>::_clr()
388{
389 m_options = {};
390 m_evt_handler = {};
391 m_pending_anchors = {};
392 m_pending_tags = {};
393 m_has_directives_yaml = false;
394 m_has_directives = false;
395 m_doc_empty = true;
396 m_prev_colon = npos;
397 m_prev_val_end = npos;
398 m_encoding = NOBOM;
399 m_newline_offsets = {};
400 m_newline_offsets_size = {};
401 m_newline_offsets_capacity = {};
402}
403
404template<class EventHandler>
405void ParseEngine<EventHandler>::_free()
406{
407 if(m_newline_offsets)
408 {
409 RYML_CB_FREE_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets, size_t, m_newline_offsets_capacity);
410 m_newline_offsets = nullptr;
411 m_newline_offsets_size = 0u;
412 m_newline_offsets_capacity = 0u;
413 }
414}
415
416
417//-----------------------------------------------------------------------------
418
419template<class EventHandler>
420void ParseEngine<EventHandler>::_reset()
421{
422 m_pending_anchors = {};
423 m_pending_tags = {};
424 m_has_directives_yaml = false;
425 m_has_directives = false;
426 m_doc_empty = true;
427 m_prev_colon = npos;
428 m_prev_val_end = npos;
429 m_bom_len = 0;
430 m_encoding = NOBOM;
431 m_bom_line = 0;
432 if(m_options.locations())
433 {
434 _prepare_locations();
435 }
436}
437
438
439//-----------------------------------------------------------------------------
440
441template<class EventHandler>
442void ParseEngine<EventHandler>::_relocate_arena(csubstr prev_arena, substr next_arena, substr *other)
443{
444 _c4dbgp("relocate to new arena");
445 const char *pb = prev_arena.str;
446 const char *pe = prev_arena.str + prev_arena.len;
447 #define _ryml_relocate(s) \
448 if((s).str >= pb && (s).str <= pe) \
449 { \
450 (s).str = next_arena.str + ((s).str - pb); \
451 } \
452 ((void)0)
453 for(ParserState &st : m_evt_handler->m_stack)
454 {
455 _ryml_relocate(st.line_contents.rem);
456 _ryml_relocate(st.line_contents.full);
457 }
458 _ryml_relocate(m_evt_handler->m_src);
459 for(size_t i = 0; i < m_pending_tags.num_entries; ++i)
460 {
461 _ryml_relocate(m_pending_tags.annotations[i].str); // LCOV_EXCL_LINE
462 _ryml_relocate(m_pending_tags.annotations[i].orig); // LCOV_EXCL_LINE
463 }
464 for(size_t i = 0; i < m_pending_anchors.num_entries; ++i)
465 {
466 _ryml_relocate(m_pending_anchors.annotations[i].str);
467 _ryml_relocate(m_pending_anchors.annotations[i].orig);
468 }
469 {
470 TagDirectives &tds = m_evt_handler->tag_directives();
471 for(size_t i = 0, sz = tds.size(); i < sz; ++i)
472 {
473 _ryml_relocate(tds.m_directives[i].handle);
474 _ryml_relocate(tds.m_directives[i].prefix);
475 }
476 }
477 {
478 TagCache &tch = m_evt_handler->tag_cache();
479 for(id_type i = 0, sz = tch.m_entries.size(); i < sz; ++i)
480 {
481 _ryml_relocate(tch.m_entries[i].tag);
482 _ryml_relocate(tch.m_entries[i].resolved);
483 }
484 }
485 if(other)
486 {
487 _ryml_relocate(*other);
488 }
489 #undef _ryml_relocate
490}
491
492/** @cond dev */
493template<class EventHandler>
494substr ParseEngine<EventHandler>::_alloc_arena(size_t len, substr *other)
495{
496 csubstr prev = m_evt_handler->arena();
497 substr out = m_evt_handler->alloc_arena(len);
498 substr curr = m_evt_handler->arena();
499 if(curr.str != prev.str)
500 _relocate_arena(prev, curr, other);
501 return out;
502}
503/** @endcond */
504
505
506//-----------------------------------------------------------------------------
507
508#ifdef RYML_DBG
509template<class EventHandler>
510template<class DumpFn>
511C4_NO_INLINE void ParseEngine<EventHandler>::_fmt_msg(DumpFn &&dumpfn) const
512{
513 ParserState const *const C4_RESTRICT st = m_evt_handler->m_curr;
514 LineContents const& C4_RESTRICT lc = st->line_contents;
515 csubstr contents = lc.full.first(lc.num_cols);
516 if(contents.len)
517 {
518 // print the yaml src line
519 size_t offs = 3u + to_chars(substr{}, st->pos.line) + to_chars(substr{}, st->pos.col);
520 csubstr m_file = m_evt_handler->m_curr->pos.name;
521 if(m_file.len)
522 {
523 dbg_dump_(std::forward<DumpFn>(dumpfn), "{}:", m_file);
524 offs += m_file.len + 1;
525 }
526 dbg_dump_(std::forward<DumpFn>(dumpfn), "{}:{}: ", st->pos.line, st->pos.col);
527 csubstr maybe_full_content = (contents.len < 80u ? contents : contents.first(80u));
528 csubstr maybe_ellipsis = (contents.len < 80u ? csubstr{} : csubstr("..."));
529 dbg_dump_(std::forward<DumpFn>(dumpfn), "{}{} (size={})\n", escaped_scalar(maybe_full_content, /*escape*/true), maybe_ellipsis, contents.len);
530 // highlight the remaining portion of the previous line
531 size_t firstcol = (size_t)(lc.rem.str - lc.full.str);
532 size_t lastcol = firstcol + lc.rem.len;
533 size_t firstcol_adj = adjust_pos_with_escapes(lc.full, firstcol);
534 size_t len = adjust_pos_with_escapes(lc.rem, lc.rem.len);
535 for(size_t i = 0; i < offs + firstcol_adj; ++i)
536 std::forward<DumpFn>(dumpfn)(" ");
537 std::forward<DumpFn>(dumpfn)("^");
538 for(size_t i = 1, e = (len < 80u ? len : 80u); i < e; ++i)
539 std::forward<DumpFn>(dumpfn)("~");
540 dbg_dump_(std::forward<DumpFn>(dumpfn), "{} (cols {}-{})\n", maybe_ellipsis, firstcol+1, lastcol+1);
541 }
542 else
543 {
544 std::forward<DumpFn>(dumpfn)("\n");
545 }
546 // next line: print the state flags
547 {
548 char flagbuf_[128];
549 dbg_dump_(std::forward<DumpFn>(dumpfn), "top state: {}\n", detail::_parser_flags_to_str(flagbuf_, m_evt_handler->m_curr->flags));
550 }
551}
552
553template<class EventHandler>
554void ParseEngine<EventHandler>::_print_state_stack(substr buf) const
555{
556 if(dbg_enabled_())
557 {
558 for(ParserState const& s : m_evt_handler->m_stack)
559 dbg_printf_("state[{}]: ind={} node={} flags={}\n", s.level, s.indref, s.node_id, detail::_parser_flags_to_str(buf, s.flags));
560 }
561}
562
563template<class EventHandler>
564void ParseEngine<EventHandler>::_print_state_stack() const
565{
566 char buf[128];
567 _print_state_stack(buf);
568}
569#endif
570
571
572//-----------------------------------------------------------------------------
573
574template<class EventHandler>
575template<class ...Args>
576C4_NORETURN C4_NO_INLINE void ParseEngine<EventHandler>::_err(Location const& cpploc, Location const& ymlloc, const char* fmt, Args const& ...args) const
577{
578 m_evt_handler->cancel_parse();
579 err_parse(m_evt_handler->m_stack.m_callbacks, ErrorDataParse{cpploc, ymlloc}, fmt, args...);
580}
581
582template<class EventHandler>
583template<class ...Args>
584C4_NORETURN C4_NO_INLINE void ParseEngine<EventHandler>::_err(Location const& cpploc, const char *fmt, Args const& ...args) const
585{
586 m_evt_handler->cancel_parse();
587 err_parse(m_evt_handler->m_stack.m_callbacks, ErrorDataParse{cpploc, m_evt_handler->m_curr->pos}, fmt, args...);
588}
589
590
591//-----------------------------------------------------------------------------
592#ifdef RYML_DBG
593template<class EventHandler>
594template<class ...Args>
595void ParseEngine<EventHandler>::_dbg(csubstr fmt, Args const& ...args) const
596{
597 if(dbg_enabled_())
598 {
599 dbg_printf_(fmt, args...);
600 dbg_dumper_("\n");
601 _fmt_msg(dbg_dumper_);
602 }
603}
604#endif
605
606
607//-----------------------------------------------------------------------------
608template<class EventHandler>
609bool ParseEngine<EventHandler>::_finished_file() const
610{
611 bool ret = m_evt_handler->m_curr->pos.offset >= _buf().len;
612 #ifdef RYML_DBG
613 if(ret)
614 {
615 _c4dbgp("finished file!!!");
616 }
617 #endif
618 return ret;
619}
620
621template<class EventHandler>
622C4_HOT C4_ALWAYS_INLINE bool ParseEngine<EventHandler>::_finished_line() const // LCOV_EXCL_LINE
623{
624 return m_evt_handler->m_curr->line_contents.rem.empty();
625}
626
627
628//-----------------------------------------------------------------------------
629
630template<class EventHandler>
631void ParseEngine<EventHandler>::_maybe_skip_whitespace_tokens()
632{
633 if(m_evt_handler->m_curr->line_contents.rem.len && (m_evt_handler->m_curr->line_contents.rem.str[0] == ' ' RYML_WITH_TAB_TOKENS_(|| m_evt_handler->m_curr->line_contents.rem.str[0] == '\t')))
634 {
635 size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of(RYML_WITH_OR_WITHOUT_TAB_TOKENS_(" \t", ' '));
636 if(pos == npos)
637 pos = m_evt_handler->m_curr->line_contents.rem.len; // maybe the line is just all whitespace
638 _c4dbgpf("skip {} whitespace characters", pos);
639 _line_progressed(pos);
640 }
641}
642
643template<class EventHandler>
644void ParseEngine<EventHandler>::_maybe_skipchars(char c)
645{
646 if(m_evt_handler->m_curr->line_contents.rem.len && m_evt_handler->m_curr->line_contents.rem.str[0] == c)
647 {
648 size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of(c);
649 if(pos == npos)
650 pos = m_evt_handler->m_curr->line_contents.rem.len; // maybe the line is just all c
651 _c4dbgpf("skip {}x'{}'", pos, _c4prc(c));
652 _line_progressed(pos);
653 }
654}
655
656template<class EventHandler>
657template<size_t N>
658void ParseEngine<EventHandler>::_skipchars(const char (&chars)[N])
659{
660 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->line_contents.rem.begins_with_any(chars), m_evt_handler->m_curr->pos);
661 size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of(chars);
662 if(pos == npos)
663 pos = m_evt_handler->m_curr->line_contents.rem.len; // maybe the line is just whitespace
664 _c4dbgpf("skip {} characters", pos);
665 _line_progressed(pos);
666}
667
668template<class EventHandler>
669void ParseEngine<EventHandler>::_skip_comment()
670{
671 LineContents const& C4_RESTRICT lc = m_evt_handler->m_curr->line_contents;
672 const size_t col = m_evt_handler->m_curr->pos.col - 1u;
673 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, lc.rem.begins_with('#'), m_evt_handler->m_curr->pos);
674 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, lc.rem.is_sub(lc.full), m_evt_handler->m_curr->pos);
675 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.col >= 1, m_evt_handler->m_curr->pos); // 1-based
676 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, col == ((size_t)(lc.rem.str - lc.full.str)), m_evt_handler->m_curr->pos);
677 // raise an error if the comment is not preceded by whitespace
678 if(lc.rem.str != lc.full.str) // not at line beginning
679 {
680 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, col > 0, m_evt_handler->m_curr->pos);
681 const char prev = lc.full.str[col - 1u];
682 if C4_UNLIKELY(prev != ' ' && prev != '\t')
683 _c4err("comment not preceded by whitespace");
684 }
685 _c4dbgpf("comment was '{}'", m_evt_handler->m_curr->line_contents.rem);
686 _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
687}
688
689template<class EventHandler>
690void ParseEngine<EventHandler>::_maybe_skip_comment_strict()
691{
692 size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of(" \t");
693 if(pos != npos)
694 {
695 if('#' == m_evt_handler->m_curr->line_contents.rem[pos])
696 {
697 _line_progressed(pos);
698 _skip_comment();
699 }
700 }
701}
702
703template<class EventHandler>
704void ParseEngine<EventHandler>::_maybe_skip_comment()
705{
706 size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of(" \t");
707 if(pos != npos)
708 {
709 if('#' == m_evt_handler->m_curr->line_contents.rem[pos])
710 {
711 _line_progressed(pos);
712 _skip_comment();
713 }
714 }
715 else
716 {
717 _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
718 }
719}
720
721template<class EventHandler>
722bool ParseEngine<EventHandler>::_maybe_scan_following_colon() noexcept
723{
724 size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of(" \t");
725 if(pos != npos)
726 {
727 if(':' == m_evt_handler->m_curr->line_contents.rem[pos])
728 {
729 // bump pos to skip the colon as well, and check the colon
730 // is followed by space or tab
731 if(++pos < m_evt_handler->m_curr->line_contents.rem.len)
732 {
733 const char next = m_evt_handler->m_curr->line_contents.rem.str[pos];
734 if(next == ' ' RYML_WITH_TAB_TOKENS_(|| next == '\t'))
735 ++pos;
736 else
737 return false;
738 }
739 _line_progressed(pos);
740 return true;
741 }
742 }
743 else
744 {
745 _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
746 }
747 return false;
748}
749
750
751//-----------------------------------------------------------------------------
752
753template<class EventHandler>
754csubstr ParseEngine<EventHandler>::_scan_anchor()
755{
756 csubstr s = m_evt_handler->m_curr->line_contents.rem;
757 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with('&'), m_evt_handler->m_curr->pos);
758 csubstr anchor = s.range(1, s.first_of(" ,]}\t"));
759 _line_progressed(1u + anchor.len);
760 _maybe_skipchars(' ');
761 return anchor;
762}
763
764template<class EventHandler>
765csubstr ParseEngine<EventHandler>::_scan_ref_seq()
766{
767 csubstr s = m_evt_handler->m_curr->line_contents.rem;
768 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with('*'), m_evt_handler->m_curr->pos);
769 _set_first(s, s.first_of(" ,]\t"));
770 _line_progressed(s.len);
771 return s;
772}
773
774template<class EventHandler>
775csubstr ParseEngine<EventHandler>::_scan_ref_map()
776{
777 csubstr s = m_evt_handler->m_curr->line_contents.rem;
778 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with('*'), m_evt_handler->m_curr->pos);
779 _set_first(s, s.first_of(" ,}\t"));
780 _line_progressed(s.len);
781 return s;
782}
783
784template<class EventHandler>
785csubstr ParseEngine<EventHandler>::_scan_tag()
786{
787 csubstr t = m_evt_handler->m_curr->line_contents.rem;
788 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, t.begins_with('!'), m_evt_handler->m_curr->pos);
789 if(!t.begins_with("!<"))
790 {
791 _c4dbgp("begins with '!'");
792 _set_first(t, t.first_of(" ,]}\t"));
793 if C4_UNLIKELY(t.first_of("[{") != npos)
794 _c4err("invalid tag");
795 _line_progressed(t.len);
796 if(m_options.resolve_tags_all() || (m_options.resolve_tags() && is_custom_tag(t)))
797 t = _resolve_tag(t);
798 }
799 else
800 {
801 _c4dbgp("begins with '!<'");
802 size_t pos = t.find('>');
803 if C4_UNLIKELY(pos == npos)
804 _c4err("invalid tag");
805 _set_first_strict(t, pos+1);
806 _line_progressed(t.len);
807 t = t.sub(1);
808 }
809 _maybe_skip_whitespace_tokens();
810 return t;
811}
812
813template<class EventHandler>
814csubstr ParseEngine<EventHandler>::_scan_tag(csubstr *orig)
815{
816 csubstr t = m_evt_handler->m_curr->line_contents.rem;
817 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, t.begins_with('!'), m_evt_handler->m_curr->pos);
818 if(!t.begins_with("!<"))
819 {
820 _c4dbgp("begins with '!'");
821 _set_first(t, t.first_of(" ,\t"));
822 if C4_UNLIKELY(t.first_of("[{") != npos)
823 _c4err("invalid tag");
824 _line_progressed(t.len);
825 *orig = t;
826 if(m_options.resolve_tags_all() || (m_options.resolve_tags() && is_custom_tag(t)))
827 t = _resolve_tag(t);
828 }
829 else
830 {
831 _c4dbgp("begins with '!<'");
832 size_t pos = t.find('>');
833 if C4_UNLIKELY(pos == npos)
834 _c4err("invalid tag");
835 _set_first_strict(t, pos+1);
836 _line_progressed(t.len);
837 *orig = t;
838 t = t.sub(1);
839 }
840 _maybe_skip_whitespace_tokens();
841 return t;
842}
843
844
845//-----------------------------------------------------------------------------
846
847template<class EventHandler>
848bool ParseEngine<EventHandler>::_is_valid_start_scalar_plain_flow_check_block_token(csubstr s)
849{
850 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.len > 0, m_evt_handler->m_curr->pos);
851 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with_any(":-"), m_evt_handler->m_curr->pos);
852 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.count('\n') == 0, m_evt_handler->m_curr->pos);
853 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.count('\r') == 0, m_evt_handler->m_curr->pos);
854 if(s.len > 1)
855 {
856 switch(s.str[1])
857 {
858 case ' ':
859 case ',':
860 case '}':
861 case ']':
862 case '\t':
863 if(s.str[0] == ':')
864 {
865 _c4dbgpf("not a scalar: found non-scalar token '{}{}'", s.str[0], s.str[1]);
866 return false;
867 }
868 else
869 {
870 _c4err("invalid scalar");
871 }
872 break;
873 case '{':
874 case '[':
875 _c4err("invalid token \":{}\"", _c4prc(s.str[1]));
876 break;
877 default:
878 break;
879 }
880 }
881 else
882 {
883 if(s.str[0] == '-')
884 _c4err("invalid scalar");
885 return false;
886 }
887 return true;
888}
889
890template<class EventHandler>
891bool ParseEngine<EventHandler>::_is_valid_start_scalar_plain_flow_check_qmrk(csubstr s)
892{
893 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.len > 0, m_evt_handler->m_curr->pos);
894 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s[0] == '?', m_evt_handler->m_curr->pos);
895 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.count('\n') == 0, m_evt_handler->m_curr->pos);
896 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.count('\r') == 0, m_evt_handler->m_curr->pos);
897 if(s.len > 1)
898 {
899 switch(s.str[1])
900 {
901 case ' ':
902 case '\t':
903 _c4dbgpf("not a scalar: found non-scalar token '?{}'", _c4prc(s.str[1]));
904 return false;
905 case '{':
906 case '}':
907 case '[':
908 case ']':
909 _c4err("invalid token \"?{}\"", _c4prc(s.str[1]));
910 break;
911 default:
912 break;
913 }
914 }
915 else
916 {
917 return false;
918 }
919 return true;
920}
921
922
923template<class EventHandler>
924bool ParseEngine<EventHandler>::_is_valid_start_scalar_plain_flow(csubstr s)
925{
926 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !s.empty(), m_evt_handler->m_curr->pos);
927 // it's not a scalar if it starts with any of these characters:
928 switch(s.str[0])
929 {
930 // these are all legal tokens which mean no scalar is starting:
931 case '[':
932 case ']':
933 case '{':
934 case '}':
935 case '&':
936 case '*':
937 case '!':
938 case '|':
939 case '>':
940 case '#':
941 case ',':
942 _c4dbgpf("not a scalar: found non-scalar token '{}'", _c4prc(s.str[0]));
943 return false;
944 // '-' and ':' are illegal at the beginning if not followed by a scalar character
945 case '-':
946 case ':':
947 _c4dbgpf("suspicious token='{}' len={}", _c4prc(s.str[0]), s.len);
948 return _is_valid_start_scalar_plain_flow_check_block_token(s);
949 case '?':
950 _c4dbgpf("qmrk='{}' len={}", _c4prc(s.str[0]), s.len);
951 return _is_valid_start_scalar_plain_flow_check_qmrk(s);
952 // everything else is a legal starting character
953 default:
954 return true;
955 }
956}
957
958
959template<class EventHandler>
960bool ParseEngine<EventHandler>::_scan_scalar_plain_handle_newline(csubstr s, size_t offs)
961{
962 _c4dbgpf("newl[PLAIN]: found '\\n'. offs={} line={} sofar={}", offs, m_evt_handler->m_curr->pos.line, prs_(s.first(offs), true));
963 if(s.len > offs + 1)
964 {
965 _c4dbgp("newl[PLAIN]: buffer continues");
966 csubstr next_line = s.sub(offs + 1);
967 size_t next_line_indentation = next_line.first_not_of(' ');
968 if(next_line_indentation != npos)
969 {
970 _c4dbgpf("newl[PLAIN]: line={} indentation={} indref={}", m_evt_handler->m_curr->pos.line + 1, next_line_indentation, m_evt_handler->m_curr->indref);
971 next_line = next_line.first(next_line.first_of("\n\r"));
972 _c4dbgpf("newl[PLAIN]: has indentation. next_line={}", prs_(next_line));
973 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, next_line_indentation <= next_line.len, m_evt_handler->m_curr->pos);
974 if C4_LIKELY(next_line_indentation >= m_evt_handler->m_curr->indref)
975 {
976 _c4dbgp("newl[PLAIN]: larger indentation");
977 next_line = next_line.sub(next_line_indentation);
978 }
979 else if C4_UNLIKELY(next_line.len && next_line.triml(' ').len)
980 {
981 _c4dbgp("newl[PLAIN]: err, smaller indentation");
982 _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
983 _line_ended();
984 _scan_line();
985 if(m_evt_handler->m_curr->line_contents.indentation != npos)
986 _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
987 _c4err("parse error"); // cannot reduce indentation here
988 }
989 _c4dbgpf("newl[PLAIN]: next_line.len={}", next_line.len);
990 if(next_line.len)
991 {
992 size_t fno = next_line.first_not_of(" \t");
993 if(fno != csubstr::npos)
994 {
995 _c4assert(fno < next_line.len);
996 switch(next_line.str[fno])
997 {
998 case ',': case ']': case '#':
999 _c4dbgpf("newl[PLAIN]: found terminating character beginning next line: '{}'", next_line.str[fno]);
1000 return false;
1001 case ':': // cannot be succeeded by whitespace
1002 _c4dbgp("newl[PLAIN]: found :");
1003 if(fno + 1 == next_line.len || _is_blck_token(next_line.sub(fno)))
1004 {
1005 _c4dbgpf("newl[PLAIN]: found terminating character beginning next line: '{}'", next_line.str[fno]);
1006 return false;
1007 }
1008 break;
1009 }
1010 }
1011 }
1012 }
1013 }
1014 _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
1015 _line_ended();
1016 _scan_line();
1017 return true;
1018}
1019
1020template<class EventHandler>
1021bool ParseEngine<EventHandler>::_scan_scalar_plain_seq_flow(ScannedScalar *C4_RESTRICT sc)
1022{
1023 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RMAP), m_evt_handler->m_curr->pos);
1024 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RBLCK), m_evt_handler->m_curr->pos);
1025 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RSEQ|RSEQIMAP), m_evt_handler->m_curr->pos);
1026 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RFLOW), m_evt_handler->m_curr->pos);
1027 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL), m_evt_handler->m_curr->pos);
1028
1029 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !m_evt_handler->m_curr->line_contents.rem.begins_with(' '), m_evt_handler->m_curr->pos);
1030 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !m_evt_handler->m_curr->line_contents.rem.begins_with('\n'), m_evt_handler->m_curr->pos);
1031
1032 if(!m_evt_handler->m_curr->line_contents.rem.len || !_is_valid_start_scalar_plain_flow(m_evt_handler->m_curr->line_contents.rem))
1033 return false;
1034
1035 substr s = _buf().sub(m_evt_handler->m_curr->pos.offset);
1036 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with(m_evt_handler->m_curr->line_contents.rem), m_evt_handler->m_curr->pos);
1037
1038 _c4dbgp("scanning seqflow scalar...");
1039
1040 bool needs_filter = false;
1041 size_t col = 0; // zero-based column
1042 size_t offs = 0; // offset
1043 for( ; offs < s.len; ++offs, ++col)
1044 {
1045 const char c = s.str[offs];
1046 switch(c)
1047 {
1048 case ',':
1049 case ']':
1050 _c4dbgpf("found terminating character at {}: '{}'", offs, c);
1051 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, offs > 0, m_evt_handler->m_curr->pos);
1052 goto ended_scalar;
1053 case '\n':
1054 _c4dbgpf("found '\\n' at col={}", col);
1055 if(!_scan_scalar_plain_handle_newline(s, offs))
1056 goto ended_scalar;
1057 col = (size_t)-1; // so that col is 0 in the next loop iteration
1058 needs_filter = true;
1059 break;
1060 case '\r':
1061 --col; // don't count \r when calling _line_progressed()
1062 needs_filter = true;
1063 break;
1064 case ':':
1065 _c4dbgp("found suspicious ':'");
1066 if(s.len > offs + 1)
1067 {
1068 char next = s.str[offs + 1];
1069 _c4dbgpf("next char is '{}'", _c4prc(next));
1070 if(next == '\r')
1071 {
1072 csubstr after = s.sub(offs + 1).triml('\r');
1073 if(after.len)
1074 {
1075 next = after.str[0];
1076 _c4dbgpf("skip \\r to '{}'", _c4prc(next));
1077 }
1078 }
1079 // no else here.
1080 if(next == ' ' RYML_WITH_TAB_TOKENS_(|| next == '\t') || next == ',' || next == '\n' || next == ']')
1081 {
1082 _c4dbgp("map starting!");
1083 goto ended_scalar;
1084 }
1085 else
1086 {
1087 _c4dbgp("':' nothing to see here");
1088 }
1089 }
1090 else
1091 {
1092 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.len == offs + 1, m_evt_handler->m_curr->pos);
1093 _line_progressed(col);
1094 _c4err("missing termination: '{}'", c); // noreturn
1095 }
1096 break;
1097 case '#':
1098 {
1099 _c4dbgp("found suspicious '#'");
1100 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, offs > 0, m_evt_handler->m_curr->pos);
1101 char prev = s.str[offs - 1];
1102 if(prev == ' ' RYML_WITH_TAB_TOKENS_(|| prev == '\t'))
1103 {
1104 _c4dbgpf("found terminating character at {}: '{}'", offs, c);
1105 goto ended_scalar;
1106 }
1107 }
1108 break;
1109 case '[':
1110 case '{':
1111 case '}':
1112 _line_progressed(col); // advance to report the proper position in the error
1113 _c4err("invalid character: '{}'", c); // noreturn
1114 case '-':
1115 case '.':
1116 _c4dbgpf("doc token character: '{}', offs={}", c, offs);
1117 if(offs == 0 && m_evt_handler->m_curr->at_line_beginning())
1118 {
1119 _c4dbgp("at line beginning");
1120 if(s.len >= 3 && s.str[1] == c && s.str[2] == c)
1121 {
1122 _c4err("parse error"); // no return
1123 }
1124 }
1125 break;
1126 default:
1127 ;
1128 }
1129 }
1130
1131ended_scalar:
1132
1133 _line_progressed(col);
1134 _set_first(s, offs);
1135 sc->scalar = s.trimr(RYML_WITH_OR_WITHOUT_TAB_TOKENS_(" \t", ' '));
1136 sc->needs_filter = needs_filter;
1137
1138 _c4prscalar("scanned plain scalar", sc->scalar, /*keep_newlines*/true);
1139
1140 return true;
1141}
1142
1143template<class EventHandler>
1144bool ParseEngine<EventHandler>::_scan_scalar_plain_map_flow(ScannedScalar *C4_RESTRICT sc)
1145{
1146 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RSEQ) || has_any(RSEQIMAP), m_evt_handler->m_curr->pos);
1147 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RBLCK), m_evt_handler->m_curr->pos);
1148 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RMAP|RSEQIMAP), m_evt_handler->m_curr->pos);
1149 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RFLOW), m_evt_handler->m_curr->pos);
1150 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY|RVAL|QMRK), m_evt_handler->m_curr->pos);
1151
1152 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !m_evt_handler->m_curr->line_contents.rem.begins_with(' '), m_evt_handler->m_curr->pos);
1153 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !m_evt_handler->m_curr->line_contents.rem.begins_with('\n'), m_evt_handler->m_curr->pos);
1154
1155 if(!m_evt_handler->m_curr->line_contents.rem.len || !_is_valid_start_scalar_plain_flow(m_evt_handler->m_curr->line_contents.rem))
1156 return false;
1157
1158 substr s = _buf().sub(m_evt_handler->m_curr->pos.offset);
1159 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with(m_evt_handler->m_curr->line_contents.rem), m_evt_handler->m_curr->pos);
1160
1161 _c4dbgp("scanning mapflow scalar...");
1162
1163 bool needs_filter = false;
1164 size_t col = 0; // zero-based column
1165 size_t offs = 0; // offset
1166 for( ; offs < s.len; ++offs, ++col)
1167 {
1168 const char c = s.str[offs];
1169 switch(c)
1170 {
1171 case ',':
1172 case '}':
1173 _c4dbgpf("found terminating character at {}: '{}'", offs, c);
1174 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, offs > 0, m_evt_handler->m_curr->pos);
1175 goto ended_scalar;
1176 case '\n':
1177 _c4dbgpf("found '\\n' at col={}", col);
1178 if(!_scan_scalar_plain_handle_newline(s, offs))
1179 goto ended_scalar;
1180 col = (size_t)-1; // so that col is 0 in the next loop iteration
1181 needs_filter = true;
1182 break;
1183 case '\r':
1184 --col; // don't count \r when calling _line_progressed()
1185 needs_filter = true;
1186 break;
1187 case ':':
1188 _c4dbgpf("found ':'", c);
1189 if(s.len == offs+1)
1190 break;
1191 {
1192 const char next = s.str[offs+1];
1193 _c4dbgpf("next='{}'", c);
1194 if(next == ' ' || next == ',' || next == '}' || next == '\n' || next == '\r' RYML_WITH_TAB_TOKENS_(|| next == '\t'))
1195 {
1196 _c4dbgpf("found terminating character: '{}'", c);
1197 goto ended_scalar;
1198 }
1199 }
1200 break;
1201 case '{':
1202 case '[':
1203 _line_progressed(col);
1204 _c4err("invalid character: '{}'", c); // noreturn
1205 break;
1206 case ']':
1207 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RSEQIMAP), m_evt_handler->m_curr->pos);
1208 goto ended_scalar;
1209 default:
1210 ;
1211 }
1212 }
1213
1214ended_scalar:
1215
1216 _line_progressed(col);
1217 s = s.first(offs);
1218 sc->scalar = s.trimr(RYML_WITH_OR_WITHOUT_TAB_TOKENS_(" \t", ' '));
1219 sc->needs_filter = needs_filter;
1220
1221 _c4prscalar("scanned plain scalar", sc->scalar, /*keep_newlines*/true);
1222
1223 return sc->scalar.len > 0u;
1224}
1225
1226template<class EventHandler>
1227bool ParseEngine<EventHandler>::_scan_scalar_seq_json(ScannedScalar *C4_RESTRICT sc)
1228{
1229 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RMAP), m_evt_handler->m_curr->pos);
1230 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RBLCK), m_evt_handler->m_curr->pos);
1231 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RSEQ), m_evt_handler->m_curr->pos);
1232 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RFLOW), m_evt_handler->m_curr->pos);
1233
1234 substr s = m_evt_handler->m_curr->line_contents.rem;
1235 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !s.begins_with(' '), m_evt_handler->m_curr->pos);
1236 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.len > 0, m_evt_handler->m_curr->pos);
1237
1238 _c4dbgp("seq_json: scanning scalar...");
1239
1240 switch(s.str[0])
1241 {
1242 case ']':
1243 case '{':
1244 case ',':
1245 _c4dbgp("seq_json: not a scalar.");
1246 return false;
1247 }
1248
1249 {
1250 const size_t len = _begins_with_special_json_scalar(s);
1251 if(len)
1252 {
1253 char c = s.len > len ? s.str[len] : ',';
1254 if(c == ',' || c == ']' || c == ' ' || c == '\n' || c == '\t' || c == '\r')
1255 {
1256 sc->scalar = s.first(len);
1257 sc->needs_filter = false;
1258 _c4dbgpf("seq_json: special scalar: '{}'", sc->scalar);
1259 _line_progressed(len);
1260 return true;
1261 }
1262 else
1263 {
1264 return false;
1265 }
1266 }
1267 }
1268
1269 // must be a number or special scalar
1270 size_t i = 0;
1271 for( ; i < s.len; ++i)
1272 {
1273 const char c = s.str[i];
1274 switch(c)
1275 {
1276 case ',':
1277 case ']':
1278 case ' ':
1279 case '\t':
1280 _c4dbgpf("seq_json: found terminating character: '{}'", c);
1281 goto ended_scalar;
1282 default:
1283 ;
1284 }
1285 }
1286
1287ended_scalar:
1288
1289 _line_progressed(i);
1290 sc->scalar = s.first(i);
1291 sc->needs_filter = false;
1292 _c4dbgpf("seq_json: scalar was {}", prs_(sc->scalar, /*escape*/true));
1293
1294 return true;
1295}
1296
1297template<class EventHandler>
1298bool ParseEngine<EventHandler>::_scan_scalar_map_json(ScannedScalar *C4_RESTRICT sc)
1299{
1300 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RSEQ), m_evt_handler->m_curr->pos);
1301 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RBLCK), m_evt_handler->m_curr->pos);
1302 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RMAP), m_evt_handler->m_curr->pos);
1303 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RFLOW), m_evt_handler->m_curr->pos);
1304 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY|RVAL), m_evt_handler->m_curr->pos);
1305
1306 substr s = m_evt_handler->m_curr->line_contents.rem;
1307 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !s.begins_with(' '), m_evt_handler->m_curr->pos);
1308 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.len > 0, m_evt_handler->m_curr->pos);
1309
1310 _c4dbgp("scanning scalar...");
1311
1312 {
1313 const size_t len = _begins_with_special_json_scalar(s);
1314 if(len)
1315 {
1316 char c = s.len > len ? s.str[len] : ',';
1317 _c4dbgpf("begins with special scalar: {} next='{}'", s.first(len), _c4prc(c));
1318 if(c == ',' || c == '}' || c == ' ' || c == '\n' || c == '\t' || c == '\r')
1319 {
1320 sc->scalar = s.first(len);
1321 sc->needs_filter = false;
1322 _c4dbgpf("special json scalar: '{}'", prs_(sc->scalar));
1323 _line_progressed(len);
1324 return true;
1325 }
1326 else
1327 {
1328 return false;
1329 }
1330 }
1331 }
1332
1333 // must be a number
1334 size_t i = 0;
1335 for( ; i < s.len; ++i)
1336 {
1337 const char c = s.str[i];
1338 switch(c)
1339 {
1340 case ',':
1341 case '}':
1342 case ' ':
1343 case '\t':
1344 _c4dbgpf("found terminating character: '{}'", c);
1345 goto ended_scalar;
1346 default:
1347 ;
1348 }
1349 }
1350
1351ended_scalar:
1352
1353 if C4_LIKELY(i > 0)
1354 {
1355 _line_progressed(i);
1356 sc->scalar = s.first(i);
1357 sc->needs_filter = false;
1358 _c4dbgpf("scalar was {}", prs_(sc->scalar));
1359 return true;
1360 }
1361
1362 return false;
1363}
1364
1365template<class EventHandler>
1366bool ParseEngine<EventHandler>::_is_doc_begin(csubstr s)
1367{
1368 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s[0] == '-', m_evt_handler->m_curr->pos);
1369 return (m_evt_handler->m_curr->line_contents.indentation == 0u && m_evt_handler->m_curr->at_line_beginning() && _is_doc_begin_token(s));
1370}
1371
1372template<class EventHandler>
1373bool ParseEngine<EventHandler>::_is_doc_end(csubstr s)
1374{
1375 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s[0] == '.', m_evt_handler->m_curr->pos);
1376 return (m_evt_handler->m_curr->line_contents.indentation == 0u && m_evt_handler->m_curr->at_line_beginning() && _is_doc_end_token(s));
1377}
1378
1379template<class EventHandler>
1380bool ParseEngine<EventHandler>::_scan_scalar_plain_blck(ScannedScalar *C4_RESTRICT sc, size_t indentation)
1381{
1382 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RFLOW), m_evt_handler->m_curr->pos);
1383 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RSEQIMAP), m_evt_handler->m_curr->pos);
1384 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RBLCK|RUNK|USTY), m_evt_handler->m_curr->pos);
1385
1386 substr s = m_evt_handler->m_curr->line_contents.rem;
1387 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !s.begins_with(' '), m_evt_handler->m_curr->pos);
1388 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.len > 0, m_evt_handler->m_curr->pos);
1389
1390 switch(s.str[0])
1391 {
1392 case '-':
1393 if(_is_blck_token(s))
1394 {
1395 return false;
1396 }
1397 else if(_is_doc_begin(s))
1398 {
1399 _c4dbgp("token is doc start");
1400 return false;
1401 }
1402 break;
1403 case ':':
1404 case '?':
1405 if(_is_blck_token(s))
1406 return false;
1407 break;
1408 case '[':
1409 case '{':
1410 case '&':
1411 case '*':
1412 case '!':
1413 case '\t':
1414 case ',':
1415 case '%':
1416 return false;
1417 case '.':
1418 if(_is_doc_end(s))
1419 {
1420 _c4dbgp("token is doc end");
1421 return false;
1422 }
1423 break;
1424 }
1425
1426 _c4dbgpf("plain scalar! indentation={}", indentation);
1427
1428 const size_t start_offset = m_evt_handler->m_curr->pos.offset;
1429 const size_t start_line = m_evt_handler->m_curr->pos.line;
1430
1431 bool needs_filter = false;
1432 while(true)
1433 {
1434 _c4dbgpf("plain scalar line: {}", prs_(s));
1435 for(size_t i = 0; i < s.len; ++i)
1436 {
1437 const char curr = s.str[i];
1438 //_c4dbgpf("[{}]='{}'", i, _c4prc(curr));
1439 switch(curr)
1440 {
1441 case ':':
1442 _c4dbgpf("[{}]: got suspicious ':'", i);
1443 // are there more characters?
1444 if((i + 1 == s.len) || ((s.str[i+1] == ' ') RYML_WITH_TAB_TOKENS_( || (s.str[i+1] == '\t'))))
1445 {
1446 _c4dbgpf("followed by '{}'", i+1 == s.len ? csubstr("\\n") : _c4prc(s.str[i+1]));
1447 _line_progressed(i);
1448 // ': ' is accepted only on the first line
1449 if C4_LIKELY(m_evt_handler->m_curr->pos.line == start_line)
1450 {
1451 _c4dbgp("start line. scalar ends here");
1452 goto ended_scalar;
1453 }
1454 else
1455 {
1456 _c4err("multiline scalars cannot be used as keys");
1457 }
1458 }
1459 else
1460 {
1461 size_t j = i;
1462 while(j + 1 < s.len && s.str[j+1] == ':')
1463 {
1464 _c4dbgp("skip colon");
1465 ++j;
1466 }
1467 i = j > i ? j-1 : i;
1468 _c4dbgp("nothing to see here");
1469 }
1470 break;
1471 case '#':
1472 _c4dbgp("got suspicious '#'");
1473 if(!i || (s.str[i-1] == ' ' || s.str[i-1] == '\t'))
1474 {
1475 _c4dbgp("comment! scalar ends here");
1476 _line_progressed(i);
1477 goto ended_scalar;
1478 }
1479 else
1480 {
1481 _c4dbgp("nothing to see here");
1482 }
1483 break;
1484 }
1485 }
1486 _line_progressed(s.len);
1487 csubstr next_peeked = _peek_next_line(m_evt_handler->m_curr->pos.offset);
1488 next_peeked = next_peeked.trimr("\n\r");
1489 const size_t next_indentation = next_peeked.first_not_of(' ');
1490 _c4dbgpf("indentation curr={} next={}", indentation, next_indentation);
1491 if(next_indentation < indentation)
1492 {
1493 _c4dbgp("smaller indentation! scalar ended");
1494 goto ended_scalar;
1495 }
1496 else if(next_indentation == 0 && next_peeked.len > 0)
1497 {
1498 const char first = next_peeked.str[0];
1499 switch(first)
1500 {
1501 case '-':
1502 _c4dbgpf("doc begin? peeked={}", prs_(next_peeked, size_t(3)));
1503 if(_is_doc_begin_token(next_peeked))
1504 {
1505 _c4dbgp("doc begin! scalar ended");
1506 goto ended_scalar;
1507 }
1508 break;
1509 case '.':
1510 _c4dbgpf("doc end? peeked={}", prs_(next_peeked, size_t(3)));
1511 if(_is_doc_end_token(next_peeked))
1512 {
1513 _c4dbgp("doc end! scalar ended");
1514 goto ended_scalar;
1515 }
1516 break;
1517 }
1518 }
1519 // load with next line
1520 _c4dbgp("next line!");
1521 if(!_finished_file())
1522 {
1523 _c4dbgp("next line!");
1524 _line_ended();
1525 _scan_line();
1526 }
1527 else
1528 {
1529 _c4dbgp("file finished!");
1530 goto ended_scalar;
1531 }
1532 s = m_evt_handler->m_curr->line_contents.rem;
1533 needs_filter = true;
1534 }
1535
1536ended_scalar:
1537
1538 sc->scalar = _buf().range(start_offset, m_evt_handler->m_curr->pos.offset).trimr(" \n\r\t");
1539 sc->needs_filter = needs_filter;
1540
1541 _c4dbgpf("scalar was {}", prs_(sc->scalar));
1542
1543 return true;
1544}
1545
1546template<class EventHandler>
1547C4_ALWAYS_INLINE bool ParseEngine<EventHandler>::_scan_scalar_plain_seq_blck(ScannedScalar *C4_RESTRICT sc) // LCOV_EXCL_LINE
1548{
1549 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RMAP), m_evt_handler->m_curr->pos);
1550 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RFLOW), m_evt_handler->m_curr->pos);
1551 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RSEQIMAP), m_evt_handler->m_curr->pos);
1552 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RSEQ), m_evt_handler->m_curr->pos);
1553 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RBLCK), m_evt_handler->m_curr->pos);
1554 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL), m_evt_handler->m_curr->pos);
1555 return _scan_scalar_plain_blck(sc, m_evt_handler->m_curr->indref + 1u);
1556}
1557
1558template<class EventHandler>
1559C4_ALWAYS_INLINE bool ParseEngine<EventHandler>::_scan_scalar_plain_map_blck(ScannedScalar *C4_RESTRICT sc) // LCOV_EXCL_LINE
1560{
1561 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RSEQ), m_evt_handler->m_curr->pos);
1562 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RFLOW), m_evt_handler->m_curr->pos);
1563 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RMAP), m_evt_handler->m_curr->pos);
1564 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RBLCK), m_evt_handler->m_curr->pos);
1565 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY|RVAL|QMRK), m_evt_handler->m_curr->pos);
1566 return _scan_scalar_plain_blck(sc, m_evt_handler->m_curr->indref + 1u);
1567}
1568
1569template<class EventHandler>
1570C4_ALWAYS_INLINE bool ParseEngine<EventHandler>::_scan_scalar_plain_unk(ScannedScalar *C4_RESTRICT sc) // LCOV_EXCL_LINE
1571{
1572 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RUNK|USTY), m_evt_handler->m_curr->pos);
1573 return _scan_scalar_plain_blck(sc, m_evt_handler->m_curr->indref);
1574}
1575
1576
1577//-----------------------------------------------------------------------------
1578
1579template<class EventHandler>
1580substr ParseEngine<EventHandler>::_peek_next_line(size_t pos) const
1581{
1582 substr rem{}; // declare here because of the goto
1583 size_t nlpos{}; // declare here because of the goto
1584 pos = pos == npos ? m_evt_handler->m_curr->pos.offset : pos;
1585 if(pos >= _buf().len)
1586 goto next_is_empty;
1587
1588 // look for the next newline chars, and jump to the right of those
1589 rem = _from_next_line(_buf().sub(pos));
1590 if(rem.empty())
1591 goto next_is_empty;
1592
1593 // now get everything up to and including the following newline chars
1594 nlpos = rem.first_of("\r\n");
1595 if((nlpos != csubstr::npos) && (nlpos + 1 < rem.len))
1596 nlpos += _extend_from_combined_newline(rem[nlpos], rem[nlpos+1]);
1597 rem = rem.left_of(nlpos, /*include_pos*/true);
1598
1599 _c4dbgpf("peek next line @ {}: (len={})'{}'", pos, rem.len, rem.trimr("\r\n"));
1600 return rem;
1601
1602next_is_empty:
1603 _c4dbgpf("peek next line @ {}: (len=0)''", pos);
1604 return rem;
1605}
1606
1607//-----------------------------------------------------------------------------
1608
1609template<class EventHandler>
1610void ParseEngine<EventHandler>::_scan_line()
1611{
1612 if C4_LIKELY(m_evt_handler->m_curr->pos.offset < _buf().len)
1613 m_evt_handler->m_curr->line_contents.reset_with_next_line(_buf(), m_evt_handler->m_curr->pos.offset);
1614 else
1615 m_evt_handler->m_curr->line_contents.reset_with_next_line(_buf().last(0), 0);
1616}
1617
1618template<class EventHandler>
1619void ParseEngine<EventHandler>::_line_progressed(size_t ahead)
1620{
1621 _c4dbgpf("line[{}] ({} cols) progressed by {}: col {}-->{} offset {}-->{}",
1622 m_evt_handler->m_curr->pos.line,
1623 m_evt_handler->m_curr->line_contents.full.len,
1624 ahead, m_evt_handler->m_curr->pos.col,
1625 m_evt_handler->m_curr->pos.col+ahead,
1626 m_evt_handler->m_curr->pos.offset,
1627 m_evt_handler->m_curr->pos.offset+ahead);
1628 m_evt_handler->m_curr->pos.offset += ahead;
1629 m_evt_handler->m_curr->pos.col += ahead;
1630 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.col <= m_evt_handler->m_curr->line_contents.num_cols+1, m_evt_handler->m_curr->pos);
1631 m_evt_handler->m_curr->line_contents.rem = m_evt_handler->m_curr->line_contents.rem.sub(ahead);
1632}
1633
1634template<class EventHandler>
1635void ParseEngine<EventHandler>::_line_ended()
1636{
1637 _c4dbgpf("line[{}] ({} cols) ended! offset {}-->{} / col {}-->{}",
1638 m_evt_handler->m_curr->pos.line,
1639 m_evt_handler->m_curr->line_contents.full.len,
1640 m_evt_handler->m_curr->pos.offset, m_evt_handler->m_curr->pos.offset + m_evt_handler->m_curr->line_contents.full.len - m_evt_handler->m_curr->line_contents.num_cols,
1641 m_evt_handler->m_curr->pos.col, 1);
1642 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.col == m_evt_handler->m_curr->line_contents.num_cols + 1, m_evt_handler->m_curr->pos);
1643 m_evt_handler->m_curr->pos.offset += m_evt_handler->m_curr->line_contents.full.len - m_evt_handler->m_curr->line_contents.num_cols;
1644 ++m_evt_handler->m_curr->pos.line;
1645 m_evt_handler->m_curr->pos.col = 1;
1646}
1647
1648template<class EventHandler>
1649void ParseEngine<EventHandler>::_line_ended_undo()
1650{
1651 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.col == 1u, m_evt_handler->m_curr->pos);
1652 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.line > 0u, m_evt_handler->m_curr->pos);
1653 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.offset >= m_evt_handler->m_curr->line_contents.full.len - m_evt_handler->m_curr->line_contents.num_cols, m_evt_handler->m_curr->pos);
1654 const size_t delta = m_evt_handler->m_curr->line_contents.full.len - m_evt_handler->m_curr->line_contents.num_cols;
1655 _c4dbgpf("line[{}] undo ended! line {}-->{}, offset {}-->{}", m_evt_handler->m_curr->pos.line, m_evt_handler->m_curr->pos.line, m_evt_handler->m_curr->pos.line - 1, m_evt_handler->m_curr->pos.offset, m_evt_handler->m_curr->pos.offset - delta);
1656 m_evt_handler->m_curr->pos.offset -= delta;
1657 --m_evt_handler->m_curr->pos.line;
1658 m_evt_handler->m_curr->pos.col = m_evt_handler->m_curr->line_contents.num_cols + 1u;
1659 // don't forget to undo also the changes to the remainder of the line
1660 //RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.offset >= _buf().len || _buf()[m_evt_handler->m_curr->pos.offset] == '\n' || _buf()[m_evt_handler->m_curr->pos.offset] == '\r', m_evt_handler->m_curr->pos);
1661 m_evt_handler->m_curr->line_contents.rem = _buf().sub(m_evt_handler->m_curr->pos.offset, 0);
1662}
1663
1664
1665//-----------------------------------------------------------------------------
1666template<class EventHandler>
1667void ParseEngine<EventHandler>::_set_indentation(size_t indentation) noexcept
1668{
1669 m_evt_handler->m_curr->indref = indentation;
1670 _c4dbgpf("state[{}]: saving indentation: {}", m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
1671}
1672
1673template<class EventHandler>
1674void ParseEngine<EventHandler>::_save_indentation()
1675{
1676 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->line_contents.rem.is_sub(m_evt_handler->m_curr->line_contents.full), m_evt_handler->m_curr->pos);
1677 m_evt_handler->m_curr->indref = m_evt_handler->m_curr->line_contents.current_col();
1678 _c4dbgpf("state[{}]: saving indentation: {}", m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
1679}
1680
1681template<class EventHandler>
1682void ParseEngine<EventHandler>::_mark_seqflow_val_end() noexcept
1683{
1684 _c4dbgpf("SEQFLOW. mark val end at line={}", m_evt_handler->m_curr->pos.line);
1685 m_prev_val_end = m_evt_handler->m_curr->pos.line;
1686}
1687
1688
1689//-----------------------------------------------------------------------------
1690
1691template<class EventHandler>
1692void ParseEngine<EventHandler>::_flow_container_was_a_key(size_t orig_indent)
1693{
1694 _c4dbgpf("flow container is followed by colon! orig_indent={}", orig_indent);
1695 m_evt_handler->actually_val_is_first_key_of_new_map_block();
1696 addrem_flags(RMAP|RVAL|RBLCK, RKCL|RSEQ|RUNK);
1697 _set_indentation(orig_indent);
1698 _maybe_skip_whitespace_tokens();
1699}
1700
1701template<class EventHandler>
1702void ParseEngine<EventHandler>::_end_flow_container(size_t orig_indent, bool multiline)
1703{
1704 // this is called AFTER ending the flow container,
1705 // so now we're at the parent container's scope
1706 if(has_all(RMAP|RBLCK) && has_none(RKCL|RVAL|RNXT))
1707 {
1708 _c4dbgp("flow container: end as vanilla block map key!");
1709 if C4_UNLIKELY(multiline)
1710 _c4err("multiline key is invalid");
1711 if C4_UNLIKELY(!_maybe_scan_following_colon())
1712 _c4err("could not find ':' colon after key");
1713 _maybe_skip_whitespace_tokens();
1714 addrem_flags(RVAL, RKEY|RKCL|RNXT);
1715 }
1716 else if(has_none(RFLOW))
1717 {
1718 _c4dbgp("end_flow_container: now not in flow!");
1719 if(has_any(RUNK|RSEQ|RKCL) && _maybe_scan_following_colon())
1720 {
1721 if C4_UNLIKELY(multiline)
1722 _c4err("multiline key is invalid");
1723 _flow_container_was_a_key(orig_indent);
1724 }
1725 else
1726 {
1727 _c4dbgp("end_flow_container: end map as key!");
1728 }
1729 }
1730 else if(has_any(RSEQ))
1731 {
1732 _c4dbgp("end_flow_container: now in a flow seq");
1733 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RFLOW), m_evt_handler->m_curr->pos);
1734 _mark_seqflow_val_end();
1735 }
1736}
1737
1738template<class EventHandler>
1739void ParseEngine<EventHandler>::_end_map_flow()
1740{
1741 bool multiline = m_evt_handler->m_parent->pos.line < m_evt_handler->m_curr->pos.line;
1742 size_t orig_indent = m_evt_handler->m_curr->indref;
1743 _c4dbgpf("mapflow: end, multiline={}", multiline);
1744 m_evt_handler->end_map_flow(multiline && m_options.detect_flow_ml(), m_options.flow_ml_style().m_bits);
1745 _end_flow_container(orig_indent, multiline);
1746}
1747
1748template<class EventHandler>
1749void ParseEngine<EventHandler>::_end_seq_flow()
1750{
1751 bool multiline = m_evt_handler->m_parent->pos.line < m_evt_handler->m_curr->pos.line;
1752 size_t orig_indent = m_evt_handler->m_curr->indref;
1753 _c4dbgpf("seqflow: end, multiline={}", multiline);
1754 m_evt_handler->end_seq_flow(multiline && m_options.detect_flow_ml(), m_options.flow_ml_style().m_bits);
1755 _end_flow_container(orig_indent, multiline);
1756}
1757
1758template<class EventHandler>
1759void ParseEngine<EventHandler>::_end_map_blck()
1760{
1761 _c4dbgp("mapblck: end");
1762 if(has_any(RKCL|RVAL))
1763 {
1764 _c4dbgp("mapblck: set missing val");
1765 _handle_annotations_before_blck_val_scalar();
1766 m_evt_handler->set_val_scalar_plain_empty();
1767 }
1768 else if(has_any(QMRK))
1769 {
1770 _c4dbgp("mapblck: set missing keyval");
1771 _handle_annotations_before_blck_key_scalar();
1772 m_evt_handler->set_key_scalar_plain_empty();
1773 _handle_annotations_before_blck_val_scalar();
1774 m_evt_handler->set_val_scalar_plain_empty();
1775 }
1776 m_evt_handler->end_map_block();
1777}
1778
1779template<class EventHandler>
1780void ParseEngine<EventHandler>::_end_seq_blck()
1781{
1782 if(has_any(RVAL))
1783 {
1784 _c4dbgp("seqblck: set missing val");
1785 _handle_annotations_before_blck_val_scalar();
1786 m_evt_handler->set_val_scalar_plain_empty();
1787 }
1788 m_evt_handler->end_seq_block();
1789}
1790
1791template<class EventHandler>
1792void ParseEngine<EventHandler>::_end2_map()
1793{
1794 _c4dbgp("map: end");
1795 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RMAP), m_evt_handler->m_curr->pos);
1796 if(has_any(RBLCK))
1797 {
1798 _end_map_blck();
1799 }
1800 else
1801 {
1802 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RFLOW), m_evt_handler->m_curr->pos);
1803 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(USTY), m_evt_handler->m_curr->pos);
1804 m_evt_handler->_pop();
1805 }
1806}
1807
1808template<class EventHandler>
1809void ParseEngine<EventHandler>::_end2_seq()
1810{
1811 _c4dbgp("seq: end");
1812 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RSEQ), m_evt_handler->m_curr->pos);
1813 if(has_any(RBLCK))
1814 {
1815 _end_seq_blck();
1816 }
1817 else
1818 {
1819 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RFLOW), m_evt_handler->m_curr->pos);
1820 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(USTY), m_evt_handler->m_curr->pos);
1821 m_evt_handler->_pop();
1822 }
1823}
1824
1825template<class EventHandler>
1826void ParseEngine<EventHandler>::_begin2_doc()
1827{
1828 _c4dbgp("begin_doc");
1829 m_has_directives_yaml = false;
1830 m_has_directives = false;
1831 m_doc_empty = true;
1832 add_flags(RDOC);
1833 m_evt_handler->begin_doc();
1834 m_evt_handler->m_curr->indref = 0; // ?
1835}
1836
1837template<class EventHandler>
1838void ParseEngine<EventHandler>::_begin2_doc_expl()
1839{
1840 _c4dbgp("begin_doc_expl");
1841 m_has_directives_yaml = false;
1842 m_has_directives = false;
1843 m_doc_empty = true;
1844 add_flags(RDOC);
1845 m_evt_handler->begin_doc_expl();
1846 m_evt_handler->m_curr->indref = 0; // ?
1847}
1848
1849template<class EventHandler>
1850void ParseEngine<EventHandler>::_end2_doc()
1851{
1852 _c4dbgp("doc: end");
1853 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RDOC), m_evt_handler->m_curr->pos);
1854 if(m_doc_empty || (m_pending_tags.num_entries || m_pending_anchors.num_entries))
1855 {
1856 _c4dbgp("doc was empty; add empty val");
1857 _handle_annotations_before_blck_val_scalar();
1858 m_evt_handler->set_val_scalar_plain_empty();
1859 }
1860 m_evt_handler->end_doc();
1861 m_bom_len = 0;
1862}
1863
1864template<class EventHandler>
1865void ParseEngine<EventHandler>::_end2_doc_expl()
1866{
1867 _c4dbgp("doc: end");
1868 if(m_doc_empty || (m_pending_tags.num_entries || m_pending_anchors.num_entries))
1869 {
1870 _c4dbgp("doc: no children; add empty val");
1871 _handle_annotations_before_blck_val_scalar();
1872 m_evt_handler->set_val_scalar_plain_empty();
1873 }
1874 m_evt_handler->end_doc_expl();
1875 m_bom_len = 0;
1876}
1877
1878template<class EventHandler>
1879void ParseEngine<EventHandler>::_maybe_begin_doc()
1880{
1881 if(has_none(RDOC))
1882 {
1883 _c4dbgp("doc must be started");
1884 _begin2_doc();
1885 }
1886}
1887template<class EventHandler>
1888void ParseEngine<EventHandler>::_maybe_end_doc()
1889{
1890 if(has_any(RDOC))
1891 {
1892 _c4dbgp("doc must be finished");
1893 _end2_doc();
1894 }
1895 else if(m_doc_empty && (m_pending_tags.num_entries || m_pending_anchors.num_entries))
1896 {
1897 _c4dbgp("no doc to finish, but pending annotations");
1898 m_evt_handler->begin_doc();
1899 _handle_annotations_before_blck_val_scalar();
1900 m_evt_handler->set_val_scalar_plain_empty();
1901 m_evt_handler->end_doc();
1902 }
1903}
1904
1905template<class EventHandler>
1906void ParseEngine<EventHandler>::_end_doc_suddenly__pop()
1907{
1908 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_stack.size() >= 1, m_evt_handler->m_curr->pos);
1909 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_stack[0].flags & RDOC, m_evt_handler->m_curr->pos);
1910 _c4dbgp("root is RDOC");
1911 if(m_evt_handler->m_curr->level != 0)
1912 _handle_indentation_pop(&m_evt_handler->m_stack[0]);
1913 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RDOC), m_evt_handler->m_curr->pos);
1914}
1915
1916/** Check whether the current parse tokens are trailing on the
1917 * previous doc, and raise an error if they are */
1918template<class EventHandler>
1919void ParseEngine<EventHandler>::_check_trailing_doc_token()
1920{
1921 const bool is_root = (m_evt_handler->m_stack.size() == 1u);
1922 const bool isndoc = (m_evt_handler->m_curr->flags & NDOC) != 0;
1923 const bool suspicious = m_evt_handler->template has_any_<MAP|SEQ|VAL>();
1924 _c4dbgpf("target={} isroot={} suspicious={} ndoc={}", m_evt_handler->m_curr->node_id, is_root, suspicious, isndoc);
1925 if((is_root || m_evt_handler->template has_any_<DOC>()) && suspicious && !isndoc)
1926 _c4err("parse error");
1927}
1928
1929template<class EventHandler>
1930void ParseEngine<EventHandler>::_end_doc_suddenly()
1931{
1932 _c4dbgp("end doc suddenly");
1933 _end_doc_suddenly__pop();
1934 _end2_doc_expl();
1935 addrem_flags(RUNK|RTOP|NDOC, RMAP|RSEQ|RDOC);
1936}
1937
1938template<class EventHandler>
1939void ParseEngine<EventHandler>::_check_doc_end_tokens() const
1940{
1941 csubstr rem = m_evt_handler->m_curr->line_contents.rem;
1942 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !rem.begins_with_any(". \t"), m_evt_handler->m_curr->pos);
1943 if C4_UNLIKELY(rem.len && !rem.begins_with('#'))
1944 {
1945 _c4err("parse error");
1946 }
1947}
1948
1949template<class EventHandler>
1950void ParseEngine<EventHandler>::_start_doc_suddenly()
1951{
1952 _c4dbgp("start doc suddenly");
1953 _end_doc_suddenly__pop();
1954 _end2_doc();
1955 _begin2_doc_expl();
1956}
1957
1958template<class EventHandler>
1959void ParseEngine<EventHandler>::_end_stream()
1960{
1961 _c4dbgpf("end_stream, level={} node_id={}", m_evt_handler->m_curr->level, m_evt_handler->m_curr->node_id);
1962 if C4_UNLIKELY(has_all(RSEQ|RFLOW))
1963 _c4err("missing terminating ]");
1964 else if C4_UNLIKELY(has_all(RMAP|RFLOW))
1965 _c4err("missing terminating }");
1966 if(m_evt_handler->m_stack.size() > 1)
1967 _handle_indentation_pop(m_evt_handler->m_stack.begin());
1968 if(has_all(RDOC))
1969 {
1970 _end2_doc();
1971 }
1972 else if(has_all(RTOP|RUNK))
1973 {
1974 if(m_pending_anchors.num_entries || m_pending_tags.num_entries)
1975 {
1976 if(m_doc_empty)
1977 {
1978 m_evt_handler->begin_doc();
1979 _handle_annotations_before_blck_val_scalar();
1980 m_evt_handler->set_val_scalar_plain_empty();
1981 m_evt_handler->end_doc();
1982 }
1983 }
1984 }
1985 m_evt_handler->end_stream();
1986 if C4_UNLIKELY(m_has_directives)
1987 _c4err("directives cannot be used without a document");
1988}
1989
1990
1991template<class EventHandler>
1992void ParseEngine<EventHandler>::_handle_indentation_pop(ParserState const* popto)
1993{
1994 _c4dbgpf("popping {} level{}: from level {}(@ind={}) to level {}(@ind={})", m_evt_handler->m_curr->level - popto->level, (((m_evt_handler->m_curr->level - popto->level) > 1) ? "s" : ""), m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref, popto->level, popto->indref);
1995 while(m_evt_handler->m_curr != popto)
1996 {
1997 if(has_any(RSEQ))
1998 {
1999 _c4dbgpf("popping seq at level {} (indentation={},addr={})", m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref, m_evt_handler->m_curr);
2000 _end2_seq();
2001 }
2002 else if(has_any(RMAP))
2003 {
2004 _c4dbgpf("popping map at level {} (indentation={},addr={})", m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref, m_evt_handler->m_curr);
2005 _end2_map();
2006 }
2007 else
2008 {
2009 break;
2010 }
2011 }
2012 _c4dbgpf("current level is {} (indentation={})", m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
2013}
2014
2015template<class EventHandler>
2016void ParseEngine<EventHandler>::_handle_indentation_pop_from_block_seq()
2017{
2018 // search the stack frame to jump to based on its indentation
2019 using state_type = typename EventHandler::state;
2020 state_type const* popto = nullptr;
2021 auto &stack = m_evt_handler->m_stack;
2022 RYML_ASSERT_PARSE_CB_(stack.m_callbacks, stack.is_contiguous(), m_evt_handler->m_curr->pos); // this search relies on the stack being contiguous
2023 RYML_ASSERT_PARSE_CB_(stack.m_callbacks, m_evt_handler->m_curr >= stack.begin() && m_evt_handler->m_curr < stack.end(), m_evt_handler->m_curr->pos);
2024 const size_t ind = m_evt_handler->m_curr->line_contents.indentation;
2025 #ifdef RYML_DBG
2026 _print_state_stack();
2027 #endif
2028 for(state_type const* s = m_evt_handler->m_curr-1; s >= stack.begin(); --s)
2029 {
2030 _c4dbgpf("searching for state with indentation {}. curr={} (level={},node={})", ind, s->indref, s->level, s->node_id);
2031 if(s->indref == ind)
2032 {
2033 _c4dbgpf("gotit!!! level={} node={}", s->level, s->node_id);
2034 popto = s;
2035 break;
2036 }
2037 }
2038 if(!popto || popto >= m_evt_handler->m_curr || popto->level >= m_evt_handler->m_curr->level)
2039 {
2040 _c4err("parse error: incorrect indentation?");
2041 }
2042 _handle_indentation_pop(popto);
2043}
2044
2045template<class EventHandler>
2046void ParseEngine<EventHandler>::_handle_indentation_pop_from_block_map()
2047{
2048 // search the stack frame to jump to based on its indentation
2049 using state_type = typename EventHandler::state;
2050 auto &stack = m_evt_handler->m_stack;
2051 RYML_ASSERT_PARSE_CB_(stack.m_callbacks, stack.is_contiguous(), m_evt_handler->m_curr->pos); // this search relies on the stack being contiguous
2052 RYML_ASSERT_PARSE_CB_(stack.m_callbacks, m_evt_handler->m_curr >= stack.begin() && m_evt_handler->m_curr < stack.end(), m_evt_handler->m_curr->pos);
2053 const size_t ind = m_evt_handler->m_curr->line_contents.indentation;
2054 state_type const* popto = nullptr;
2055 #ifdef RYML_DBG
2056 char flagbuf_[128];
2057 _print_state_stack(flagbuf_);
2058 #endif
2059 for(state_type const* s = m_evt_handler->m_curr-1; s > stack.begin(); --s) // never go to the stack bottom. that's the root
2060 {
2061 _c4dbgpf("searching for state with indentation {}. current: ind={},level={},node={},flags={}", ind, s->indref, s->level, s->node_id, detail::_parser_flags_to_str(flagbuf_, s->flags));
2062 if(s->indref < ind)
2063 {
2064 break;
2065 }
2066 else if(s->indref == ind)
2067 {
2068 _c4dbgpf("same indentation!!! level={} node={}", s->level, s->node_id);
2069 if(popto && has_any(RTOP, s) && has_none(RMAP|RSEQ, s))
2070 {
2071 break;
2072 }
2073 popto = s;
2074 if(has_all(RSEQ|RBLCK, s))
2075 {
2076 csubstr rem = m_evt_handler->m_curr->line_contents.rem;
2077 const size_t first = rem.first_not_of(' ');
2078 RYML_ASSERT_PARSE_CB_(stack.m_callbacks, first == ind || first == npos, m_evt_handler->m_curr->pos);
2079 rem = rem.right_of(first, true);
2080 _c4dbgpf("indentless? rem='{}' first={}", rem, first);
2081 if(rem.begins_with('-') && _is_blck_token(rem))
2082 {
2083 _c4dbgp("parent was indentless seq");
2084 break;
2085 }
2086 }
2087 }
2088 }
2089 if(!popto || popto >= m_evt_handler->m_curr || popto->level >= m_evt_handler->m_curr->level)
2090 {
2091 _c4err("parse error: incorrect indentation?");
2092 }
2093 _handle_indentation_pop(popto);
2094}
2095
2096
2097//-----------------------------------------------------------------------------
2098template<class EventHandler>
2099void ParseEngine<EventHandler>::_check_valid_newline_in_quoted_scalar()
2100{
2101 if C4_UNLIKELY(has_all(RMAP|RBLCK|RKEY))
2102 {
2103 _c4err("multiline quoted keys are invalid");
2104 }
2105 else // check contextual indentation
2106 {
2107 const size_t minindent = m_evt_handler->m_curr->indref + ((has_any(RMAP|RSEQ) && has_any(RBLCK)));
2108 _c4dbgpf("indent={} vs minindent={} indref={}", m_evt_handler->m_curr->line_contents.indentation, minindent, m_evt_handler->m_curr->indref);
2109 if(m_evt_handler->m_curr->line_contents.indentation < minindent)
2110 {
2111 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks,
2112 m_evt_handler->m_curr->line_contents.indentation == m_evt_handler->m_curr->line_contents.rem.first_not_of(' '),
2113 m_evt_handler->m_curr->pos);
2114 csubstr trimmed = m_evt_handler->m_curr->line_contents.rem.sub(m_evt_handler->m_curr->line_contents.indentation);
2115 _c4dbgpf("trimmed.len={} line={}", trimmed.len, prs_(m_evt_handler->m_curr->line_contents.rem, true));
2116 if C4_UNLIKELY(!!trimmed.len)
2117 {
2118 _c4err("bad indentation");
2119 }
2120 }
2121 }
2122}
2123
2124
2125//-----------------------------------------------------------------------------
2126template<class EventHandler>
2127ScannedScalar ParseEngine<EventHandler>::_scan_scalar_squot()
2128{
2129 // quoted scalars can spread over multiple lines!
2130 // nice explanation here: http://yaml-multiline.info/
2131
2132 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, _buf().sub(m_evt_handler->m_curr->pos.offset).begins_with('\''), m_evt_handler->m_curr->pos);
2133
2134 // a span to the end of the file, skipping the opening quote
2135 substr s = _buf().sub(m_evt_handler->m_curr->pos.offset + 1);
2136 _line_progressed(1); // advance over the opening quote
2137 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !m_evt_handler->m_curr->at_line_beginning(), m_evt_handler->m_curr->pos);
2138
2139 bool needs_filter = false;
2140 size_t pos = npos; // find the pos of the matching quote
2141 while( ! _finished_file())
2142 {
2143 const csubstr line = m_evt_handler->m_curr->line_contents.rem;
2144 _c4dbgpf("scanning single quoted scalar @ line[{}]: {}", m_evt_handler->m_curr->pos.line, prs_(line));
2145 if C4_UNLIKELY(m_evt_handler->m_curr->at_line_beginning() && _is_doc_token(line))
2146 _c4err("token can not appear at line begin");
2147 for(size_t i = 0; i < line.len; ++i)
2148 {
2149 const char curr = line.str[i];
2150 if(curr == '\'') // single quotes are escaped with two single quotes
2151 {
2152 const char next = i+1 < line.len ? line.str[i+1] : '~';
2153 if(next != '\'') // so just look for the first quote
2154 { // without another after it
2155 _line_progressed(i + 1); // progress beyond the quote
2156 pos = i + (size_t)(line.str - s.str); // set pos to before the quote
2157 goto found_close;
2158 }
2159 else
2160 {
2161 needs_filter = true; // needs filter to remove escaped quotes
2162 ++i; // skip the escaped quote
2163 }
2164 }
2165 }
2166
2167 needs_filter = true;
2168 _line_progressed(line.len);
2169 _line_ended();
2170 _scan_line();
2171 _check_valid_newline_in_quoted_scalar();
2172 }
2173
2174 _c4err("reached end of file while looking for closing quote");
2175
2176found_close:
2177
2178 _c4dbgpf("found closing quote at: {}", pos);
2179 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, pos != npos, m_evt_handler->m_curr->pos);
2180 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, pos >= 0, m_evt_handler->m_curr->pos);
2181 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.end() >= _buf().begin() && s.end() <= _buf().end(), m_evt_handler->m_curr->pos);
2182 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.end() == _buf().end() || *s.end() == '\'', m_evt_handler->m_curr->pos);
2183 _set_first_strict(s, pos);
2184
2185 _c4prscalar("scanned squoted scalar", s, /*keep_newlines*/true);
2186
2187 return ScannedScalar { s, needs_filter };
2188}
2189
2190
2191//-----------------------------------------------------------------------------
2192template<class EventHandler>
2193ScannedScalar ParseEngine<EventHandler>::_scan_scalar_dquot()
2194{
2195 // quoted scalars can spread over multiple lines!
2196 // nice explanation here: http://yaml-multiline.info/
2197
2198 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, _buf().sub(m_evt_handler->m_curr->pos.offset).begins_with('"'), m_evt_handler->m_curr->pos);
2199
2200 // a span to the end of the file, skipping the opening quote
2201 substr s = _buf().sub(m_evt_handler->m_curr->pos.offset + 1);
2202 _line_progressed(1); // advance over the opening quote
2203 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, !m_evt_handler->m_curr->at_line_beginning(), m_evt_handler->m_curr->pos);
2204
2205 bool needs_filter = false;
2206 size_t pos = npos; // find the pos of the matching quote
2207 while( ! _finished_file())
2208 {
2209 #if defined(__GNUC__) && (/*__GNUC__ == 12 || */__GNUC__ == 13)
2210 C4_DONT_OPTIMIZE(m_evt_handler->m_curr->line_contents.rem); // prevent hoisting
2211 #endif
2212 csubstr rem = m_evt_handler->m_curr->line_contents.rem;
2213 _c4dbgpf("scanning double quoted scalar @ line[{}]: line='{}'", m_evt_handler->m_curr->pos.line, rem);
2214 if C4_UNLIKELY(m_evt_handler->m_curr->at_line_beginning() && _is_doc_token(rem))
2215 _c4err("token can not appear at line begin");
2216 for(size_t i = 0; i < rem.len; ++i)
2217 {
2218 const char curr = rem.str[i];
2219 // every \ is an escape
2220 if(curr == '\\')
2221 {
2222 const char next = i+1 < rem.len ? rem.str[i+1] : '~';
2223 needs_filter = true;
2224 if(next == '"' || next == '\\')
2225 ++i;
2226 }
2227 else if(curr == '"')
2228 {
2229 _line_progressed(i + 1); // progress beyond the quote
2230 pos = i + (size_t)(rem.str - s.str); // set pos to before the quote
2231 goto found_close;
2232 }
2233 }
2234
2235 // leading whitespace also needs filtering
2236 needs_filter = true;
2237 _line_progressed(rem.len);
2238 _line_ended();
2239 _scan_line();
2240 _check_valid_newline_in_quoted_scalar();
2241 }
2242
2243 _c4err("reached end of file while looking for closing quote");
2244
2245found_close:
2246
2247 _c4dbgpf("found closing quote at: {}", pos);
2248 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, pos != npos, m_evt_handler->m_curr->pos);
2249 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, pos >= 0, m_evt_handler->m_curr->pos);
2250 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.end() >= _buf().begin() && s.end() <= _buf().end(), m_evt_handler->m_curr->pos);
2251 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.end() == _buf().end() || *s.end() == '"', m_evt_handler->m_curr->pos);
2252 _set_first_strict(s, pos);
2253
2254 _c4prscalar("scanned dquoted scalar", s, /*keep_newlines*/true);
2255
2256 return ScannedScalar{s, needs_filter};
2257}
2258
2259
2260//-----------------------------------------------------------------------------
2261template<class EventHandler>
2262void ParseEngine<EventHandler>::_scan_block(ScannedBlock *C4_RESTRICT sb, size_t indref)
2263{
2264 _c4dbgpf("blck: indref={}", indref);
2265 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, indref != npos, m_evt_handler->m_curr->pos);
2266
2267 // nice explanation here: http://yaml-multiline.info/
2268 csubstr s = m_evt_handler->m_curr->line_contents.rem;
2269 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with('|') || s.begins_with('>'), m_evt_handler->m_curr->pos);
2270
2271 _c4dbgpf("blck: specs={}", prs_(s));
2272
2273 // parse the spec
2274 BlockChomp_e chomp = CHOMP_CLIP; // default to clip unless + or - are used
2275 size_t indentation = npos; // have to find out if no spec is given
2276 if(s.len > 1)
2277 {
2278 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.begins_with_any("|>"), m_evt_handler->m_curr->pos);
2279 csubstr t = s.sub(1);
2280 _c4dbgpf("blck: spec is multichar: {}", prs_(t));
2281 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, t.len >= 1, m_evt_handler->m_curr->pos);
2282 size_t pos = t.first_of("-+");
2283 _c4dbgpf("blck: spec chomp char: pos={}", pos);
2284 if(pos != npos)
2285 {
2286 _c4dbgpf("blck: spec chomp char: {}", _c4prc(t[pos]));
2287 if(t[pos] == '-')
2288 {
2289 _c4dbgp("blck: chomp=STRIP");
2290 chomp = CHOMP_STRIP;
2291 }
2292 else if(t[pos] == '+')
2293 {
2294 _c4dbgp("blck: chomp=KEEP");
2295 chomp = CHOMP_KEEP;
2296 }
2297 if(pos == 0)
2298 t = t.sub(1);
2299 else
2300 t = t.first(pos);
2301 _c4dbgpf("blck: spec is now: {}", prs_(t));
2302 }
2303 // from here to the end, only digits are considered
2304 pos = t.first_not_of("0123456789");
2305 csubstr rest = t.first(pos);
2306 if( ! rest.empty())
2307 {
2308 _c4dbgpf("blck: parse indentation digits: {}", prs_(rest));
2309 if C4_UNLIKELY(rest.len > 1)
2310 _c4err("parse error: invalid indentation");
2311 if C4_UNLIKELY( ! c4::atou(rest, &indentation))
2312 _c4err("parse error: could not read indentation as decimal"); // LCOV_EXCL_LINE
2313 if C4_UNLIKELY( ! indentation)
2314 _c4err("parse error: null indentation");
2315 _c4dbgpf("blck: indentation specified: {}. add {} from curr state -> {}", indentation, m_evt_handler->m_curr->indref, indentation+indref);
2316 indentation += m_evt_handler->m_curr->indref;
2317 }
2318 else
2319 {
2320 rest = t.triml(" \t");
2321 _c4dbgpf("blck: digits empty. t={} trimmed={} iscomm={} t.iscomm={}", prs_(t), prs_(rest), rest.begins_with('#'), t.begins_with('#'));
2322 if C4_UNLIKELY(rest.len && (rest.str[0] != '#' || t.str[0] == '#'))
2323 _c4err("parse error: invalid token");
2324 }
2325 }
2326
2327 _c4dbgpf("blck: style={} chomp={} indentation={}", s.begins_with('>') ? "fold" : "literal", chomp==CHOMP_CLIP ? "clip" : (chomp==CHOMP_STRIP ? "strip" : "keep"), indentation);
2328
2329 // finish the current line
2330 _line_progressed(s.len);
2331 _line_ended();
2332 _scan_line();
2333
2334 // start with a zero-length block, already pointing at the right place
2335 substr raw_block(_buf().data() + m_evt_handler->m_curr->pos.offset, size_t(0));
2336 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, raw_block.begin() == m_evt_handler->m_curr->line_contents.full.str, m_evt_handler->m_curr->pos);
2337
2338 // read every full line into a raw block,
2339 // from which newlines are to be stripped as needed.
2340 //
2341 // If no explicit indentation was given, pick it from the first
2342 // non-empty line. See
2343 // https://yaml.org/spec/1.2.2/#8111-block-indentation-indicator
2344 size_t num_lines = 0;
2345 size_t first = m_evt_handler->m_curr->pos.line;
2346 size_t provisional_indentation = npos;
2347 LineContents lc;
2348 while(( ! _finished_file()))
2349 {
2350 // peek next line, but do not advance immediately
2351 lc.reset_with_next_line(_buf(), m_evt_handler->m_curr->pos.offset);
2352 #if defined(__GNUC__) && (__GNUC__ == 12 || __GNUC__ == 13)
2353 C4_DONT_OPTIMIZE(lc.rem);
2354 #endif
2355 _c4dbgpf("blck: peeking at {}", prs_(lc.rem.trimr("\r\n"), true));
2356 // evaluate termination conditions
2357 if(indentation != npos)
2358 {
2359 _c4dbgpf("blck: indentation={}", indentation);
2360 // stop when the line is deindented and not empty
2361 if(lc.indentation < indentation && ( ! lc.rem.trim(" \t").empty()))
2362 {
2363 if(raw_block.len)
2364 {
2365 _c4dbgpf("blck: indentation decreased ref={} thisline={}", indentation, lc.indentation);
2366 }
2367 else
2368 {
2369 _c4err("indentation decreased without any scalar");
2370 }
2371 break;
2372 }
2373 else if(indentation == 0)
2374 {
2375 _c4dbgpf("blck: noindent. lc.rem={}", prs_(lc.rem));
2376 if(_is_doc_token(lc.rem))
2377 {
2378 _c4dbgp("blck: stop. indentation=0 and doc ended");
2379 break;
2380 }
2381 }
2382 }
2383 else
2384 {
2385 const size_t fns = lc.rem.first_not_of(' ');
2386 _c4dbgpf("blck: indentation ref not set. firstnonws={}", fns);
2387 if(fns != npos) // non-empty line
2388 {
2389 _c4dbgpf("blck: line not empty. indref={} indprov={} indentation={}", indref, provisional_indentation, lc.indentation);
2390 if C4_UNLIKELY(lc.full.begins_with('\t'))
2391 _c4err("parse error");
2392 if(provisional_indentation == npos)
2393 {
2394 if(lc.indentation < indref)
2395 {
2396 _c4dbgpf("blck: block terminated indentation={} < indref={}", lc.indentation, indref);
2397 if(raw_block.len == 0)
2398 {
2399 _c4dbgp("blck: was empty, undo next line");
2400 _line_ended_undo();
2401 }
2402 break;
2403 }
2404 else if(lc.indentation == m_evt_handler->m_curr->indref)
2405 {
2406 if(has_any(RSEQ|RMAP))
2407 {
2408 _c4dbgpf("blck: block terminated. reading container and indentation={}==indref={}", lc.indentation, m_evt_handler->m_curr->indref);
2409 break;
2410 }
2411 }
2412 _c4dbgpf("blck: set indentation ref from this line: ref={}", lc.indentation);
2413 indentation = lc.indentation;
2414 }
2415 else
2416 {
2417 if(lc.indentation >= provisional_indentation)
2418 {
2419 _c4dbgpf("blck: set indentation ref from provisional indentation: provisional_ref={}, thisline={}", provisional_indentation, lc.indentation);
2420 //indentation = provisional_indentation ? provisional_indentation : lc.indentation;
2421 indentation = lc.indentation;
2422 }
2423 else
2424 {
2425 if(lc.indentation >= indref)
2426 _c4err("parse error: first non-empty block line should have at least the original indentation");
2427 _c4dbgp("blck: finished");
2428 break;
2429 }
2430 }
2431 }
2432 else // empty line
2433 {
2434 _c4dbgpf("blck: line empty or {} spaces. line_indentation={} prov_indentation={}", lc.rem.len, lc.indentation, provisional_indentation);
2435 if(provisional_indentation != npos)
2436 {
2437 if(lc.rem.len >= provisional_indentation)
2438 {
2439 _c4dbgpf("blck: increase provisional_ref {} -> {}", provisional_indentation, lc.rem.len);
2440 provisional_indentation = lc.rem.len;
2441 }
2442 }
2443 else
2444 {
2445 provisional_indentation = lc.indentation ? lc.indentation : has_any(RSEQ|RVAL);
2446 _c4dbgpf("blck: initialize provisional_ref={}", provisional_indentation);
2447 if(provisional_indentation == npos)
2448 {
2449 provisional_indentation = lc.rem.len ? lc.rem.len : has_any(RSEQ|RVAL);
2450 _c4dbgpf("blck: initialize provisional_ref={}", provisional_indentation);
2451 }
2452 if(provisional_indentation < indref)
2453 {
2454 provisional_indentation = indref;
2455 _c4dbgpf("blck: initialize provisional_ref={}", provisional_indentation);
2456 }
2457 }
2458 }
2459 }
2460 // advance now that we know the folded scalar continues
2461 m_evt_handler->m_curr->line_contents = lc;
2462 _c4dbgpf("blck: append '{}'", m_evt_handler->m_curr->line_contents.rem);
2463 raw_block.len += m_evt_handler->m_curr->line_contents.full.len;
2464 _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
2465 _line_ended();
2466 ++num_lines;
2467 }
2468 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->pos.line == (first + num_lines) || (raw_block.len == 0), m_evt_handler->m_curr->pos);
2469 C4_UNUSED(num_lines);
2470 C4_UNUSED(first);
2471
2472 if(indentation == npos)
2473 {
2474 _c4dbgpf("blck: set indentation from provisional: {}", provisional_indentation);
2475 indentation = provisional_indentation;
2476 }
2477
2478 if(num_lines)
2479 _line_ended_undo();
2480
2481 _c4prscalar("scanned block", raw_block, /*keep_newlines*/true);
2482
2483 sb->scalar = raw_block;
2484 sb->indentation = indentation;
2485 sb->chomp = chomp;
2486}
2487
2488
2489//-----------------------------------------------------------------------------
2490//-----------------------------------------------------------------------------
2491//-----------------------------------------------------------------------------
2492/** @cond dev */
2493
2494// a debugging scaffold:
2495#if 0
2496#define _c4dbgfws(fmt, ...) _c4dbgpf("filt_ws[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
2497#else
2498#define _c4dbgfws(...)
2499#endif
2500
2501template<class EventHandler>
2502template<class FilterProcessor>
2503bool ParseEngine<EventHandler>::_filter_ws_handle_to_first_non_space(FilterProcessor &proc)
2504{
2505 _c4dbgfws("found whitespace '{}'", _c4prc(proc.curr()));
2506 RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.curr() == ' ' || proc.curr() == '\t', m_evt_handler->m_curr->pos);
2507
2508 const size_t first_pos = proc.rpos > 0 ? proc.src.first_not_of(" \t", proc.rpos) : proc.src.first_not_of(' ', proc.rpos);
2509 if(first_pos != npos)
2510 {
2511 const char first_char = proc.src[first_pos];
2512 _c4dbgfws("firstnonws='{}'@{}", _c4prc(first_char), first_pos);
2513 if(first_char == '\n' || first_char == '\r') // skip trailing whitespace
2514 {
2515 _c4dbgfws("whitespace is trailing on line", "");
2516 proc.skip(first_pos - proc.rpos);
2517 }
2518 else // a legit whitespace
2519 {
2520 proc.copy();
2521 _c4dbgfws("legit whitespace. sofar={}", prs_(proc.sofar()));
2522 }
2523 return true;
2524 }
2525 _c4dbgfws("whitespace is trailing on line", "");
2526 return false;
2527}
2528
2529template<class EventHandler>
2530template<class FilterProcessor>
2531void ParseEngine<EventHandler>::_filter_ws_copy_trailing(FilterProcessor &proc)
2532{
2533 if(!_filter_ws_handle_to_first_non_space(proc))
2534 {
2535 _c4dbgfws("... everything else is trailing whitespace - copy {} chars", proc.src.len - proc.rpos);
2536 proc.copy(proc.src.len - proc.rpos);
2537 }
2538}
2539
2540template<class EventHandler>
2541template<class FilterProcessor>
2542void ParseEngine<EventHandler>::_filter_ws_skip_trailing(FilterProcessor &proc)
2543{
2544 if(!_filter_ws_handle_to_first_non_space(proc))
2545 {
2546 _c4dbgfws("... everything else is trailing whitespace - skip {} chars", proc.src.len - proc.rpos);
2547 proc.skip(proc.src.len - proc.rpos);
2548 }
2549}
2550
2551#undef _c4dbgfws
2552
2553
2554//-----------------------------------------------------------------------------
2555//-----------------------------------------------------------------------------
2556//-----------------------------------------------------------------------------
2557/* plain scalars */
2558
2559// a debugging scaffold:
2560#if 0
2561#define _c4dbgfps(fmt, ...) _c4dbgpf("filt_plain[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
2562#else
2563#define _c4dbgfps(fmt, ...)
2564#endif
2565
2566template<class EventHandler>
2567template<class FilterProcessor>
2568void ParseEngine<EventHandler>::_filter_nl_plain(FilterProcessor &C4_RESTRICT proc, size_t indentation)
2569{
2570 RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.curr() == '\n', m_evt_handler->m_curr->pos);
2571
2572 _c4dbgfps("found newline. sofar={}", prs_(proc.sofar()));
2573 size_t ii = proc.rpos;
2574 const size_t numnl_following = _count_following_newlines(proc.src, &ii, indentation);
2575 if(numnl_following)
2576 {
2577 proc.set('\n', numnl_following);
2578 _c4dbgfps("{} consecutive (empty) lines {}. totalws={}", 1+numnl_following, ii < proc.src.len ? "in the middle" : "at the end", proc.rpos-ii);
2579 }
2580 else
2581 {
2582 const size_t ret = proc.src.first_not_of(" \t", proc.rpos+1);
2583 if(ret != npos)
2584 {
2585 proc.set(' ');
2586 _c4dbgfps("single newline. convert to space. ret={}/{}. sofar={}", ii, proc.src.len, prs_(proc.sofar()));
2587 }
2588 else
2589 {
2590 _c4dbgfps("last newline, everything else is whitespace. ii={}/{}", ii, proc.src.len);
2591 ii = proc.src.len;
2592 }
2593 }
2594 proc.rpos = ii;
2595}
2596
2597template<class EventHandler>
2598template<class FilterProcessor>
2599auto ParseEngine<EventHandler>::_filter_plain(FilterProcessor &C4_RESTRICT proc, size_t indentation) -> decltype(proc.result())
2600{
2601 RYML_ASSERT_PARSE_CB_(this->callbacks(), indentation != npos, m_evt_handler->m_curr->pos);
2602 _c4dbgfps("before={}", prs_(proc.src));
2603
2604 while(proc.has_more_chars())
2605 {
2606 const char curr = proc.curr();
2607 _c4dbgfps("'{}', sofar={}", _c4prc(curr), prs_(proc.sofar()));
2608 switch(curr)
2609 {
2610 case ' ':
2611 RYML_WITH_TAB_TOKENS_(case '\t':)
2612 _c4dbgfps("whitespace", curr);
2613 _filter_ws_skip_trailing(proc);
2614 break;
2615 case '\n':
2616 _c4dbgfps("newline", curr);
2617 _filter_nl_plain(proc, /*indentation*/indentation);
2618 break;
2619 case '\r': // skip \r --- https://stackoverflow.com/questions/1885900
2620 _c4dbgfps("carriage return, ignore", curr);
2621 proc.skip();
2622 break;
2623 default:
2624 proc.copy();
2625 break;
2626 }
2627 }
2628
2629 _c4dbgfps("after={}", prs_(proc.sofar()));
2630
2631 return proc.result();
2632}
2633
2634#undef _c4dbgfps
2635
2636
2637template<class EventHandler>
2638FilterResult ParseEngine<EventHandler>::filter_scalar_plain(csubstr scalar, substr dst, size_t indentation)
2639{
2640 FilterProcessorSrcDst proc(scalar, dst);
2641 return _filter_plain(proc, indentation);
2642}
2643
2644template<class EventHandler>
2645FilterResult ParseEngine<EventHandler>::filter_scalar_plain_in_place(substr dst, size_t cap, size_t indentation)
2646{
2647 FilterProcessorInplaceEndExtending proc(dst, cap);
2648 return _filter_plain(proc, indentation);
2649}
2650
2651
2652//-----------------------------------------------------------------------------
2653//-----------------------------------------------------------------------------
2654//-----------------------------------------------------------------------------
2655/* single quoted */
2656
2657// a debugging scaffold:
2658#if 0
2659#define _c4dbgfsq(fmt, ...) _c4dbgpf("filt_squo[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
2660#else
2661#define _c4dbgfsq(fmt, ...)
2662#endif
2663
2664template<class EventHandler>
2665template<class FilterProcessor>
2666void ParseEngine<EventHandler>::_filter_nl_squoted(FilterProcessor &C4_RESTRICT proc)
2667{
2668 RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.curr() == '\n', m_evt_handler->m_curr->pos);
2669
2670 _c4dbgfsq("found newline. sofar={}", prs_(proc.sofar()));
2671 size_t ii = proc.rpos;
2672 const size_t numnl_following = _count_following_newlines(proc.src, &ii);
2673 if(numnl_following)
2674 {
2675 proc.set('\n', numnl_following);
2676 _c4dbgfsq("{} consecutive (empty) lines {}. totalws={}", 1+numnl_following, ii < proc.src.len ? "in the middle" : "at the end", proc.rpos-ii);
2677 }
2678 else
2679 {
2680 const size_t ret = proc.src.first_not_of(" \t", proc.rpos+1);
2681 if(ret != npos)
2682 {
2683 proc.set(' ');
2684 _c4dbgfsq("single newline. convert to space. ret={}/{}. sofar={}", ii, proc.src.len, prs_(proc.sofar()));
2685 }
2686 else
2687 {
2688 proc.set(' ');
2689 _c4dbgfsq("single newline. convert to space. ii={}/{}. sofar={}", ii, proc.src.len, prs_(proc.sofar()));
2690 }
2691 }
2692 proc.rpos = ii;
2693}
2694
2695template<class EventHandler>
2696template<class FilterProcessor>
2697auto ParseEngine<EventHandler>::_filter_squoted(FilterProcessor &C4_RESTRICT proc) -> decltype(proc.result())
2698{
2699 _c4dbgfsq("before={}", prs_(proc.src));
2700
2701 // from the YAML spec for double-quoted scalars:
2702 // https://yaml.org/spec/1.2-old/spec.html#style/flow/single-quoted
2703 while(proc.has_more_chars())
2704 {
2705 const char curr = proc.curr();
2706 _c4dbgfsq("'{}', sofar={}", _c4prc(curr), prs_(proc.sofar()));
2707 switch(curr)
2708 {
2709 case ' ':
2710 case '\t':
2711 _c4dbgfsq("whitespace", curr);
2712 _filter_ws_copy_trailing(proc);
2713 break;
2714 case '\n':
2715 _c4dbgfsq("newline", curr);
2716 _filter_nl_squoted(proc);
2717 break;
2718 case '\r': // skip \r --- https://stackoverflow.com/questions/1885900
2719 _c4dbgfsq("skip cr", curr);
2720 proc.skip();
2721 break;
2722 case '\'':
2723 _c4dbgfsq("squote", curr);
2724 if(proc.next() == '\'')
2725 {
2726 _c4dbgfsq("two consecutive squotes", curr);
2727 proc.skip();
2728 proc.copy();
2729 }
2730 else
2731 {
2732 _c4err("filter error");
2733 }
2734 break;
2735 default:
2736 proc.copy();
2737 break;
2738 }
2739 }
2740
2741 _c4dbgfsq(": #filteredchars={} after={}", proc.src.len-proc.sofar().len, prs_(proc.sofar()));
2742
2743 return proc.result();
2744}
2745
2746#undef _c4dbgfsq
2747
2748template<class EventHandler>
2749FilterResult ParseEngine<EventHandler>::filter_scalar_squoted(csubstr scalar, substr dst)
2750{
2751 FilterProcessorSrcDst proc(scalar, dst);
2752 return _filter_squoted(proc);
2753}
2754
2755template<class EventHandler>
2756FilterResult ParseEngine<EventHandler>::filter_scalar_squoted_in_place(substr dst, size_t cap)
2757{
2758 FilterProcessorInplaceEndExtending proc(dst, cap);
2759 return _filter_squoted(proc);
2760}
2761
2762
2763//-----------------------------------------------------------------------------
2764//-----------------------------------------------------------------------------
2765//-----------------------------------------------------------------------------
2766/* double quoted */
2767
2768// a debugging scaffold:
2769#if 0
2770#define _c4dbgfdq(fmt, ...) _c4dbgpf("filt_dquo[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
2771#else
2772#define _c4dbgfdq(...)
2773#endif
2774
2775template<class EventHandler>
2776template<class FilterProcessor>
2777void ParseEngine<EventHandler>::_filter_nl_dquoted(FilterProcessor &C4_RESTRICT proc)
2778{
2779 RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.curr() == '\n', m_evt_handler->m_curr->pos);
2780
2781 _c4dbgfdq("found newline. sofar={}", prs_(proc.sofar()));
2782 size_t ii = proc.rpos;
2783 const size_t numnl_following = _count_following_newlines(proc.src, &ii);
2784 if(numnl_following)
2785 {
2786 proc.set('\n', numnl_following);
2787 _c4dbgfdq("{} consecutive (empty) lines {}. totalws={}", 1+numnl_following, ii < proc.src.len ? "in the middle" : "at the end", proc.rpos-ii);
2788 }
2789 else
2790 {
2791 const size_t ret = proc.src.first_not_of(" \t", proc.rpos+1);
2792 if(ret != npos)
2793 {
2794 proc.set(' ');
2795 _c4dbgfdq("single newline. convert to space. ret={}/{}. sofar={}", ii, proc.src.len, prs_(proc.sofar()));
2796 }
2797 else
2798 {
2799 proc.set(' ');
2800 _c4dbgfdq("single newline. convert to space. ii={}/{}. sofar={}", ii, proc.src.len, prs_(proc.sofar()));
2801 }
2802 if(ii < proc.src.len && proc.src.str[ii] == '\\')
2803 {
2804 _c4dbgfdq("backslash at [{}]", ii);
2805 const char next = ii+1 < proc.src.len ? proc.src.str[ii+1] : '\0';
2806 if(next == ' ' || next == '\t')
2807 {
2808 _c4dbgfdq("extend skip to backslash", "");
2809 ++ii;
2810 }
2811 }
2812 }
2813 proc.rpos = ii;
2814}
2815
2816template<class EventHandler>
2817template<class FilterProcessor>
2818void ParseEngine<EventHandler>::_filter_dquoted_backslash_decode(FilterProcessor &C4_RESTRICT proc, size_t sz)
2819{
2820 const size_t szp1 = sz + 1u;
2821 if C4_UNLIKELY(proc.rpos + szp1 >= proc.src.len)
2822 _c4err("codepoint requires {} hex digits. scalar pos={}", sz, proc.rpos);
2823 char readbuf[8];
2824 csubstr codepoint = proc.src.sub(proc.rpos + 2u, sz);
2825 _c4dbgfdq("utf8 ~~~{}~~~ rpos={} rem=~~~{}~~~", codepoint, proc.rpos, proc.src.sub(proc.rpos));
2826 uint32_t codepoint_val = {};
2827 if C4_UNLIKELY(!read_hex(codepoint, &codepoint_val))
2828 _c4err("failed to parse codepoint. scalar pos={}", proc.rpos);
2829 const size_t numbytes = decode_code_point((uint8_t*)readbuf, sizeof(readbuf), codepoint_val);
2830 if C4_UNLIKELY(numbytes == 0)
2831 _c4err("failed to decode code point={}", proc.rpos);
2832 RYML_ASSERT_PARSE_CB_(callbacks(), numbytes <= 4, m_evt_handler->m_curr->pos);
2833 proc.translate_esc_bulk(readbuf, numbytes, /*nread*/szp1);
2834 _c4dbgfdq("utf8 after rpos={} rem=~~~{}~~~", proc.rpos, proc.src.sub(proc.rpos));
2835}
2836
2837template<class EventHandler>
2838template<class FilterProcessor>
2839void ParseEngine<EventHandler>::_filter_dquoted_backslash(FilterProcessor &C4_RESTRICT proc)
2840{
2841 char next = proc.next();
2842 _c4dbgfdq("backslash, next='{}'", _c4prc(next));
2843 if(next == '\r')
2844 {
2845 if(proc.rpos+2 < proc.src.len && proc.src.str[proc.rpos+2] == '\n')
2846 {
2847 proc.skip(); // newline escaped with \ -- skip both (add only one as i is loop-incremented)
2848 next = '\n';
2849 _c4dbgfdq("[{}]: was \\r\\n, now next='\\n'", proc.rpos);
2850 }
2851 }
2852
2853 if(next == '\n')
2854 {
2855 size_t ii = proc.rpos + 2;
2856 for( ; ii < proc.src.len; ++ii)
2857 {
2858 // skip leading whitespace
2859 if(proc.src.str[ii] == ' ' || proc.src.str[ii] == '\t')
2860 ;
2861 else
2862 break;
2863 }
2864 proc.skip(ii - proc.rpos);
2865 }
2866 else if(next == '"' || next == '/' || next == ' ' || next == '\t')
2867 {
2868 // escapes for json compatibility
2869 proc.translate_esc(next);
2870 _c4dbgfdq("here, used '{}'", _c4prc(next));
2871 }
2872 else if(next == '\r')
2873 {
2874 proc.skip();
2875 }
2876 else if(next == 'n')
2877 {
2878 proc.translate_esc('\n');
2879 }
2880 else if(next == 'r')
2881 {
2882 proc.translate_esc('\r');
2883 }
2884 else if(next == 't')
2885 {
2886 proc.translate_esc('\t');
2887 }
2888 else if(next == '\\')
2889 {
2890 proc.translate_esc('\\');
2891 }
2892 else if(next == 'x') // 2-digit Unicode escape (\xXX), code point 0x00–0xFF
2893 {
2894 _filter_dquoted_backslash_decode(proc, 2u);
2895 }
2896 else if(next == 'u') // 4-digit Unicode escape (\uXXXX), code point 0x0000–0xFFFF
2897 {
2898 _filter_dquoted_backslash_decode(proc, 4u);
2899 }
2900 else if(next == 'U') // 8-digit Unicode escape (\UXXXXXXXX), full 32-bit code point
2901 {
2902 _filter_dquoted_backslash_decode(proc, 8u);
2903 }
2904 // https://yaml.org/spec/1.2.2/#rule-c-ns-esc-char
2905 else if(next == '0')
2906 {
2907 proc.translate_esc('\0');
2908 }
2909 else if(next == 'b') // backspace
2910 {
2911 proc.translate_esc('\b');
2912 }
2913 else if(next == 'f') // form feed
2914 {
2915 proc.translate_esc('\f');
2916 }
2917 else if(next == 'a') // bell character
2918 {
2919 proc.translate_esc('\a');
2920 }
2921 else if(next == 'v') // vertical tab
2922 {
2923 proc.translate_esc('\v');
2924 }
2925 else if(next == 'e') // escape character
2926 {
2927 proc.translate_esc('\x1b');
2928 }
2929 else if(next == '_') // unicode non breaking space \u00a0
2930 {
2931 // https://www.compart.com/en/unicode/U+00a0
2932 const char payload[] = {
2933 RYML_CHCONST_(-0x3e, 0xc2),
2934 RYML_CHCONST_(-0x60, 0xa0),
2935 };
2936 proc.translate_esc_bulk(payload, /*nwrite*/2, /*nread*/1);
2937 }
2938 else if(next == 'N') // unicode next line \u0085
2939 {
2940 // https://www.compart.com/en/unicode/U+0085
2941 const char payload[] = {
2942 RYML_CHCONST_(-0x3e, 0xc2),
2943 RYML_CHCONST_(-0x7b, 0x85),
2944 };
2945 proc.translate_esc_bulk(payload, /*nwrite*/2, /*nread*/1);
2946 }
2947 else if(next == 'L') // unicode line separator \u2028
2948 {
2949 // https://www.utf8-chartable.de/unicode-utf8-table.pl?start=8192&number=1024&names=-&utf8=0x&unicodeinhtml=hex
2950 const char payload[] = {
2951 RYML_CHCONST_(-0x1e, 0xe2),
2952 RYML_CHCONST_(-0x80, 0x80),
2953 RYML_CHCONST_(-0x58, 0xa8),
2954 };
2955 proc.translate_esc_extending(payload, /*nwrite*/3, /*nread*/1);
2956 }
2957 else if(next == 'P') // unicode paragraph separator \u2029
2958 {
2959 // https://www.utf8-chartable.de/unicode-utf8-table.pl?start=8192&number=1024&names=-&utf8=0x&unicodeinhtml=hex
2960 const char payload[] = {
2961 RYML_CHCONST_(-0x1e, 0xe2),
2962 RYML_CHCONST_(-0x80, 0x80),
2963 RYML_CHCONST_(-0x57, 0xa9),
2964 };
2965 proc.translate_esc_extending(payload, /*nwrite*/3, /*nread*/1);
2966 }
2967 else if(next == '\0')
2968 {
2969 proc.skip();
2970 }
2971 else
2972 {
2973 _c4err("unknown character '{}' after '\\' pos={}", _c4prc(next), proc.rpos);
2974 }
2975 _c4dbgfdq("backslash...sofar={}", prs_(proc.sofar()));
2976}
2977
2978
2979template<class EventHandler>
2980template<class FilterProcessor>
2981auto ParseEngine<EventHandler>::_filter_dquoted(FilterProcessor &C4_RESTRICT proc) -> decltype(proc.result())
2982{
2983 _c4dbgfdq("before={}", prs_(proc.src));
2984 // from the YAML spec for double-quoted scalars:
2985 // https://yaml.org/spec/1.2-old/spec.html#style/flow/double-quoted
2986 while(proc.has_more_chars())
2987 {
2988 const char curr = proc.curr();
2989 _c4dbgfdq("'{}' sofar={}", _c4prc(curr), prs_(proc.sofar()));
2990 switch(curr)
2991 {
2992 case ' ':
2993 case '\t':
2994 {
2995 _c4dbgfdq("whitespace", curr);
2996 _filter_ws_copy_trailing(proc);
2997 break;
2998 }
2999 case '\n':
3000 {
3001 _c4dbgfdq("newline", curr);
3002 _filter_nl_dquoted(proc);
3003 break;
3004 }
3005 case '\r': // skip \r --- https://stackoverflow.com/questions/1885900
3006 {
3007 _c4dbgfdq("carriage return, ignore", curr);
3008 proc.skip();
3009 break;
3010 }
3011 case '\\':
3012 {
3013 _filter_dquoted_backslash(proc);
3014 break;
3015 }
3016 default:
3017 {
3018 proc.copy();
3019 break;
3020 }
3021 }
3022 }
3023 _c4dbgfdq("after={}", prs_(proc.sofar()));
3024 return proc.result();
3025}
3026
3027#undef _c4dbgfdq
3028
3029
3030template<class EventHandler>
3031FilterResult ParseEngine<EventHandler>::filter_scalar_dquoted(csubstr scalar, substr dst)
3032{
3033 FilterProcessorSrcDst proc(scalar, dst);
3034 return _filter_dquoted(proc);
3035}
3036
3037template<class EventHandler>
3038FilterResultExtending ParseEngine<EventHandler>::filter_scalar_dquoted_in_place(substr dst, size_t cap)
3039{
3040 FilterProcessorInplaceMidExtending proc(dst, cap);
3041 return _filter_dquoted(proc);
3042}
3043
3044
3045//-----------------------------------------------------------------------------
3046//-----------------------------------------------------------------------------
3047//-----------------------------------------------------------------------------
3048// block filtering helpers
3049
3050template<class EventHandler>
3051template<class FilterProcessor>
3052void ParseEngine<EventHandler>::_filter_chomp(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp, size_t indentation)
3053{
3054 RYML_ASSERT_PARSE_CB_(this->callbacks(), chomp == CHOMP_CLIP || chomp == CHOMP_KEEP || chomp == CHOMP_STRIP, m_evt_handler->m_curr->pos);
3055 RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.rem().first_not_of(" \n\r") == npos, m_evt_handler->m_curr->pos);
3056
3057 // a debugging scaffold:
3058 #if 0
3059 #define _c4dbgchomp(fmt, ...) _c4dbgpf("chomp[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
3060 #else
3061 #define _c4dbgchomp(...)
3062 #endif
3063
3064 // advance to the last line having spaces beyond the indentation
3065 {
3066 size_t last = _find_last_newline_and_larger_indentation(proc.rem(), indentation);
3067 if(last != npos)
3068 {
3069 _c4dbgchomp("found newline and larger indentation. last={}", last);
3070 last = proc.rpos + last + size_t(1) + indentation; // last started at to-be-read.
3071 RYML_ASSERT_PARSE_CB_(this->callbacks(), last <= proc.src.len, m_evt_handler->m_curr->pos);
3072 // remove indentation spaces, copy the rest
3073 while((proc.rpos < last) && proc.has_more_chars())
3074 {
3075 const char curr = proc.curr();
3076 _c4dbgchomp("curr='{}'", _c4prc(curr));
3077 switch(curr)
3078 {
3079 case '\n':
3080 {
3081 _c4dbgchomp("newline! remlen={}", proc.rem().len);
3082 proc.copy();
3083 // are there spaces after the newline?
3084 csubstr at_next_line = proc.rem();
3085 if(at_next_line.begins_with(' '))
3086 {
3087 _c4dbgchomp("next line begins with spaces. indentation={}", indentation);
3088 // there are spaces.
3089 size_t first_non_space = at_next_line.first_not_of(' ');
3090 _c4dbgchomp("first_non_space={}", first_non_space);
3091 if(first_non_space == npos)
3092 {
3093 _c4dbgchomp("{} spaces, to the end", at_next_line.len);
3094 first_non_space = at_next_line.len;
3095 }
3096 if(first_non_space <= indentation)
3097 {
3098 _c4dbgchomp("skip spaces={}<=indentation={}", first_non_space, indentation);
3099 proc.skip(first_non_space);
3100 }
3101 else
3102 {
3103 _c4dbgchomp("skip indentation={}<spaces={}", indentation, first_non_space);
3104 proc.skip(indentation);
3105 // copy the spaces after the indentation
3106 _c4dbgchomp("copy {}={}-{} spaces", first_non_space - indentation, first_non_space, indentation);
3107 proc.copy(first_non_space - indentation);
3108 }
3109 }
3110 break;
3111 }
3112 case '\r':
3113 proc.skip();
3114 break;
3115 }
3116 }
3117 }
3118 }
3119
3120 // from now on, we only have line ends (or indentation spaces)
3121 switch(chomp)
3122 {
3123 case CHOMP_CLIP:
3124 {
3125 bool had_one = false;
3126 while(proc.has_more_chars())
3127 {
3128 const char curr = proc.curr();
3129 _c4dbgchomp("CLIP: '{}'", _c4prc(curr));
3130 switch(curr)
3131 {
3132 case '\n':
3133 {
3134 _c4dbgchomp("copy newline!", curr);
3135 proc.copy();
3136 proc.set_at_end();
3137 had_one = true;
3138 break;
3139 }
3140 case ' ':
3141 case '\r':
3142 _c4dbgchomp("skip!", curr);
3143 proc.skip();
3144 break;
3145 }
3146 }
3147 if(!had_one) // there were no newline characters. add one.
3148 {
3149 _c4dbgchomp("chomp=CLIP: add missing newline @{}", proc.wpos);
3150 proc.set('\n');
3151 }
3152 break;
3153 }
3154 case CHOMP_KEEP:
3155 {
3156 _c4dbgchomp("chomp=KEEP: copy all remaining new lines of {} characters", proc.rem().len);
3157 while(proc.has_more_chars())
3158 {
3159 const char curr = proc.curr();
3160 _c4dbgchomp("KEEP: '{}'", _c4prc(curr));
3161 switch(curr)
3162 {
3163 case '\n':
3164 _c4dbgchomp("copy newline!", curr);
3165 proc.copy();
3166 break;
3167 case ' ':
3168 case '\r':
3169 _c4dbgchomp("skip!", curr);
3170 proc.skip();
3171 break;
3172 }
3173 }
3174 break;
3175 }
3176 case CHOMP_STRIP:
3177 {
3178 _c4dbgchomp("chomp=STRIP: strip {} characters", proc.rem().len);
3179 // nothing to do!
3180 break;
3181 }
3182 }
3183
3184 #undef _c4dbgchomp
3185}
3186
3187
3188// a debugging scaffold:
3189#if 0
3190#define _c4dbgfb(fmt, ...) _c4dbgpf("filt_block[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
3191#else
3192#define _c4dbgfb(...)
3193#endif
3194
3195template<class EventHandler>
3196template<class FilterProcessor>
3197void ParseEngine<EventHandler>::_filter_block_indentation(FilterProcessor &C4_RESTRICT proc, size_t indentation)
3198{
3199 csubstr rem = proc.rem(); // remaining
3200 if(rem.len)
3201 {
3202 size_t first = rem.first_not_of(' ');
3203 if(first != npos)
3204 {
3205 _c4dbgfb("{} spaces follow before next nonws character", first);
3206 if(first < indentation)
3207 {
3208 _c4dbgfb("skip {}<{} spaces from indentation", first, indentation);
3209 proc.skip(first);
3210 }
3211 else
3212 {
3213 _c4dbgfb("skip {} spaces from indentation", indentation);
3214 proc.skip(indentation);
3215 }
3216 }
3217 #ifdef RYML_NO_COVERAGE__TO_BE_DELETED
3218 else
3219 {
3220 _c4dbgfb("all spaces to the end: {} spaces", first);
3221 first = rem.len;
3222 if(first)
3223 {
3224 if(first < indentation)
3225 {
3226 _c4dbgfb("skip everything", first);
3227 proc.skip(proc.src.len - proc.rpos);
3228 }
3229 else
3230 {
3231 _c4dbgfb("skip {} spaces from indentation", indentation);
3232 proc.skip(indentation);
3233 }
3234 }
3235 }
3236 #endif
3237 }
3238}
3239
3240template<class EventHandler>
3241template<class FilterProcessor>
3242size_t ParseEngine<EventHandler>::_handle_all_whitespace(FilterProcessor &C4_RESTRICT proc, BlockChomp_e chomp)
3243{
3244 csubstr contents = proc.src.trimr(" \n\r");
3245 _c4dbgfb("ws: contents_len={} wslen={}", contents.len, proc.src.len-contents.len);
3246 if(!contents.len)
3247 {
3248 _c4dbgfb("ws: all whitespace: len={}", proc.src.len);
3249 if(chomp == CHOMP_KEEP && proc.src.len)
3250 {
3251 _c4dbgfb("ws: chomp=KEEP all {} newlines", proc.src.count('\n'));
3252 while(proc.has_more_chars())
3253 {
3254 const char curr = proc.curr();
3255 if(curr == '\n')
3256 proc.copy();
3257 else
3258 proc.skip();
3259 }
3260 if(!proc.wpos)
3261 {
3262 proc.set('\n');
3263 }
3264 }
3265 }
3266 return contents.len;
3267}
3268
3269template<class EventHandler>
3270template<class FilterProcessor>
3271size_t ParseEngine<EventHandler>::_extend_to_chomp(FilterProcessor &C4_RESTRICT proc, size_t contents_len)
3272{
3273 _c4dbgfb("contents_len={}", contents_len);
3274
3275 RYML_ASSERT_PARSE_CB_(this->callbacks(), contents_len > 0u, m_evt_handler->m_curr->pos);
3276
3277 // extend contents to just before the first newline at the end,
3278 // in case it is preceded by spaces
3279 size_t firstnewl = proc.src.first_of('\n', contents_len);
3280 if(firstnewl != npos)
3281 {
3282 contents_len = firstnewl;
3283 _c4dbgfb("contents_len={} <--- firstnewl={}", contents_len, firstnewl);
3284 }
3285 else
3286 {
3287 contents_len = proc.src.len;
3288 _c4dbgfb("contents_len={} <--- src.len={}", contents_len, proc.src.len);
3289 }
3290
3291 return contents_len;
3292}
3293
3294#undef _c4dbgfb
3295
3296
3297//-----------------------------------------------------------------------------
3298//-----------------------------------------------------------------------------
3299//-----------------------------------------------------------------------------
3300
3301// a debugging scaffold:
3302#if 0
3303#define _c4dbgfbl(fmt, ...) _c4dbgpf("filt_block_lit[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
3304#else
3305#define _c4dbgfbl(...)
3306#endif
3307
3308template<class EventHandler>
3309template<class FilterProcessor>
3310auto ParseEngine<EventHandler>::_filter_block_literal(FilterProcessor &C4_RESTRICT proc, size_t indentation, BlockChomp_e chomp) -> decltype(proc.result())
3311{
3312 _c4dbgfbl("indentation={} before={}", indentation, prs_(proc.src));
3313
3314 size_t contents_len = _handle_all_whitespace(proc, chomp);
3315 if(!contents_len)
3316 return proc.result();
3317
3318 contents_len = _extend_to_chomp(proc, contents_len);
3319
3320 _c4dbgfbl("to filter={}", prs_(proc.src.first(contents_len)));
3321
3322 _filter_block_indentation(proc, indentation);
3323
3324 // now filter the bulk
3325 while(proc.has_more_chars(/*maxpos*/contents_len))
3326 {
3327 const char curr = proc.curr();
3328 _c4dbgfbl("'{}' sofar={}", _c4prc(curr), prs_(proc.sofar()));
3329 switch(curr)
3330 {
3331 case '\n':
3332 {
3333 _c4dbgfbl("found newline. skip indentation on the next line", curr);
3334 proc.copy(); // copy the newline
3335 _filter_block_indentation(proc, indentation);
3336 break;
3337 }
3338 case '\r':
3339 proc.skip();
3340 break;
3341 default:
3342 proc.copy();
3343 break;
3344 }
3345 }
3346
3347 _c4dbgfbl("before chomp: #tochomp={} sofar={}", proc.rem().len, prs_(proc.sofar()));
3348
3349 _filter_chomp(proc, chomp, indentation);
3350
3351 _c4dbgfbl("final={}", prs_(proc.sofar()));
3352
3353 return proc.result();
3354}
3355
3356#undef _c4dbgfbl
3357
3358template<class EventHandler>
3359FilterResult ParseEngine<EventHandler>::filter_scalar_block_literal(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp)
3360{
3361 FilterProcessorSrcDst proc(scalar, dst);
3362 return _filter_block_literal(proc, indentation, chomp);
3363}
3364
3365template<class EventHandler>
3366FilterResult ParseEngine<EventHandler>::filter_scalar_block_literal_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp)
3367{
3368 FilterProcessorInplaceEndExtending proc(scalar, cap);
3369 return _filter_block_literal(proc, indentation, chomp);
3370}
3371
3372
3373//-----------------------------------------------------------------------------
3374//-----------------------------------------------------------------------------
3375//-----------------------------------------------------------------------------
3376
3377// a debugging scaffold:
3378#if 0
3379#define _c4dbgfbf(fmt, ...) _c4dbgpf("filt_block_folded[{}->{}]: " fmt, proc.rpos, proc.wpos, __VA_ARGS__)
3380#else
3381#define _c4dbgfbf(...)
3382#endif
3383
3384
3385template<class EventHandler>
3386template<class FilterProcessor>
3387void ParseEngine<EventHandler>::_filter_block_folded_newlines_leading(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len)
3388{
3389 _filter_block_indentation(proc, indentation);
3390 while(proc.has_more_chars(len))
3391 {
3392 const char curr = proc.curr();
3393 _c4dbgfbf("'{}' sofar={}", _c4prc(curr), prs_(proc.sofar()));
3394 switch(curr)
3395 {
3396 case '\n':
3397 _c4dbgfbf("newline.", curr);
3398 proc.copy();
3399 _filter_block_indentation(proc, indentation);
3400 break;
3401 case '\r':
3402 proc.skip();
3403 break;
3404 case ' ':
3405 case '\t':
3406 {
3407 size_t first = proc.rem().first_not_of(" \t");
3408 _c4dbgfbf("space. first={}", first);
3409 if(first == npos)
3410 first = proc.rem().len;
3411 _c4dbgfbf("... indentation increased to {}", first);
3412 _filter_block_folded_indented_block(proc, indentation, len, first);
3413 break;
3414 }
3415 default:
3416 _c4dbgfbf("newl leading: not space, not newline. stop.", 0);
3417 return;
3418 }
3419 }
3420}
3421
3422template<class EventHandler>
3423template<class FilterProcessor>
3424size_t ParseEngine<EventHandler>::_filter_block_folded_newlines_compress(FilterProcessor &C4_RESTRICT proc, size_t num_newl, size_t wpos_at_first_newl)
3425{
3426 switch(num_newl)
3427 {
3428 case 1u:
3429 _c4dbgfbf("... this is the first newline. turn into space. wpos={}", proc.wpos);
3430 wpos_at_first_newl = proc.wpos;
3431 proc.skip();
3432 proc.set(' ');
3433 break;
3434 case 2u:
3435 _c4dbgfbf("... this is the second newline. prev space (at wpos={}) must be newline", wpos_at_first_newl);
3436 RYML_ASSERT_PARSE_CB_(this->callbacks(), wpos_at_first_newl != npos, m_evt_handler->m_curr->pos);
3437 RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.sofar()[wpos_at_first_newl] == ' ', m_evt_handler->m_curr->pos);
3438 RYML_ASSERT_PARSE_CB_(this->callbacks(), wpos_at_first_newl + 1u == proc.wpos, m_evt_handler->m_curr->pos);
3439 proc.skip();
3440 proc.set_at(wpos_at_first_newl, '\n');
3441 RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.sofar()[wpos_at_first_newl] == '\n', m_evt_handler->m_curr->pos);
3442 break;
3443 default:
3444 _c4dbgfbf("... subsequent newline (num_newl={}). copy", num_newl);
3445 proc.copy();
3446 break;
3447 }
3448 return wpos_at_first_newl;
3449}
3450
3451template<class EventHandler>
3452template<class FilterProcessor>
3453void ParseEngine<EventHandler>::_filter_block_folded_newlines(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len)
3454{
3455 RYML_ASSERT_PARSE_CB_(this->callbacks(), proc.curr() == '\n', m_evt_handler->m_curr->pos);
3456 size_t num_newl = 0;
3457 size_t wpos_at_first_newl = npos;
3458 while(proc.has_more_chars(len))
3459 {
3460 const char curr = proc.curr();
3461 _c4dbgfbf("'{}' sofar={}", _c4prc(curr), prs_(proc.sofar()));
3462 switch(curr)
3463 {
3464 case '\n':
3465 {
3466 _c4dbgfbf("newline. sofar={}", num_newl);
3467 // NOTE: vs2022-32bit-release builds were giving wrong
3468 // results in this block, if it was written as either
3469 // as a switch(num_newl) or its equivalent if-form.
3470 //
3471 // For this reason, we're using a dedicated function
3472 // (**_compress), which seems to work around the issue.
3473 //
3474 // The manifested problem was that somewhere between the
3475 // assignment to curr and this point, proc.wpos (the
3476 // write-position of the processor) jumped to npos, which
3477 // made the write wrap-around! To make things worse,
3478 // enabling prints via _c4dbgpf() and _c4dbgfbf() made the
3479 // problem go away!
3480 //
3481 // The only way to make the problem appear with prints
3482 // enabled was by disabling all prints in this function
3483 // (including in the block which was moved to the compress
3484 // function) and then selectively enabling only some of
3485 // those prints.
3486 //
3487 // This may be due to some bug in the cl-x86 optimizer; or
3488 // it may be triggered by some UB which may be
3489 // inadvertedly present in this function or in the filter
3490 // processor. This is despite our best efforts to weed out
3491 // any such UB problem: neither clang-tidy nor none of the
3492 // sanitizers, or gcc's -fanalyzer pointed to any problems
3493 // in this code.
3494 //
3495 // In the end, moving this block to a separate function
3496 // was the only way to bury the problem. But it may
3497 // resurface again, as The Undead, rising to from the
3498 // grave to haunt us with his terrible presence.
3499 //
3500 // We may have to revisit this. With a stake, and lots of
3501 // garlic.
3502 wpos_at_first_newl = _filter_block_folded_newlines_compress(proc, ++num_newl, wpos_at_first_newl);
3503 _filter_block_indentation(proc, indentation);
3504 break;
3505 }
3506 case ' ':
3507 case '\t':
3508 {
3509 size_t first = proc.rem().first_not_of(" \t");
3510 _c4dbgfbf("space. first={}", first);
3511 if(first == npos)
3512 first = proc.rem().len;
3513 _c4dbgfbf("... indentation increased to {}", first);
3514 if(num_newl)
3515 {
3516 _c4dbgfbf("... prev space (at wpos={}) must be newline", wpos_at_first_newl);
3517 proc.set_at(wpos_at_first_newl, '\n');
3518 }
3519 if(num_newl > 1u)
3520 {
3521 _c4dbgfbf("... add missing newline", wpos_at_first_newl);
3522 proc.set('\n');
3523 }
3524 _filter_block_folded_indented_block(proc, indentation, len, first);
3525 num_newl = 0;
3526 wpos_at_first_newl = npos;
3527 break;
3528 }
3529 case '\r':
3530 proc.skip();
3531 break;
3532 default:
3533 _c4dbgfbf("not space, not newline. stop.", 0);
3534 return;
3535 }
3536 }
3537}
3538
3539
3540template<class EventHandler>
3541template<class FilterProcessor>
3542void ParseEngine<EventHandler>::_filter_block_folded_indented_block(FilterProcessor &C4_RESTRICT proc, size_t indentation, size_t len, size_t curr_indentation) noexcept
3543{
3544 RYML_ASSERT_PARSE_CB_(this->callbacks(), (proc.rem().first_not_of(" \t") == curr_indentation) || (proc.rem().first_not_of(" \t") == npos), m_evt_handler->m_curr->pos);
3545 if(curr_indentation)
3546 proc.copy(curr_indentation);
3547 while(proc.has_more_chars(len))
3548 {
3549 const char curr = proc.curr();
3550 _c4dbgfbf("'{}' sofar={}", _c4prc(curr), prs_(proc.sofar()));
3551 switch(curr)
3552 {
3553 case '\n':
3554 {
3555 proc.copy();
3556 _filter_block_indentation(proc, indentation);
3557 csubstr rem = proc.rem();
3558 const size_t first = rem.first_not_of(' ');
3559 _c4dbgfbf("newline. firstns={}", first);
3560 if(first == 0)
3561 {
3562 const char c = rem[first];
3563 _c4dbgfbf("firstns={}='{}'", first, _c4prc(c));
3564 if(c != '\n' && c != '\r')
3565 {
3566 _c4dbgfbf("done with indented block", first);
3567 goto endloop;
3568 }
3569 }
3570 else if(first != npos)
3571 {
3572 proc.copy(first);
3573 _c4dbgfbf("copy all {} spaces", first);
3574 }
3575 break;
3576 }
3577 break;
3578 case '\r':
3579 proc.skip();
3580 break;
3581 default:
3582 proc.copy();
3583 break;
3584 }
3585 }
3586 endloop:
3587 return;
3588}
3589
3590
3591template<class EventHandler>
3592template<class FilterProcessor>
3593auto ParseEngine<EventHandler>::_filter_block_folded(FilterProcessor &C4_RESTRICT proc, size_t indentation, BlockChomp_e chomp) -> decltype(proc.result())
3594{
3595 _c4dbgfbf("indentation={} before={}", indentation, prs_(proc.src));
3596
3597 size_t contents_len = _handle_all_whitespace(proc, chomp);
3598 if(!contents_len)
3599 return proc.result();
3600
3601 contents_len = _extend_to_chomp(proc, contents_len);
3602
3603 _c4dbgfbf("to filter={}", prs_(proc.src.first(contents_len)));
3604
3605 _filter_block_folded_newlines_leading(proc, indentation, contents_len);
3606
3607 // now filter the bulk
3608 while(proc.has_more_chars(/*maxpos*/contents_len))
3609 {
3610 const char curr = proc.curr();
3611 _c4dbgfbf("'{}' sofar={}", _c4prc(curr), prs_(proc.sofar()));
3612 switch(curr)
3613 {
3614 case '\n':
3615 {
3616 _c4dbgfbf("found newline", curr);
3617 _filter_block_folded_newlines(proc, indentation, contents_len);
3618 break;
3619 }
3620 case '\r':
3621 proc.skip();
3622 break;
3623 default:
3624 proc.copy();
3625 break;
3626 }
3627 }
3628
3629 _c4dbgfbf("before chomp: #tochomp={} sofar={}", proc.rem().len, prs_(proc.sofar()));
3630
3631 _filter_chomp(proc, chomp, indentation);
3632
3633 _c4dbgfbf("final={}", proc.sofar().len, prs_(proc.sofar()));
3634
3635 return proc.result();
3636}
3637
3638#undef _c4dbgfbf
3639
3640template<class EventHandler>
3641FilterResult ParseEngine<EventHandler>::filter_scalar_block_folded(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp)
3642{
3643 FilterProcessorSrcDst proc(scalar, dst);
3644 return _filter_block_folded(proc, indentation, chomp);
3645}
3646
3647template<class EventHandler>
3648FilterResult ParseEngine<EventHandler>::filter_scalar_block_folded_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp)
3649{
3650 FilterProcessorInplaceEndExtending proc(scalar, cap);
3651 return _filter_block_folded(proc, indentation, chomp);
3652}
3653
3654
3655//-----------------------------------------------------------------------------
3656//-----------------------------------------------------------------------------
3657//-----------------------------------------------------------------------------
3658
3659template<class EventHandler>
3660csubstr ParseEngine<EventHandler>::_filter_scalar_plain(substr s, size_t indentation)
3661{
3662 _c4dbgpf("filtering plain scalar: s={}", prs_(s));
3663 FilterResult r = this->filter_scalar_plain_in_place(s, s.len, indentation);
3664 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, r.valid(), m_evt_handler->m_curr->pos);
3665 _c4dbgpf("filtering plain scalar: success! s={}", prs_(r.get()));
3666 return r.get();
3667}
3668
3669//-----------------------------------------------------------------------------
3670
3671template<class EventHandler>
3672csubstr ParseEngine<EventHandler>::_filter_scalar_squot(substr s)
3673{
3674 _c4dbgpf("filtering squo scalar: s={}", prs_(s));
3675 FilterResult r = this->filter_scalar_squoted_in_place(s, s.len);
3676 RYML_ASSERT_PARSE_CB_(this->callbacks(), r.valid(), m_evt_handler->m_curr->pos);
3677 _c4dbgpf("filtering squo scalar: success! s={}", prs_(r.get()));
3678 return r.get();
3679}
3680
3681
3682//-----------------------------------------------------------------------------
3683
3684template<class EventHandler>
3685csubstr ParseEngine<EventHandler>::_filter_scalar_dquot(substr s)
3686{
3687 _c4dbgpf("filtering dquo scalar: s={}", prs_(s));
3688 FilterResultExtending r = this->filter_scalar_dquoted_in_place(s, s.len);
3689 if C4_LIKELY(r.valid())
3690 {
3691 _c4dbgpf("filtering dquo scalar: success! s={}", prs_(r.get()));
3692 return r.get();
3693 }
3694 else
3695 {
3696 const size_t len = r.required_len();
3697 _c4dbgpf("filtering dquo scalar: not enough space: needs {}, have {}", len, s.len);
3698 substr dst = _alloc_arena(len, &s);
3699 _c4dbgpf("filtering dquo scalar: dst.len={}", dst.len);
3700 if(dst.str)
3701 {
3702 RYML_ASSERT_PARSE_CB_(this->callbacks(), dst.len == len, m_evt_handler->m_curr->pos);
3703 FilterResult rsd = this->filter_scalar_dquoted(s, dst);
3704 _c4dbgpf("filtering dquo scalar: ... result now needs {} was {}", rsd.required_len(), len);
3705 RYML_ASSERT_PARSE_CB_(this->callbacks(), rsd.required_len() <= len, m_evt_handler->m_curr->pos); // may be smaller!
3706 RYML_CHECK_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, rsd.valid(), m_evt_handler->m_curr->pos);
3707 _c4dbgpf("filtering dquo scalar: success! s={}", prs_(rsd.get()));
3708 return rsd.get();
3709 }
3710 return dst;
3711 }
3712}
3713
3714
3715//-----------------------------------------------------------------------------
3716
3717template<class EventHandler>
3718csubstr ParseEngine<EventHandler>::_move_scalar_left_and_add_newline(substr s)
3719{
3720 if(s.is_sub(_buf()))
3721 {
3722 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.str > _buf().str, m_evt_handler->m_curr->pos);
3723 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, s.str-1 >= _buf().str, m_evt_handler->m_curr->pos);
3724 if(s.len)
3725 memmove(s.str - 1, s.str, s.len);
3726 --s.str;
3727 s.str[s.len] = '\n';
3728 ++s.len;
3729 return s;
3730 }
3731 else
3732 {
3733 substr dst = _alloc_arena(s.len + 1, &s);
3734 if(s.len)
3735 memcpy(dst.str, s.str, s.len);
3736 dst[s.len] = '\n';
3737 return dst;
3738 }
3739}
3740
3741template<class EventHandler>
3742csubstr ParseEngine<EventHandler>::_filter_scalar_literal(substr s, size_t indentation, BlockChomp_e chomp)
3743{
3744 _c4dbgpf("filtering block literal scalar: s={}", prs_(s));
3745 FilterResult r = this->filter_scalar_block_literal_in_place(s, s.len, indentation, chomp);
3746 csubstr result;
3747 if C4_LIKELY(r.valid())
3748 {
3749 result = r.get();
3750 }
3751 else
3752 {
3753 _c4dbgpf("filtering block literal scalar: not enough space: needs {}, have {}", r.required_len(), s.len);
3754 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, r.required_len() == s.len + 1, m_evt_handler->m_curr->pos);
3755 // this can only happen when adding a single newline in clip mode.
3756 // so we shift left the scalar by one place
3757 result = _move_scalar_left_and_add_newline(s);
3758 }
3759 _c4dbgpf("filtering block literal scalar: success! s={}", prs_(result));
3760 return result;
3761}
3762
3763
3764//-----------------------------------------------------------------------------
3765template<class EventHandler>
3766csubstr ParseEngine<EventHandler>::_filter_scalar_folded(substr s, size_t indentation, BlockChomp_e chomp)
3767{
3768 _c4dbgpf("filtering block folded scalar: s={}", prs_(s));
3769 FilterResult r = this->filter_scalar_block_folded_in_place(s, s.len, indentation, chomp);
3770 csubstr result;
3771 if C4_LIKELY(r.valid())
3772 {
3773 result = r.get();
3774 }
3775 else
3776 {
3777 _c4dbgpf("filtering block folded scalar: not enough space: needs {}, have {}", r.required_len(), s.len);
3778 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, r.required_len() == s.len + 1, m_evt_handler->m_curr->pos);
3779 // this can only happen when adding a single newline in clip mode.
3780 // so we shift left the scalar by one place
3781 result = _move_scalar_left_and_add_newline(s);
3782 }
3783 _c4dbgpf("filtering block folded scalar: success! s={}", prs_(result));
3784 return result;
3785}
3786
3787
3788//-----------------------------------------------------------------------------
3789
3790template<class EventHandler>
3791csubstr ParseEngine<EventHandler>::_maybe_filter_key_scalar_plain(ScannedScalar const& C4_RESTRICT sc, size_t indentation)
3792{
3793 if(sc.needs_filter)
3794 {
3795 if(m_options.scalar_filtering())
3796 {
3797 return _filter_scalar_plain(sc.scalar, indentation);
3798 }
3799 else
3800 {
3801 _c4dbgp("plain scalar left unfiltered");
3802 m_evt_handler->mark_key_scalar_unfiltered();
3803 }
3804 }
3805 else
3806 {
3807 _c4dbgp("plain scalar doesn't need filtering");
3808 }
3809 return sc.scalar;
3810}
3811
3812template<class EventHandler>
3813csubstr ParseEngine<EventHandler>::_maybe_filter_val_scalar_plain(ScannedScalar const& C4_RESTRICT sc, size_t indentation)
3814{
3815 if(sc.needs_filter)
3816 {
3817 if(m_options.scalar_filtering())
3818 {
3819 return _filter_scalar_plain(sc.scalar, indentation);
3820 }
3821 else
3822 {
3823 _c4dbgp("plain scalar left unfiltered");
3824 m_evt_handler->mark_val_scalar_unfiltered();
3825 }
3826 }
3827 else
3828 {
3829 _c4dbgp("plain scalar doesn't need filtering");
3830 }
3831 return sc.scalar;
3832}
3833
3834
3835//-----------------------------------------------------------------------------
3836
3837template<class EventHandler>
3838csubstr ParseEngine<EventHandler>::_maybe_filter_key_scalar_squot(ScannedScalar const& C4_RESTRICT sc)
3839{
3840 if(sc.needs_filter)
3841 {
3842 if(m_options.scalar_filtering())
3843 {
3844 return _filter_scalar_squot(sc.scalar);
3845 }
3846 else
3847 {
3848 _c4dbgp("squo key scalar left unfiltered");
3849 m_evt_handler->mark_key_scalar_unfiltered();
3850 }
3851 }
3852 else
3853 {
3854 _c4dbgp("squo key scalar doesn't need filtering");
3855 }
3856 return sc.scalar;
3857}
3858
3859template<class EventHandler>
3860csubstr ParseEngine<EventHandler>::_maybe_filter_val_scalar_squot(ScannedScalar const& C4_RESTRICT sc)
3861{
3862 if(sc.needs_filter)
3863 {
3864 if(m_options.scalar_filtering())
3865 {
3866 return _filter_scalar_squot(sc.scalar);
3867 }
3868 else
3869 {
3870 _c4dbgp("squo val scalar left unfiltered");
3871 m_evt_handler->mark_val_scalar_unfiltered();
3872 }
3873 }
3874 else
3875 {
3876 _c4dbgp("squo val scalar doesn't need filtering");
3877 }
3878 return sc.scalar;
3879}
3880
3881
3882//-----------------------------------------------------------------------------
3883
3884template<class EventHandler>
3885csubstr ParseEngine<EventHandler>::_maybe_filter_key_scalar_dquot(ScannedScalar const& C4_RESTRICT sc)
3886{
3887 if(sc.needs_filter)
3888 {
3889 if(m_options.scalar_filtering())
3890 {
3891 return _filter_scalar_dquot(sc.scalar);
3892 }
3893 else
3894 {
3895 _c4dbgp("dquo scalar left unfiltered");
3896 m_evt_handler->mark_key_scalar_unfiltered();
3897 }
3898 }
3899 else
3900 {
3901 _c4dbgp("dquo scalar doesn't need filtering");
3902 }
3903 return sc.scalar;
3904}
3905
3906template<class EventHandler>
3907csubstr ParseEngine<EventHandler>::_maybe_filter_val_scalar_dquot(ScannedScalar const& C4_RESTRICT sc)
3908{
3909 if(sc.needs_filter)
3910 {
3911 if(m_options.scalar_filtering())
3912 {
3913 return _filter_scalar_dquot(sc.scalar);
3914 }
3915 else
3916 {
3917 _c4dbgp("dquo scalar left unfiltered");
3918 m_evt_handler->mark_val_scalar_unfiltered();
3919 }
3920 }
3921 else
3922 {
3923 _c4dbgp("dquo scalar doesn't need filtering");
3924 }
3925 return sc.scalar;
3926}
3927
3928
3929//-----------------------------------------------------------------------------
3930
3931template<class EventHandler>
3932csubstr ParseEngine<EventHandler>::_maybe_filter_key_scalar_literal(ScannedBlock const& C4_RESTRICT sb)
3933{
3934 if(m_options.scalar_filtering())
3935 {
3936 return _filter_scalar_literal(sb.scalar, sb.indentation, sb.chomp);
3937 }
3938 else
3939 {
3940 _c4dbgp("literal scalar left unfiltered");
3941 m_evt_handler->mark_key_scalar_unfiltered();
3942 }
3943 return sb.scalar;
3944}
3945
3946template<class EventHandler>
3947csubstr ParseEngine<EventHandler>::_maybe_filter_val_scalar_literal(ScannedBlock const& C4_RESTRICT sb)
3948{
3949 if(m_options.scalar_filtering())
3950 {
3951 return _filter_scalar_literal(sb.scalar, sb.indentation, sb.chomp);
3952 }
3953 else
3954 {
3955 _c4dbgp("literal scalar left unfiltered");
3956 m_evt_handler->mark_val_scalar_unfiltered();
3957 }
3958 return sb.scalar;
3959}
3960
3961
3962//-----------------------------------------------------------------------------
3963
3964template<class EventHandler>
3965csubstr ParseEngine<EventHandler>::_maybe_filter_key_scalar_folded(ScannedBlock const& C4_RESTRICT sb)
3966{
3967 if(m_options.scalar_filtering())
3968 {
3969 return _filter_scalar_folded(sb.scalar, sb.indentation, sb.chomp);
3970 }
3971 else
3972 {
3973 _c4dbgp("folded scalar left unfiltered");
3974 m_evt_handler->mark_key_scalar_unfiltered();
3975 }
3976 return sb.scalar;
3977}
3978
3979template<class EventHandler>
3980csubstr ParseEngine<EventHandler>::_maybe_filter_val_scalar_folded(ScannedBlock const& C4_RESTRICT sb)
3981{
3982 if(m_options.scalar_filtering())
3983 {
3984 return _filter_scalar_folded(sb.scalar, sb.indentation, sb.chomp);
3985 }
3986 else
3987 {
3988 _c4dbgp("folded scalar left unfiltered");
3989 m_evt_handler->mark_val_scalar_unfiltered();
3990 }
3991 return sb.scalar;
3992}
3993
3994
3995//-----------------------------------------------------------------------------
3996//-----------------------------------------------------------------------------
3997//-----------------------------------------------------------------------------
3998
3999#ifdef RYML_DBG // !!! <----------------------------------
4000
4001template<class EventHandler>
4002void ParseEngine<EventHandler>::add_flags(ParserFlag_t on)
4003{
4004 ParserState *s = m_evt_handler->m_curr;
4005 char buf1_[64], buf2_[64], buf3_[64];
4006 csubstr buf1 = detail::_parser_flags_to_str(buf1_, on);
4007 csubstr buf2 = detail::_parser_flags_to_str(buf2_, s->flags);
4008 csubstr buf3 = detail::_parser_flags_to_str(buf3_, s->flags|on);
4009 _c4dbgpf("state[{}]: add {}: before={} after={}", s->level, buf1, buf2, buf3);
4010 s->flags |= on;
4011}
4012
4013template<class EventHandler>
4014void ParseEngine<EventHandler>::addrem_flags(ParserFlag_t on, ParserFlag_t off)
4015{
4016 ParserState *s = m_evt_handler->m_curr;
4017 char buf1_[64], buf2_[64], buf3_[64], buf4_[64];
4018 csubstr buf1 = detail::_parser_flags_to_str(buf1_, on);
4019 csubstr buf2 = detail::_parser_flags_to_str(buf2_, off);
4020 csubstr buf3 = detail::_parser_flags_to_str(buf3_, s->flags);
4021 csubstr buf4 = detail::_parser_flags_to_str(buf4_, (~off)&((s->flags|on)));
4022 _c4dbgpf("state[{}]: add {} / rem {}: before={} after={}", s->level, buf1, buf2, buf3, buf4);
4023 RYML_ASSERT_BASIC_((on & off) == ParserFlag_t(0));
4024 s->flags &= ~off;
4025 s->flags |= on;
4026}
4027
4028template<class EventHandler>
4029void ParseEngine<EventHandler>::rem_flags(ParserFlag_t off)
4030{
4031 ParserState *s = m_evt_handler->m_curr;
4032 char buf1_[64], buf2_[64], buf3_[64];
4033 csubstr buf1 = detail::_parser_flags_to_str(buf1_, off);
4034 csubstr buf2 = detail::_parser_flags_to_str(buf2_, s->flags);
4035 csubstr buf3 = detail::_parser_flags_to_str(buf3_, s->flags&(~off));
4036 _c4dbgpf("state[{}]: rem {}: before={} after={}", s->level, buf1, buf2, buf3);
4037 s->flags &= ~off;
4038}
4039
4040inline C4_NO_INLINE csubstr detail::_parser_flags_to_str(substr buf, ParserFlag_t flags)
4041{
4042 size_t pos = 0;
4043 bool gotone = false;
4044
4045 #define _prflag(fl) \
4046 if((flags & fl) == (fl)) \
4047 { \
4048 if(gotone) \
4049 { \
4050 if(pos + 1 < buf.len) \
4051 buf[pos] = '|'; \
4052 ++pos; \
4053 } \
4054 csubstr fltxt = #fl; \
4055 if(pos + fltxt.len <= buf.len) \
4056 memcpy(buf.str + pos, fltxt.str, fltxt.len); \
4057 pos += fltxt.len; \
4058 gotone = true; \
4059 }
4060
4061 _prflag(RTOP);
4062 _prflag(RUNK);
4063 _prflag(RMAP);
4064 _prflag(RSEQ);
4065 _prflag(RFLOW);
4066 _prflag(RBLCK);
4067 _prflag(QMRK);
4068 _prflag(RKEY);
4069 _prflag(RVAL);
4070 _prflag(RKCL);
4071 _prflag(RNXT);
4072 _prflag(SSCL);
4073 _prflag(QSCL);
4074 _prflag(RSET);
4075 _prflag(RDOC);
4076 _prflag(NDOC);
4077 _prflag(USTY);
4078 _prflag(RSEQIMAP);
4079
4080 #undef _prflag
4081
4082 if(pos == 0)
4083 if(buf.len > 0)
4084 buf[pos++] = '0';
4085
4086 RYML_CHECK_BASIC_(pos <= buf.len);
4087
4088 return buf.first(pos);
4089}
4090
4091#endif // RYML_DBG !!! <----------------------------------
4092
4093
4094//-----------------------------------------------------------------------------
4095//-----------------------------------------------------------------------------
4096//-----------------------------------------------------------------------------
4097
4098template<class EventHandler>
4099csubstr ParseEngine<EventHandler>::location_contents(Location const& loc) const
4100{
4101 RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, loc.offset < _buf().len);
4102 return _buf().sub(loc.offset);
4103}
4104
4105template<class EventHandler>
4106Location ParseEngine<EventHandler>::val_location(const char *val) const
4107{
4108 if C4_UNLIKELY(val == nullptr)
4109 return {m_evt_handler->m_curr->pos.name, 0, 0, 0};
4110 RYML_CHECK_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_options.locations());
4111 // NOTE: if any of these checks fails, the parser needs to be
4112 // instantiated with locations enabled.
4113 RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_options.locations());
4114 RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, !_locations_dirty());
4115 RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets != nullptr);
4116 RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets_size > 0);
4117 // NOTE: the pointer needs to belong to the buffer that was used to parse.
4118 csubstr src = _buf();
4119 RYML_CHECK_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, val != nullptr || src.str == nullptr);
4120 RYML_CHECK_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, (val >= src.begin() && val <= src.end()) || (src.str == nullptr && val == nullptr));
4121 // ok. search the first stored newline after the given ptr
4122 using lineptr_type = size_t const* C4_RESTRICT;
4123 lineptr_type lineptr = nullptr;
4124 size_t offset = (size_t)(val - src.begin());
4125 if(m_newline_offsets_size < RYML_LOCATIONS_SMALL_THRESHOLD)
4126 {
4127 // just do a linear search if the size is small.
4128 for(lineptr_type curr = m_newline_offsets, last = m_newline_offsets + m_newline_offsets_size; curr < last; ++curr)
4129 {
4130 if(*curr > offset)
4131 {
4132 lineptr = curr;
4133 break;
4134 }
4135 }
4136 }
4137 else
4138 {
4139 // do a bisection search if the size is not small.
4140 //
4141 // We could use std::lower_bound but this is simple enough and
4142 // spares the costly include of <algorithm>.
4143 size_t count = m_newline_offsets_size;
4144 lineptr = m_newline_offsets;
4145 while(count)
4146 {
4147 size_t step = count >> 1;
4148 lineptr_type it = lineptr + step;
4149 if(*it < offset)
4150 {
4151 lineptr = ++it;
4152 count -= step + 1;
4153 }
4154 else
4155 {
4156 count = step;
4157 }
4158 }
4159 }
4160 RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, lineptr);
4161 RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, lineptr >= m_newline_offsets);
4162 RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, lineptr <= m_newline_offsets + m_newline_offsets_size);
4163 RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, lineptr && (*lineptr > offset));
4164 Location loc;
4165 loc.name = m_evt_handler->m_curr->pos.name;
4166 loc.offset = offset;
4167 loc.line = (size_t)(lineptr - m_newline_offsets);
4168 if(lineptr > m_newline_offsets)
4169 loc.col = (offset - *(lineptr-1) - 1u);
4170 else
4171 loc.col = offset;
4172 return loc;
4173}
4174
4175template<class EventHandler>
4176void ParseEngine<EventHandler>::_prepare_locations()
4177{
4178 csubstr src = _buf();
4179 size_t numnewlines = 1u + src.count('\n');
4180 _resize_locations(numnewlines);
4181 m_newline_offsets_size = 0;
4182 for(size_t i = 0; i < src.len; i++)
4183 if(src.str[i] == '\n')
4184 m_newline_offsets[m_newline_offsets_size++] = i; // NOLINT
4185 m_newline_offsets[m_newline_offsets_size++] = src.len; // NOLINT
4186 RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets_size == numnewlines);
4187}
4188
4189template<class EventHandler>
4190void ParseEngine<EventHandler>::_resize_locations(size_t numnewlines)
4191{
4192 numnewlines = numnewlines >= 16 ? numnewlines : 16;
4193 if(numnewlines > m_newline_offsets_capacity)
4194 {
4195 if(m_newline_offsets)
4196 RYML_CB_FREE_(m_evt_handler->m_stack.m_callbacks, m_newline_offsets, size_t, m_newline_offsets_capacity);
4197 m_newline_offsets = RYML_CB_ALLOC_HINT_(m_evt_handler->m_stack.m_callbacks, size_t, numnewlines, m_newline_offsets);
4198 m_newline_offsets_capacity = numnewlines;
4199 }
4200}
4201
4202template<class EventHandler>
4203bool ParseEngine<EventHandler>::_locations_dirty() const
4204{
4205 return !m_newline_offsets_size;
4206}
4207
4208
4209//-----------------------------------------------------------------------------
4210//-----------------------------------------------------------------------------
4211//-----------------------------------------------------------------------------
4212
4213template<class EventHandler>
4214void ParseEngine<EventHandler>::_handle_flow_skip_whitespace()
4215{
4216 // don't assign to csubstr rem: otherwise, gcc12,13,14 -O3 -m32 misbuilds
4217 if(m_evt_handler->m_curr->line_contents.rem.len > 0)
4218 {
4219 if(m_evt_handler->m_curr->line_contents.rem.str[0] == ' ' || m_evt_handler->m_curr->line_contents.rem.str[0] == '\t')
4220 {
4221 _c4dbgpf("starts with whitespace: '{}'", _c4prc(m_evt_handler->m_curr->line_contents.rem.str[0]));
4222 _skipchars(" \t");
4223 }
4224 // comments
4225 if(m_evt_handler->m_curr->line_contents.rem.begins_with('#'))
4226 {
4227 _c4dbgpf("it's a comment: {}", m_evt_handler->m_curr->line_contents.rem);
4228 _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
4229 }
4230 }
4231}
4232
4233
4234template<class EventHandler>
4235void ParseEngine<EventHandler>::_handle_flow_line_beginning()
4236{
4237 _c4dbgpf("flow: indref={} indentation={}", m_evt_handler->m_curr->indref, m_evt_handler->m_curr->line_contents.indentation);
4238 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->at_line_beginning(), m_evt_handler->m_curr->pos);
4239 if C4_UNLIKELY(m_evt_handler->m_curr->indentation_lt())
4240 {
4241 csubstr trimmed = m_evt_handler->m_curr->line_contents.rem.sub(m_evt_handler->m_curr->line_contents.indentation);
4242 _c4dbgpf("flow: after indentation={}", prs_(trimmed));
4243 if(trimmed.len && trimmed.triml(" \t").len)
4244 {
4245 _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
4246 _c4err("bad indentation");
4247 }
4248 }
4249}
4250
4251template<class EventHandler>
4252size_t ParseEngine<EventHandler>::_handle_block_skip_leading_whitespace()
4253{
4254 const size_t mark = m_evt_handler->m_curr->pos.offset;
4255 const size_t firstpos = m_evt_handler->m_curr->line_contents.rem.first_not_of(" \t");
4256 _c4dbgpf("block: mark={} firstpos={}", mark, firstpos);
4257 if(firstpos != npos)
4258 {
4259 _c4dbgp("block: non empty line");
4260 _line_progressed(firstpos);
4261 return mark;
4262 }
4263 else
4264 {
4265 _c4dbgp("block: rest of line is whitespace");
4266 _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
4267 return npos;
4268 }
4269}
4270
4271template<class EventHandler>
4272void ParseEngine<EventHandler>::_handle_block_check_leading_tabs(size_t start_mark, size_t end_mark)
4273{
4274 _c4dbgpf("block: start_mark={} end_mark={}", start_mark, end_mark);
4275 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, end_mark >= start_mark, m_evt_handler->m_curr->pos);
4276 if(end_mark != start_mark)
4277 {
4278 csubstr leading = _buf().range(start_mark, end_mark);
4279 _c4dbgpf("block: leading[{}-{}]={}", start_mark, end_mark, prs_(leading, true));
4280 size_t pos = leading.find('\t');
4281 if(pos != npos)
4282 {
4283 size_t fno = leading.first_not_of(" \t");
4284 if(fno == npos || pos < fno)
4285 _c4err("invalid tab character to the left");
4286 }
4287 (void)leading;
4288 }
4289}
4290
4291
4292//-----------------------------------------------------------------------------
4293
4294
4295template<class EventHandler>
4296void ParseEngine<EventHandler>::_handle_colon()
4297{
4298 size_t curr = m_evt_handler->m_curr->pos.line;
4299 if C4_UNLIKELY(m_prev_colon != npos && curr == m_prev_colon)
4300 {
4301 _c4dbgpf("colon: prevline={} currline={}", m_prev_colon, curr);
4302 _c4err("two colons on same line");
4303 }
4304 _c4dbgpf("colon: set prevline={}->{}", m_prev_colon, curr);
4305 m_prev_colon = curr;
4306}
4307
4308template<class EventHandler>
4309void ParseEngine<EventHandler>::_add_annotation(Annotation *C4_RESTRICT dst, csubstr str)
4310{
4311 _c4dbgpf("store annotation[{}]: {}", dst->num_entries, prs_(str));
4312 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, dst->num_entries < C4_COUNTOF(dst->annotations), m_evt_handler->m_curr->pos); // NOLINT(bugprone-sizeof-expression)
4313 dst->annotations[dst->num_entries].str = str;
4314 dst->annotations[dst->num_entries].indentation = {};
4315 dst->annotations[dst->num_entries].line = {};
4316 dst->annotations[dst->num_entries].orig = {};
4317 ++dst->num_entries;
4318}
4319
4320template<class EventHandler>
4321void ParseEngine<EventHandler>::_add_annotation(Annotation *C4_RESTRICT dst, csubstr str, size_t indentation, size_t line)
4322{
4323 _c4dbgpf("store annotation[{}]: '{}' indentation={} line={}", dst->num_entries, maybe_null_str_(str), indentation, line);
4324 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, dst->num_entries < C4_COUNTOF(dst->annotations), m_evt_handler->m_curr->pos); // NOLINT(bugprone-sizeof-expression)
4325 if C4_UNLIKELY(dst->num_entries && dst->annotations[0].line == line)
4326 {
4327 _c4err("parse error");
4328 }
4329 dst->annotations[dst->num_entries].str = str;
4330 dst->annotations[dst->num_entries].indentation = indentation;
4331 dst->annotations[dst->num_entries].line = line;
4332 dst->annotations[dst->num_entries].orig = {};
4333 ++dst->num_entries;
4334}
4335
4336template<class EventHandler>
4337void ParseEngine<EventHandler>::_add_annotation(Annotation *C4_RESTRICT dst, csubstr str, size_t indentation, size_t line, csubstr orig)
4338{
4339 _c4dbgpf("store annotation[{}]: '{}'->'{}' indentation={} line={}", dst->num_entries, orig, maybe_null_str_(str), indentation, line);
4340 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, dst->num_entries < C4_COUNTOF(dst->annotations), m_evt_handler->m_curr->pos); // NOLINT(bugprone-sizeof-expression)
4341 if C4_UNLIKELY(dst->num_entries && dst->annotations[0].line == line)
4342 {
4343 _c4err("parse error");
4344 }
4345 dst->annotations[dst->num_entries].str = str;
4346 dst->annotations[dst->num_entries].indentation = indentation;
4347 dst->annotations[dst->num_entries].line = line;
4348 dst->annotations[dst->num_entries].orig = orig;
4349 ++dst->num_entries;
4350}
4351
4352template<class EventHandler>
4353bool ParseEngine<EventHandler>::_annotations_require_key_container() const
4354{
4355 return m_pending_tags.num_entries > 1 || m_pending_anchors.num_entries > 1;
4356}
4357
4358template<class EventHandler>
4359bool ParseEngine<EventHandler>::_handle_annotations_before_unexpected_flow_token_rkey()
4360{
4361 if(!(m_pending_tags.num_entries | m_pending_anchors.num_entries))
4362 return false;
4363 _c4dbgpf("handle_annotations_before_unexpected_flow_comma_rkey, node={}", m_evt_handler->m_curr->node_id);
4364 if(m_pending_tags.num_entries)
4365 {
4366 _c4dbgpf("handle_annotations_before_unexpected_flow_comma_rkey, #tags={}", m_pending_tags.num_entries);
4367 if C4_LIKELY(m_pending_tags.num_entries == 1)
4368 {
4369 m_evt_handler->set_key_tag(m_pending_tags.annotations[0].str);
4370 _clear_annotations(&m_pending_tags);
4371 }
4372 else
4373 {
4374 _c4err("too many tags");
4375 }
4376 }
4377 if(m_pending_anchors.num_entries)
4378 {
4379 _c4dbgpf("handle_annotations_before_unexpected_flow_comma, #anchors={}", m_pending_tags.num_entries);
4380 if C4_LIKELY(m_pending_anchors.num_entries == 1)
4381 {
4382 m_evt_handler->set_key_anchor(m_pending_anchors.annotations[0].str);
4383 _clear_annotations(&m_pending_anchors);
4384 }
4385 else
4386 {
4387 _c4err("too many anchors");
4388 }
4389 }
4390 m_evt_handler->set_key_scalar_plain_empty();
4391 m_evt_handler->set_val_scalar_plain_empty();
4392 return true;
4393}
4394
4395template<class EventHandler>
4396void ParseEngine<EventHandler>::_handle_annotations_before_blck_key_scalar()
4397{
4398 _c4dbgpf("annotations_before_blck_key_scalar, node={}", m_evt_handler->m_curr->node_id);
4399 if(m_pending_tags.num_entries)
4400 {
4401 _c4dbgpf("annotations_before_blck_key_scalar, #tags={}", m_pending_tags.num_entries);
4402 if C4_LIKELY(m_pending_tags.num_entries == 1)
4403 {
4404 m_evt_handler->set_key_tag(m_pending_tags.annotations[0].str);
4405 _clear_annotations(&m_pending_tags);
4406 }
4407 else
4408 {
4409 _c4err("too many tags"); // LCOV_EXCL_LINE
4410 }
4411 }
4412 if(m_pending_anchors.num_entries)
4413 {
4414 _c4dbgpf("annotations_before_blck_key_scalar, #anchors={}", m_pending_anchors.num_entries);
4415 if C4_LIKELY(m_pending_anchors.num_entries == 1)
4416 {
4417 m_evt_handler->set_key_anchor(m_pending_anchors.annotations[0].str);
4418 _clear_annotations(&m_pending_anchors);
4419 }
4420 else
4421 {
4422 _c4err("too many anchors"); // LCOV_EXCL_LINE
4423 }
4424 }
4425}
4426
4427template<class EventHandler>
4428void ParseEngine<EventHandler>::_handle_annotations_before_blck_val_scalar()
4429{
4430 _c4dbgpf("annotations_before_blck_val_scalar, node={}", m_evt_handler->m_curr->node_id);
4431 if(m_pending_tags.num_entries)
4432 {
4433 _c4dbgpf("annotations_before_blck_val_scalar, #tags={}", m_pending_tags.num_entries);
4434 if C4_LIKELY(m_pending_tags.num_entries == 1)
4435 {
4436 m_evt_handler->set_val_tag(m_pending_tags.annotations[0].str);
4437 _clear_annotations(&m_pending_tags);
4438 }
4439 else
4440 {
4441 _c4err("too many tags");
4442 }
4443 }
4444 if(m_pending_anchors.num_entries)
4445 {
4446 _c4dbgpf("annotations_before_blck_val_scalar, #anchors={}", m_pending_anchors.num_entries);
4447 if C4_LIKELY(m_pending_anchors.num_entries == 1)
4448 {
4449 m_evt_handler->set_val_anchor(m_pending_anchors.annotations[0].str);
4450 _clear_annotations(&m_pending_anchors);
4451 }
4452 else
4453 {
4454 _c4err("too many anchors");
4455 }
4456 }
4457}
4458
4459template<class EventHandler>
4460void ParseEngine<EventHandler>::_handle_annotations_before_start_mapblck(size_t current_line)
4461{
4462 _c4dbgpf("annotations_before_start_mapblck, current_line={}", current_line);
4463 if(m_pending_tags.num_entries == 2)
4464 {
4465 _c4dbgp("2 tags, setting entry 0");
4466 m_evt_handler->set_val_tag(m_pending_tags.annotations[0].str);
4467 }
4468 else if(m_pending_tags.num_entries == 1)
4469 {
4470 _c4dbgpf("1 tag. line={}, curr={}", m_pending_tags.annotations[0].line, current_line);
4471 if(m_pending_tags.annotations[0].line < current_line)
4472 {
4473 _c4dbgp("...tag is for the map. setting it.");
4474 m_evt_handler->set_val_tag(m_pending_tags.annotations[0].str);
4475 _clear_annotations(&m_pending_tags);
4476 }
4477 }
4478 //
4479 if(m_pending_anchors.num_entries == 2)
4480 {
4481 _c4dbgp("2 anchors, setting entry 0");
4482 m_evt_handler->set_val_anchor(m_pending_anchors.annotations[0].str);
4483 }
4484 else if(m_pending_anchors.num_entries == 1)
4485 {
4486 _c4dbgpf("1 anchor. line={}, curr={}", m_pending_anchors.annotations[0].line, current_line);
4487 if(m_pending_anchors.annotations[0].line < current_line)
4488 {
4489 _c4dbgp("...anchor is for the map. setting it.");
4490 m_evt_handler->set_val_anchor(m_pending_anchors.annotations[0].str);
4491 _clear_annotations(&m_pending_anchors);
4492 }
4493 }
4494}
4495
4496template<class EventHandler>
4497void ParseEngine<EventHandler>::_handle_annotations_before_start_mapblck_as_key()
4498{
4499 _c4dbgp("annotations_before_start_mapblck_as_key");
4500 switch(m_pending_tags.num_entries)
4501 {
4502 case 1u:
4503 _c4dbgpf("annotations_after_start_mapblck_as_key: 1 tag={} line={} currline=", prs_(m_pending_tags.annotations[0].str), m_pending_tags.annotations[0].line, m_evt_handler->m_curr->pos.line);
4504 if(m_pending_tags.annotations[0].line != m_evt_handler->m_curr->pos.line)
4505 {
4506 _c4dbgp("annotations_after_start_mapblck_as_key: is map tag");
4507 m_evt_handler->set_key_tag(m_pending_tags.annotations[0].str);
4508 _clear_annotations(&m_pending_tags);
4509 }
4510 break;
4511 case 2u:
4512 _c4dbgpf("annotations_after_start_mapblck_as_key: 2 tags: {} -> {}", prs_(m_pending_tags.annotations[0].str), prs_(m_pending_tags.annotations[1].str));
4513 m_evt_handler->set_key_tag(m_pending_tags.annotations[0].str);
4514 break;
4515 }
4516 switch(m_pending_anchors.num_entries)
4517 {
4518 case 1u:
4519 _c4dbgpf("annotations_after_start_mapblck_as_key: 1 anchor={} line={} currline=", m_pending_anchors.annotations[0].str, m_pending_anchors.annotations[0].line, m_evt_handler->m_curr->pos.line);
4520 if(m_pending_anchors.annotations[0].line != m_evt_handler->m_curr->pos.line)
4521 {
4522 _c4dbgp("annotations_after_start_mapblck_as_key: is map anchor");
4523 m_evt_handler->set_key_anchor(m_pending_anchors.annotations[0].str);
4524 _clear_annotations(&m_pending_anchors);
4525 }
4526 break;
4527 case 2u:
4528 _c4dbgpf("annotations_after_start_mapblck_as_key: 2 anchors: {} -> {}", m_pending_anchors.annotations[0].str, m_pending_anchors.annotations[1].str);
4529 m_evt_handler->set_key_anchor(m_pending_anchors.annotations[0].str);
4530 break;
4531 }
4532}
4533
4534template<class EventHandler>
4535void ParseEngine<EventHandler>::_handle_annotations_and_indentation_after_start_mapblck(size_t key_indentation, size_t key_line)
4536{
4537 _c4dbgp("annotations_after_start_mapblck");
4538 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_pending_tags.num_entries <= 2, m_evt_handler->m_curr->pos);
4539 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_pending_anchors.num_entries <= 2, m_evt_handler->m_curr->pos);
4540 if(m_pending_anchors.num_entries || m_pending_tags.num_entries)
4541 {
4542 key_indentation = _select_indentation_from_annotations(key_indentation, key_line);
4543 switch(m_pending_tags.num_entries)
4544 {
4545 case 1u:
4546 _c4dbgpf("annotations_after_start_mapblck: 1 tag: {}", prs_(m_pending_tags.annotations[0].str));
4547 m_evt_handler->set_key_tag(m_pending_tags.annotations[0].str);
4548 _clear_annotations(&m_pending_tags);
4549 break;
4550 case 2u:
4551 _c4dbgpf("annotations_after_start_mapblck: 2 tags: {} -> {}", prs_(m_pending_tags.annotations[0].str), prs_(m_pending_tags.annotations[1].str));
4552 m_evt_handler->set_key_tag(m_pending_tags.annotations[1].str);
4553 _clear_annotations(&m_pending_tags);
4554 break;
4555 }
4556 switch(m_pending_anchors.num_entries)
4557 {
4558 case 1u:
4559 _c4dbgpf("annotations_after_start_mapblck: 1 anchors: {} -> {}", m_pending_anchors.annotations[0].str);
4560 m_evt_handler->set_key_anchor(m_pending_anchors.annotations[0].str);
4561 _clear_annotations(&m_pending_anchors);
4562 break;
4563 case 2u:
4564 _c4dbgpf("annotations_after_start_mapblck: 2 anchors: {} -> {}", m_pending_anchors.annotations[0].str, m_pending_anchors.annotations[1].str);
4565 m_evt_handler->set_key_anchor(m_pending_anchors.annotations[1].str);
4566 _clear_annotations(&m_pending_anchors);
4567 break;
4568 }
4569 }
4570 _set_indentation(key_indentation);
4571}
4572
4573template<class EventHandler>
4574size_t ParseEngine<EventHandler>::_select_indentation_from_annotations(size_t val_indentation, size_t val_line)
4575{
4576 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_pending_tags.num_entries | m_pending_anchors.num_entries, m_evt_handler->m_curr->pos);
4577 // select the left-most annotation on the max line
4578 auto const *C4_RESTRICT curr = m_pending_anchors.num_entries ? &m_pending_anchors.annotations[0] : &m_pending_tags.annotations[0];
4579 for(size_t i = 0; i < m_pending_anchors.num_entries; ++i)
4580 {
4581 auto const& C4_RESTRICT ann = m_pending_anchors.annotations[i];
4582 if(ann.line > curr->line)
4583 curr = &ann;
4584 else if(ann.indentation < curr->indentation)
4585 curr = &ann;
4586 }
4587 for(size_t j = 0; j < m_pending_tags.num_entries; ++j)
4588 {
4589 auto const& C4_RESTRICT ann = m_pending_tags.annotations[j];
4590 if(ann.line > curr->line)
4591 curr = &ann;
4592 else if(ann.indentation < curr->indentation)
4593 curr = &ann;
4594 }
4595 return curr->line < val_line ? val_indentation : curr->indentation;
4596}
4597
4598template<class EventHandler>
4599void ParseEngine<EventHandler>::_handle_keyref(csubstr alias)
4600{
4601 if C4_LIKELY(!(m_pending_anchors.num_entries | m_pending_tags.num_entries))
4602 m_evt_handler->set_key_ref(alias);
4603 else
4604 _c4err("aliases cannot have anchors or tags");
4605}
4606
4607template<class EventHandler>
4608void ParseEngine<EventHandler>::_handle_valref(csubstr alias)
4609{
4610 if C4_LIKELY(!(m_pending_anchors.num_entries | m_pending_tags.num_entries))
4611 m_evt_handler->set_val_ref(alias);
4612 else
4613 _c4err("aliases cannot have anchors or tags");
4614}
4615
4616template<class EventHandler>
4617csubstr ParseEngine<EventHandler>::_resolve_tag(csubstr tag)
4618{
4619 _c4dbgpf("resolving tag: {} curr_doc={}", prs_(tag), m_evt_handler->m_curr_doc);
4620 _c4assert(tag.is_sub(_buf()));
4621 TagCache::LookupResult ret = m_evt_handler->tag_cache().find(tag, m_evt_handler->m_curr_doc);
4622 if(ret)
4623 {
4624 _c4dbgpf("resolving tag: found in cache[{}]: {}", ret.pos, prs_(ret.resolved));
4625 return ret.resolved;
4626 }
4627 _c4dbgpf("resolving tag: not in cache: {} curr_doc={}", prs_(tag), m_evt_handler->m_curr_doc);
4628 size_t bufsz = 0;
4629 substr buf = m_evt_handler->arena_rem();
4630 TagDirectives const& C4_RESTRICT tds = m_evt_handler->tag_directives();
4631 csubstr ttag = tds.resolve(buf, &bufsz, tag, m_evt_handler->m_curr_doc,
4632 m_evt_handler->m_curr->pos,
4633 m_evt_handler->m_stack.m_callbacks);
4634 _c4dbgpf("resolving tag: bufsz={} ttag.len={} !!ttag.str={}", bufsz, ttag.len, !!ttag.str);
4635 _c4assert((bufsz > buf.len) == (!ttag.str));
4636 _c4assert(!!bufsz == (ttag.len == bufsz));
4637 // try again if the arena size was not enough
4638 if(!ttag.str)
4639 {
4640 _c4dbgpf("tag requires arena, but it was small. arena.len={} arena.slack={} tag.required={}", m_evt_handler->arena_rem().len, m_evt_handler->arena().len, ttag.len);
4641 _c4assert(ttag.len == bufsz);
4642 buf = _alloc_arena(bufsz, &tag);
4643 if(buf.str) // the alloc may fail eg with the ints handler
4644 {
4645 ttag = tds.resolve(buf, &bufsz, tag, m_evt_handler->m_curr_doc,
4646 m_evt_handler->m_curr->pos,
4647 m_evt_handler->m_stack.m_callbacks);
4648 }
4649 _c4assert(ttag.len == bufsz);
4650 _c4assert(!ttag.str || ttag.is_sub(m_evt_handler->arena()));
4651 }
4652 else if(bufsz) // if we succeeded writing into the arena, grow it as needed
4653 {
4654 _c4dbgp("tag required arena. update size");
4655 _c4assert(ttag.len == bufsz);
4656 _c4assert(ttag.is_sub(buf));
4657 (void)_alloc_arena(bufsz);
4658 }
4659 C4_SUPPRESS_WARNING_MSVC_WITH_PUSH(4127) // conditional expression is constant
4660 if C4_IF_CONSTEXPR (EventHandler::requires_strings_on_buffers) // NOLINT
4661 {
4662 _c4dbgpf("handler requires tags in buffers. !!ttag.str={} in_arena={} in_src={}", !!ttag.str, ttag.is_sub(m_evt_handler->arena()), ttag.is_sub(_buf()));
4663 // is the resolved tag not in any of those buffers?
4664 if(ttag.str && !ttag.is_sub(m_evt_handler->arena()) && !ttag.is_sub(_buf()))
4665 {
4666 _c4dbgpf("copying resolved tag to arena: slack={} required={}", m_evt_handler->arena_rem().len, ttag.len);
4667 buf = _alloc_arena(ttag.len, &tag);
4668 if(buf.str) // the alloc may fail eg with the ints handler
4669 memcpy(buf.str, ttag.str, ttag.len);
4670 ttag.str = buf.str; // keep the current len!
4671 _c4assert(!ttag.str || ttag.is_sub(m_evt_handler->arena()));
4672 }
4673 }
4674 C4_SUPPRESS_WARNING_MSVC_POP
4675 _c4dbgpf("resolved tag: {} --> [{}]~~~{}~~~", prs_(tag), ttag.len, maybe_null_str_(ttag));
4676 _c4assert(ttag.len > 0);
4677 // cache the hard-earned result!
4678 m_evt_handler->tag_cache().add(tag, ttag, m_evt_handler->m_curr_doc, ret.pos);
4679 return ttag;
4680}
4681
4682template<class EventHandler>
4683bool ParseEngine<EventHandler>::_validate_directive_yaml(csubstr *C4_RESTRICT directive, csubstr *C4_RESTRICT version) const
4684{
4685 _c4assert(directive->begins_with("%YAML"));
4686 size_t version_start = directive->first_not_of(" \t", 5);
4687 if(version_start != npos)
4688 {
4689 csubstr digits = "0123456789";
4690 size_t major_end = directive->first_not_of(digits, version_start);
4691 if(major_end != npos && directive->str[major_end] == '.') // single dot
4692 {
4693 size_t minor_end = directive->first_not_of(digits, major_end + 1);
4694 if(minor_end == npos)
4695 minor_end = directive->len;
4696 _set_first_strict(*directive, minor_end);
4697 *version = directive->range(version_start, minor_end);
4698 _c4dbgpf("%YAML: version={} full={}", *version, prs_(*directive, true));
4699 return true;
4700 }
4701 }
4702 return false;
4703}
4704
4705template<class EventHandler>
4706bool ParseEngine<EventHandler>::_validate_directive_tag(csubstr *C4_RESTRICT directive, csubstr *C4_RESTRICT handle, csubstr *C4_RESTRICT prefix) const
4707{
4708 _c4assert(directive->begins_with("%TAG"));
4709 csubstr whitespace = " \t";
4710 size_t handle_start = directive->first_not_of(whitespace, 4);
4711 if(handle_start != npos && directive->str[handle_start] == '!')
4712 {
4713 size_t handle_end = directive->first_of(whitespace, handle_start);
4714 if(handle_end != npos)
4715 {
4716 size_t prefix_start = directive->first_not_of(whitespace, handle_end);
4717 if(prefix_start != npos)
4718 {
4719 size_t prefix_end = directive->first_of(whitespace, prefix_start);
4720 if(prefix_end == npos)
4721 prefix_end = directive->len;
4722 _set_first_strict(*directive, prefix_end);
4723 *handle = directive->range(handle_start, handle_end);
4724 *prefix = directive->range(prefix_start, prefix_end);
4725 _c4dbgpf("%TAG: handle={} prefix={} full={}", *handle, *prefix, prs_(*directive, true));
4726 if(is_valid_tag_handle(*handle))
4727 return true;
4728 }
4729 }
4730 }
4731 return false;
4732}
4733
4734template<class EventHandler>
4735void ParseEngine<EventHandler>::_handle_directive(csubstr directive)
4736{
4737 _c4dbgpf("handle_directive: rem={}", prs_(directive, true));
4738 _c4assert(m_evt_handler->m_curr->line_contents.rem.begins_with('%'));
4739 _c4assert(directive.str == m_evt_handler->m_curr->line_contents.rem.str);
4740 const char *err = nullptr;
4741 csubstr rem;
4742 size_t pos;
4743 auto isdirective = [](csubstr str, csubstr dir) {
4744 if(str.begins_with(dir))
4745 {
4746 csubstr rest = str.sub(dir.len);
4747 return (!rest.len || rest.str[0] == ' ' || rest.str[0] == '\t');
4748 }
4749 return false;
4750 };
4751 if(isdirective(directive, "%TAG"))
4752 {
4753 csubstr handle;
4754 csubstr prefix;
4755 if C4_UNLIKELY(!_validate_directive_tag(&directive, &handle, &prefix))
4756 {
4757 err = "invalid %TAG directive";
4758 goto directive_error; // NOLINT
4759 }
4760 m_evt_handler->add_directive_tag(handle, prefix);
4761 }
4762 else if(isdirective(directive, "%YAML"))
4763 {
4764 csubstr version;
4765 if C4_UNLIKELY(!_validate_directive_yaml(&directive, &version))
4766 {
4767 err = "invalid %YAML directive";
4768 goto directive_error; // NOLINT
4769 }
4770 if C4_UNLIKELY(m_has_directives_yaml)
4771 {
4772 err = "multiple %YAML directives";
4773 goto directive_error; // NOLINT
4774 }
4775 m_has_directives_yaml = true;
4776 m_evt_handler->add_directive_yaml(version);
4777 }
4778 m_has_directives = true;
4779 rem = m_evt_handler->m_curr->line_contents.rem;
4780 pos = rem.first_not_of(" \t", directive.len);
4781 pos = pos != npos ? pos : rem.len;
4782 _line_progressed(pos);
4783 rem = rem.sub(pos);
4784 _c4dbgpf("handle_directive: rest={}", prs_(rem));
4785 if C4_UNLIKELY(rem.len && !rem.begins_with('#'))
4786 {
4787 err = "invalid tokens after directive";
4788 goto directive_error; // NOLINT
4789 }
4790directive_error:
4791 if C4_UNLIKELY(err != nullptr)
4792 _c4err(err);
4793}
4794
4795template<class EventHandler>
4796bool ParseEngine<EventHandler>::_handle_bom()
4797{
4798 const csubstr rem = m_evt_handler->m_curr->line_contents.rem;
4799 if(rem.len)
4800 {
4801 const csubstr rest = rem.sub(1);
4802 // https://yaml.org/spec/1.2.2/#52-character-encodings
4803 #define _rymlisascii(c) ((c) > '\0' && (c) <= '\x7f') // is the character ASCII?
4804 if(rem.begins_with(csubstr{"\x00\x00\xfe\xff", 4})
4805 // no bom:
4806 || (rem.begins_with(csubstr{"\x00\x00\x00", 3}) && rem.len >= 4u && _rymlisascii(rem.str[3])))
4807 {
4808 _c4err("UTF32BE not supported");
4809 }
4810 else if(rem.begins_with(csubstr{"\xff\xfe\x00\x00", 4})
4811 // no bom:
4812 || (rest.begins_with(csubstr{"\x00\x00\x00", 3}) && rem.len >= 4u && _rymlisascii(rem.str[0])))
4813 {
4814 _c4err("UTF32LE not supported");
4815 }
4816 else if(rem.begins_with("\xfe\xff") || (rem.begins_with('\x00') && rem.len >= 2u && _rymlisascii(rem.str[1])))
4817 {
4818 _c4err("UTF16BE not supported");
4819 }
4820 else if(rem.begins_with("\xff\xfe") || (rest.begins_with('\x00') && rem.len >= 2u && _rymlisascii(rem.str[0])))
4821 {
4822 _c4err("UTF16LE not supported");
4823 }
4824 else if(rem.begins_with("\xef\xbb\xbf"))
4825 {
4826 _c4dbgp("byte order mark: UTF8");
4827 _handle_bom(UTF8);
4828 _line_progressed(3);
4829 m_bom_len = 3;
4830 return true;
4831 }
4832 #undef _rymlisascii
4833 }
4834 return false;
4835}
4836
4837template<class EventHandler>
4838void ParseEngine<EventHandler>::_handle_bom(Encoding_e enc)
4839{
4840 if(m_encoding == NOBOM)
4841 {
4842 if(enc == UTF8 || /*beginning of file*/(m_evt_handler->m_curr->line_contents.rem.str == _buf().str))
4843 m_encoding = enc;
4844 else
4845 _c4err("non-UTF8 byte order mark can appear only at the beginning of the file"); // LCOV_EXCL_LINE
4846 }
4847 else if(enc != m_encoding)
4848 {
4849 _c4err("byte order mark can only be set once"); // LCOV_EXCL_LINE
4850 }
4851}
4852
4853
4854//-----------------------------------------------------------------------------
4855
4856template<class EventHandler>
4857void ParseEngine<EventHandler>::_handle_seq_json()
4858{
4859seqjson_start:
4860 _c4dbgpf("handle2_seq_json: node_id={} level={} indentation={}", m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
4861
4862 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
4863 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RSEQ), m_evt_handler->m_curr->pos);
4864 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RFLOW), m_evt_handler->m_curr->pos);
4865 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL|RNXT), m_evt_handler->m_curr->pos);
4866 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RVAL) != has_all(RNXT), m_evt_handler->m_curr->pos);
4867
4868 _handle_flow_skip_whitespace();
4869 csubstr rem = m_evt_handler->m_curr->line_contents.rem;
4870 if(!rem.len)
4871 goto seqjson_again;
4872
4873 if(has_any(RVAL))
4874 {
4875 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
4876 const char first = rem.str[0];
4877 _c4dbgpf("mapjson[RVAL]: '{}'", first);
4878 switch(first)
4879 {
4880 case '"':
4881 {
4882 _c4dbgp("seqjson[RVAL]: scanning double-quoted scalar");
4883 ScannedScalar sc = _scan_scalar_dquot();
4884 csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
4885 m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
4886 addrem_flags(RNXT, RVAL);
4887 break;
4888 }
4889 case '[':
4890 {
4891 _c4dbgp("seqjson[RVAL]: start child seqjson");
4892 addrem_flags(RNXT, RVAL);
4893 m_evt_handler->begin_seq_val_flow();
4894 addrem_flags(RVAL, RNXT);
4895 _line_progressed(1);
4896 break;
4897 }
4898 case '{':
4899 {
4900 _c4dbgp("seqjson[RVAL]: start child mapjson");
4901 addrem_flags(RNXT, RVAL);
4902 m_evt_handler->begin_map_val_flow();
4903 addrem_flags(RMAP|RKEY, RSEQ|RVAL|RNXT);
4904 _line_progressed(1);
4905 goto seqjson_finish;
4906 }
4907 case ']': // this happens on a trailing comma like ", ]"
4908 {
4909 _c4dbgp("seqjson[RVAL]: end!");
4910 rem_flags(RSEQ);
4911 _end_seq_flow();
4912 _line_progressed(1);
4913 if(!has_all(RSEQ|RFLOW))
4914 goto seqjson_finish;
4915 break;
4916 }
4917 default:
4918 {
4919 ScannedScalar sc;
4920 if(_scan_scalar_seq_json(&sc))
4921 {
4922 _c4dbgp("seqjson[RVAL]: it's a plain scalar.");
4923 csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref);
4924 m_evt_handler->set_val_scalar_plain(maybe_filtered);
4925 addrem_flags(RNXT, RVAL);
4926 }
4927 else
4928 {
4929 _c4err("parse error");
4930 }
4931 }
4932 }
4933 }
4934 else // RNXT
4935 {
4936 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RNXT), m_evt_handler->m_curr->pos);
4937 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
4938 const char first = rem.str[0];
4939 _c4dbgpf("mapjson[RNXT]: '{}'", first);
4940 switch(first)
4941 {
4942 case ',':
4943 {
4944 _c4dbgp("seqjson[RNXT]: expect next val");
4945 addrem_flags(RVAL, RNXT);
4946 m_evt_handler->add_sibling();
4947 _line_progressed(1);
4948 break;
4949 }
4950 case ']':
4951 {
4952 _c4dbgp("seqjson[RNXT]: end!");
4953 _end_seq_flow();
4954 _line_progressed(1);
4955 goto seqjson_finish;
4956 }
4957 default:
4958 _c4err("parse error");
4959 }
4960 }
4961
4962 seqjson_again:
4963 _c4dbgt("seqjson: go again", 0);
4964 if(_finished_line())
4965 {
4966 if C4_LIKELY(!_finished_file())
4967 {
4968 _line_ended();
4969 _scan_line();
4970 _c4dbgnextline();
4971 }
4972 else
4973 {
4974 _c4err("missing terminating ]");
4975 }
4976 }
4977 goto seqjson_start;
4978
4979 seqjson_finish:
4980 _c4dbgp("seqjson: finish");
4981}
4982
4983
4984//-----------------------------------------------------------------------------
4985
4986template<class EventHandler>
4987void ParseEngine<EventHandler>::_handle_map_json()
4988{
4989mapjson_start:
4990 _c4dbgpf("handle2_map_json: node_id={} level={} indentation={}", m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
4991
4992 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RMAP), m_evt_handler->m_curr->pos);
4993 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RFLOW), m_evt_handler->m_curr->pos);
4994 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
4995 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY|RKCL|RVAL|RNXT), m_evt_handler->m_curr->pos);
4996 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, 1 == (has_any(RKEY) + has_any(RKCL) + has_any(RVAL) + has_any(RNXT)), m_evt_handler->m_curr->pos);
4997
4998 _handle_flow_skip_whitespace();
4999 csubstr rem = m_evt_handler->m_curr->line_contents.rem;
5000 if(!rem.len)
5001 goto mapjson_again;
5002
5003 if(has_any(RKEY))
5004 {
5005 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5006 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5007 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5008 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5009 const char first = rem.str[0];
5010 _c4dbgpf("mapjson[RKEY]: '{}'", first);
5011 switch(first)
5012 {
5013 case '"':
5014 {
5015 _c4dbgp("mapjson[RKEY]: scanning double-quoted scalar");
5016 ScannedScalar sc = _scan_scalar_dquot();
5017 csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc);
5018 m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
5019 addrem_flags(RKCL, RKEY);
5020 break;
5021 }
5022 case '}': // this happens on a trailing comma like ", }"
5023 {
5024 _c4dbgp("mapjson[RKEY]: end!");
5025 _end_map_flow();
5026 _line_progressed(1);
5027 goto mapjson_finish;
5028 }
5029 default:
5030 _c4err("parse error");
5031 }
5032 }
5033 else if(has_any(RVAL))
5034 {
5035 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5036 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5037 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5038 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5039 const char first = rem.str[0];
5040 _c4dbgpf("mapjson[RVAL]: '{}'", first);
5041 switch(first)
5042 {
5043 case '"':
5044 {
5045 _c4dbgp("mapjson[RVAL]: scanning double-quoted scalar");
5046 ScannedScalar sc = _scan_scalar_dquot();
5047 csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
5048 m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
5049 addrem_flags(RNXT, RVAL);
5050 break;
5051 }
5052 case '[':
5053 {
5054 _c4dbgp("mapjson[RVAL]: start val seqjson");
5055 addrem_flags(RNXT, RVAL);
5056 m_evt_handler->begin_seq_val_flow();
5057 _set_indentation(m_evt_handler->m_parent->indref);
5058 addrem_flags(RSEQ|RVAL, RMAP|RNXT);
5059 _line_progressed(1);
5060 goto mapjson_finish;
5061 }
5062 case '{':
5063 {
5064 _c4dbgp("mapjson[RVAL]: start val mapjson");
5065 addrem_flags(RNXT, RVAL);
5066 m_evt_handler->begin_map_val_flow();
5067 _set_indentation(m_evt_handler->m_parent->indref);
5068 addrem_flags(RKEY, RNXT);
5069 _line_progressed(1);
5070 // keep going in this function
5071 break;
5072 }
5073 default:
5074 {
5075 ScannedScalar sc;
5076 if(_scan_scalar_map_json(&sc))
5077 {
5078 _c4dbgp("mapjson[RVAL]: plain scalar.");
5079 csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref);
5080 m_evt_handler->set_val_scalar_plain(maybe_filtered);
5081 addrem_flags(RNXT, RVAL);
5082 }
5083 else
5084 {
5085 _c4err("parse error");
5086 }
5087 break;
5088 }
5089 }
5090 }
5091 else if(has_any(RKCL)) // read the key colon
5092 {
5093 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5094 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5095 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5096 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5097 const char first = rem.str[0];
5098 _c4dbgpf("mapjson[RKCL]: '{}'", first);
5099 if(first == ':')
5100 {
5101 _c4dbgp("mapjson[RKCL]: found the colon");
5102 addrem_flags(RVAL, RKCL);
5103 _line_progressed(1);
5104 }
5105 else
5106 {
5107 _c4err("parse error");
5108 }
5109 }
5110 else if(has_any(RNXT))
5111 {
5112 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5113 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5114 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5115 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5116 _c4dbgpf("mapjson[RNXT]: '{}'", rem.str[0]);
5117 if(rem.begins_with(','))
5118 {
5119 _c4dbgp("mapjson[RNXT]: expect next keyval");
5120 m_evt_handler->add_sibling();
5121 addrem_flags(RKEY, RNXT);
5122 _line_progressed(1);
5123 }
5124 else if(rem.begins_with('}'))
5125 {
5126 _c4dbgp("mapjson[RNXT]: end!");
5127 _end_map_flow();
5128 _line_progressed(1);
5129 goto mapjson_finish;
5130 }
5131 else
5132 {
5133 _c4err("parse error"); // LCOV_EXCL_LINE
5134 }
5135 }
5136
5137 mapjson_again:
5138 _c4dbgt("mapjson: go again", 0);
5139 if(_finished_line())
5140 {
5141 if C4_LIKELY(!_finished_file())
5142 {
5143 _line_ended();
5144 _scan_line();
5145 _c4dbgnextline();
5146 }
5147 else
5148 {
5149 _c4err("missing terminating }");
5150 }
5151 }
5152 goto mapjson_start;
5153
5154 mapjson_finish:
5155 _c4dbgp("mapjson: finish");
5156}
5157
5158
5159//-----------------------------------------------------------------------------
5160
5161template<class EventHandler>
5162void ParseEngine<EventHandler>::_handle_seq_imap()
5163{
5164seqimap_start:
5165 _c4dbgpf("handle2_seq_imap: node_id={} level={} indref={}", m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
5166
5167 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RSEQIMAP), m_evt_handler->m_curr->pos);
5168 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5169 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL|RNXT|QMRK|RKCL), m_evt_handler->m_curr->pos);
5170 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, 1 == has_all(RVAL) + has_all(RNXT) + has_all(QMRK) + has_all(RKCL), m_evt_handler->m_curr->pos);
5171 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_stack.size() >= 3, m_evt_handler->m_curr->pos);
5172
5173 _handle_flow_skip_whitespace();
5174 csubstr rem = m_evt_handler->m_curr->line_contents.rem;
5175 if(!rem.len)
5176 goto seqimap_again;
5177
5178 if(has_any(RVAL))
5179 {
5180 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL), m_evt_handler->m_curr->pos);
5181 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5182 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5183 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5184 const char first = rem.str[0];
5185 _c4dbgpf("seqimap[RVAL]: '{}'", _c4prc(first));
5186 ScannedScalar sc;
5187 if(first == '\'')
5188 {
5189 _c4dbgp("seqimap[RVAL]: scanning single-quoted scalar");
5190 sc = _scan_scalar_squot();
5191 csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
5192 _handle_annotations_before_blck_val_scalar();
5193 m_evt_handler->set_val_scalar_squoted(maybe_filtered);
5194 _end_map_flow();
5195 goto seqimap_finish;
5196 }
5197 else if(first == '"')
5198 {
5199 _c4dbgp("seqimap[RVAL]: scanning double-quoted scalar");
5200 sc = _scan_scalar_dquot();
5201 csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
5202 _handle_annotations_before_blck_val_scalar();
5203 m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
5204 _end_map_flow();
5205 goto seqimap_finish;
5206 }
5207 // block scalars (ie | and >) cannot appear in flow containers
5208 else if(_scan_scalar_plain_map_flow(&sc))
5209 {
5210 _c4dbgp("seqimap[RVAL]: it's a scalar.");
5211 csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref);
5212 _handle_annotations_before_blck_val_scalar();
5213 m_evt_handler->set_val_scalar_plain(maybe_filtered);
5214 _end_map_flow();
5215 goto seqimap_finish;
5216 }
5217 else if(first == '[')
5218 {
5219 _c4dbgp("seqimap[RVAL]: start child seqflow");
5220 addrem_flags(RNXT, RVAL);
5221 _handle_annotations_before_blck_val_scalar();
5222 m_evt_handler->begin_seq_val_flow();
5223 addrem_flags(RVAL, RNXT|RSEQIMAP);
5224 _set_indentation(m_evt_handler->m_parent->indref);
5225 _line_progressed(1);
5226 goto seqimap_finish;
5227 }
5228 else if(first == '{')
5229 {
5230 _c4dbgp("seqimap[RVAL]: start child mapflow");
5231 addrem_flags(RNXT, RVAL);
5232 _handle_annotations_before_blck_val_scalar();
5233 m_evt_handler->begin_map_val_flow();
5234 addrem_flags(RMAP|RKEY, RSEQ|RVAL|RSEQIMAP|RNXT);
5235 _set_indentation(m_evt_handler->m_parent->indref);
5236 _line_progressed(1);
5237 goto seqimap_finish;
5238 }
5239 else if(first == ',' || first == ']')
5240 {
5241 _c4dbgp("seqimap[RVAL]: finish without val.");
5242 _handle_annotations_before_blck_val_scalar();
5243 m_evt_handler->set_val_scalar_plain_empty();
5244 _end_map_flow();
5245 goto seqimap_finish;
5246 }
5247 else if(first == '*')
5248 {
5249 csubstr ref = _scan_ref_seq();
5250 _c4dbgpf("seqimap[RVAL]: ref! {}", prs_(ref));
5251 _handle_valref(ref);
5252 addrem_flags(RNXT, RVAL);
5253 }
5254 else if(first == '&')
5255 {
5256 csubstr anchor = _scan_anchor();
5257 _c4dbgpf("seqimap[RVAL]: anchor! {}", prs_(anchor));
5258 _add_annotation(&m_pending_anchors, anchor);
5259 }
5260 else if(first == '!')
5261 {
5262 csubstr tag = _scan_tag();
5263 _c4dbgpf("seqimap[RVAL]: tag! {}", prs_(tag));
5264 _add_annotation(&m_pending_tags, tag);
5265 }
5266 else
5267 {
5268 _c4err("parse error"); // LCOV_EXCL_LINE
5269 }
5270 }
5271 else if(has_any(RNXT))
5272 {
5273 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RNXT), m_evt_handler->m_curr->pos);
5274 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5275 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5276 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5277 const char first = rem.str[0];
5278 _c4dbgpf("seqimap[RNXT]: '{}'", _c4prc(first));
5279 if(first == ',' || first == ']')
5280 {
5281 // we may get here because a map or a seq started and we
5282 // return later
5283 _c4dbgp("seqimap: done");
5284 _end_map_flow();
5285 goto seqimap_finish;
5286 }
5287 else
5288 {
5289 _c4err("parse error"); // LCOV_EXCL_LINE
5290 }
5291 }
5292 else if(has_any(QMRK))
5293 {
5294 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(QMRK), m_evt_handler->m_curr->pos);
5295 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5296 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5297 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5298 const char first = rem.str[0];
5299 _c4dbgpf("seqimap[QMRK]: '{}'", _c4prc(first));
5300 ScannedScalar sc;
5301 if(first == '\'')
5302 {
5303 _c4dbgp("seqimap[QMRK]: scanning single-quoted scalar");
5304 sc = _scan_scalar_squot();
5305 csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc);
5306 m_evt_handler->set_key_scalar_squoted(maybe_filtered);
5307 addrem_flags(RKCL, QMRK);
5308 goto seqimap_again;
5309 }
5310 else if(first == '"')
5311 {
5312 _c4dbgp("seqimap[QMRK]: scanning double-quoted scalar");
5313 sc = _scan_scalar_dquot();
5314 csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc);
5315 m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
5316 addrem_flags(RKCL, QMRK);
5317 goto seqimap_again;
5318 }
5319 // block scalars (ie | and >) cannot appear in flow containers
5320 else if(_scan_scalar_plain_map_flow(&sc))
5321 {
5322 _c4dbgp("seqimap[QMRK]: it's a scalar.");
5323 csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, m_evt_handler->m_curr->indref);
5324 m_evt_handler->set_key_scalar_plain(maybe_filtered);
5325 addrem_flags(RKCL, QMRK);
5326 goto seqimap_again;
5327 }
5328 else if(first == '[')
5329 {
5330 _c4dbgp("seqimap[QMRK]: start child seqflow");
5331 addrem_flags(RKCL, QMRK);
5332 m_evt_handler->begin_seq_key_flow();
5333 addrem_flags(RSEQ|RVAL, RKCL|RSEQIMAP);
5334 _set_indentation(m_evt_handler->m_parent->indref);
5335 _line_progressed(1);
5336 goto seqimap_finish;
5337 }
5338 else if(first == '{')
5339 {
5340 _c4dbgp("seqimap[QMRK]: start child mapflow");
5341 addrem_flags(RKCL, QMRK);
5342 m_evt_handler->begin_map_key_flow();
5343 addrem_flags(RMAP|RKEY, RSEQ|RKCL|RSEQIMAP);
5344 _set_indentation(m_evt_handler->m_parent->indref);
5345 _line_progressed(1);
5346 goto seqimap_finish;
5347 }
5348 else if(first == ',' || first == ']')
5349 {
5350 _c4dbgp("seqimap[QMRK]: finish without key.");
5351 m_evt_handler->set_key_scalar_plain_empty();
5352 m_evt_handler->set_val_scalar_plain_empty();
5353 _end_map_flow();
5354 goto seqimap_finish;
5355 }
5356 else if(first == '&')
5357 {
5358 csubstr anchor = _scan_anchor();
5359 _c4dbgp("seqimap[QMRK]: anchor!");
5360 m_evt_handler->set_key_anchor(anchor);
5361 }
5362 else if(first == '*')
5363 {
5364 csubstr ref = _scan_ref_seq();
5365 _c4dbgp("seqimap[QMRK]: ref!");
5366 _handle_keyref(ref);
5367 addrem_flags(RKCL, QMRK);
5368 }
5369 else
5370 {
5371 _c4err("parse error"); // LCOV_EXCL_LINE
5372 }
5373 }
5374 else if(has_any(RKCL))
5375 {
5376 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5377 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5378 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5379 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKCL), m_evt_handler->m_curr->pos);
5380 const char first = rem.str[0];
5381 _c4dbgpf("seqimap[RKCL]: '{}'", _c4prc(first));
5382 if(first == ':')
5383 {
5384 _c4dbgp("seqimap[RKCL]: found ':'");
5385 addrem_flags(RVAL, RKCL);
5386 _line_progressed(1);
5387 goto seqimap_again;
5388 }
5389 else if(first == ',' || first == ']')
5390 {
5391 _c4dbgp("seqimap[RKCL]: found ','. finish without val");
5392 m_evt_handler->set_val_scalar_plain_empty();
5393 _end_map_flow();
5394 goto seqimap_finish;
5395 }
5396 else
5397 {
5398 _c4err("parse error"); // LCOV_EXCL_LINE
5399 }
5400 }
5401
5402 seqimap_again:
5403 _c4dbgt("seqimap: go again", 0);
5404 if(_finished_line())
5405 {
5406 if C4_LIKELY(!_finished_file())
5407 {
5408 _line_ended();
5409 _scan_line();
5410 _c4dbgnextline();
5411 }
5412 else
5413 {
5414 _c4err("parse error");
5415 }
5416 }
5417 goto seqimap_start;
5418
5419 seqimap_finish:
5420 _c4dbgp("seqimap: finish");
5421}
5422
5423
5424//-----------------------------------------------------------------------------
5425
5426template<class EventHandler>
5427void ParseEngine<EventHandler>::_handle_seq_flow()
5428{
5429seqflow_start:
5430 _c4dbgpf("handle_seq_flow: node_id={} level={} indentation={}", m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
5431
5432 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5433 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RSEQ), m_evt_handler->m_curr->pos);
5434 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RFLOW), m_evt_handler->m_curr->pos);
5435 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL|RNXT), m_evt_handler->m_curr->pos);
5436 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RVAL) != has_all(RNXT), m_evt_handler->m_curr->pos);
5437 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indref != npos, m_evt_handler->m_curr->pos);
5438
5439 if(m_evt_handler->m_curr->at_line_beginning())
5440 {
5441 _handle_flow_line_beginning();
5442 }
5443
5444 _handle_flow_skip_whitespace();
5445 if(!m_evt_handler->m_curr->line_contents.rem.len)
5446 goto seqflow_again;
5447
5448 if(has_any(RVAL))
5449 {
5450 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5451 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
5452 ScannedScalar sc;
5453 if(first == '\'')
5454 {
5455 _c4dbgp("seqflow[RVAL]: scanning single-quoted scalar");
5456 sc = _scan_scalar_squot();
5457 csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
5458 _handle_annotations_before_blck_val_scalar();
5459 m_evt_handler->set_val_scalar_squoted(maybe_filtered);
5460 addrem_flags(RNXT, RVAL);
5461 _mark_seqflow_val_end();
5462 }
5463 else if(first == '"')
5464 {
5465 _c4dbgp("seqflow[RVAL]: scanning double-quoted scalar");
5466 sc = _scan_scalar_dquot();
5467 csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
5468 _handle_annotations_before_blck_val_scalar();
5469 m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
5470 addrem_flags(RNXT, RVAL);
5471 _mark_seqflow_val_end();
5472 }
5473 // block scalars (ie | and >) cannot appear in flow containers
5474 else if(_scan_scalar_plain_seq_flow(&sc))
5475 {
5476 _c4dbgp("seqflow[RVAL]: it's a scalar.");
5477 csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref);
5478 _handle_annotations_before_blck_val_scalar();
5479 m_evt_handler->set_val_scalar_plain(maybe_filtered);
5480 addrem_flags(RNXT, RVAL);
5481 _mark_seqflow_val_end();
5482 }
5483 else if(first == '[')
5484 {
5485 _c4dbgp("seqflow[RVAL]: start child seqflow");
5486 addrem_flags(RNXT, RVAL);
5487 _handle_annotations_before_blck_val_scalar();
5488 m_evt_handler->begin_seq_val_flow();
5489 _set_indentation(m_evt_handler->m_parent->indref);
5490 addrem_flags(RVAL, RNXT);
5491 _line_progressed(1);
5492 }
5493 else if(first == '{')
5494 {
5495 _c4dbgp("seqflow[RVAL]: start child mapflow");
5496 addrem_flags(RNXT, RVAL);
5497 _handle_annotations_before_blck_val_scalar();
5498 m_evt_handler->begin_map_val_flow();
5499 _set_indentation(m_evt_handler->m_parent->indref);
5500 addrem_flags(RMAP|RKEY, RSEQ|RVAL|RNXT);
5501 _line_progressed(1);
5502 goto seqflow_finish;
5503 }
5504 else if(first == ']') // this happens on cases such as [] or [.., ]
5505 {
5506 _c4dbgp("seqflow[RVAL]: end!");
5507 if(m_pending_anchors.num_entries | m_pending_tags.num_entries)
5508 {
5509 _c4dbgp("seqflow[RVAL]: add pending annotations");
5510 _handle_annotations_before_blck_val_scalar();
5511 m_evt_handler->set_val_scalar_plain_empty();
5512 }
5513 _line_progressed(1);
5514 _end_seq_flow();
5515 goto seqflow_finish;
5516 }
5517 else if(first == '*')
5518 {
5519 csubstr ref = _scan_ref_seq();
5520 _c4dbgpf("seqflow[RVAL]: ref! {}", prs_(ref));
5521 _handle_valref(ref);
5522 addrem_flags(RNXT, RVAL);
5523 }
5524 else if(first == '&')
5525 {
5526 csubstr anchor = _scan_anchor();
5527 _c4dbgpf("seqflow[RVAL]: anchor! {}", prs_(anchor));
5528 _add_annotation(&m_pending_anchors, anchor);
5529 }
5530 else if(first == '!')
5531 {
5532 csubstr tag = _scan_tag();
5533 _c4dbgpf("seqflow[RVAL]: tag! {}", prs_(tag));
5534 _add_annotation(&m_pending_tags, tag);
5535 }
5536 else if(first == ':')
5537 {
5538 _c4dbgpf("seqflow[RVAL]: actually seqimap at node[{}], with empty key", m_evt_handler->m_curr->node_id);
5539 addrem_flags(RNXT, RVAL);
5540 m_evt_handler->begin_map_val_flow();
5541 _set_indentation(m_evt_handler->m_parent->indref);
5542 _handle_annotations_before_blck_key_scalar();
5543 m_evt_handler->set_key_scalar_plain_empty();
5544 addrem_flags(RSEQIMAP|RVAL, RSEQ|RNXT);
5545 _line_progressed(1);
5546 goto seqflow_finish;
5547 }
5548 else if(first == '?')
5549 {
5550 _c4dbgp("seqflow[RVAL]: start child mapflow, explicit key");
5551 addrem_flags(RNXT, RVAL);
5552 m_evt_handler->begin_map_val_flow();
5553 _set_indentation(m_evt_handler->m_parent->indref);
5554 addrem_flags(RSEQIMAP|QMRK, RSEQ|RNXT);
5555 _line_progressed(1);
5556 _maybe_skip_whitespace_tokens();
5557 goto seqflow_finish;
5558 }
5559 else if(first == ',')
5560 {
5561 if(m_pending_anchors.num_entries || m_pending_tags.num_entries)
5562 {
5563 _c4dbgp("seqflow[RVAL]: add pending annotations");
5564 _handle_annotations_before_blck_val_scalar();
5565 m_evt_handler->set_val_scalar_plain_empty();
5566 addrem_flags(RNXT, RVAL);
5567 _mark_seqflow_val_end();
5568 }
5569 else
5570 {
5571 _c4err("parse error");
5572 }
5573 }
5574 else
5575 {
5576 _c4err("parse error");
5577 }
5578 }
5579 else // RNXT
5580 {
5581 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RNXT), m_evt_handler->m_curr->pos);
5582 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5583 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
5584 if(first == ',')
5585 {
5586 _c4dbgp("seqflow[RNXT]: expect next val");
5587 addrem_flags(RVAL, RNXT);
5588 m_evt_handler->add_sibling();
5589 _line_progressed(1);
5590 if(m_evt_handler->m_curr->line_contents.rem.begins_with('#'))
5591 {
5592 _c4err("parse error: invalid comment after comma");
5593 }
5594 _mark_seqflow_val_end();
5595 }
5596 else if(first == ']')
5597 {
5598 _c4dbgp("seqflow[RNXT]: end!");
5599 _line_progressed(1);
5600 _end_seq_flow();
5601 goto seqflow_finish;
5602 }
5603 else if(first == ':')
5604 {
5605 _c4dbgpf("seqflow[RNXT]: line@valend={} line@now={}", m_prev_val_end, m_evt_handler->m_curr->pos.line);
5606 if(m_prev_val_end != NONE && m_evt_handler->m_curr->pos.line == m_prev_val_end)
5607 {
5608 _c4dbgpf("seqflow[RNXT]: actually seqimap at node[{}]", m_evt_handler->m_curr->node_id);
5609 m_evt_handler->actually_val_is_first_key_of_new_map_flow();
5610 _set_indentation(m_evt_handler->m_parent->indref);
5611 _line_progressed(1);
5612 addrem_flags(RSEQIMAP|RVAL, RNXT);
5613 goto seqflow_finish;
5614 }
5615 else
5616 {
5617 _c4err("parse error");
5618 }
5619 }
5620 else
5621 {
5622 _c4err("parse error");
5623 }
5624 }
5625
5626 seqflow_again:
5627 _c4dbgt("seqflow: go again", 0);
5628 if(_finished_line())
5629 {
5630 if C4_LIKELY(!_finished_file())
5631 {
5632 _line_ended();
5633 _scan_line();
5634 _c4dbgnextline();
5635 }
5636 else
5637 {
5638 _c4err("missing terminating ]");
5639 }
5640 }
5641 goto seqflow_start;
5642
5643 seqflow_finish:
5644 _c4dbgp("seqflow: finish");
5645}
5646
5647
5648//-----------------------------------------------------------------------------
5649
5650template<class EventHandler>
5651void ParseEngine<EventHandler>::_handle_map_flow()
5652{
5653mapflow_start:
5654 _c4dbgpf("handle_map_flow: node_id={} level={} indentation={}", m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
5655
5656 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RMAP), m_evt_handler->m_curr->pos);
5657 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RFLOW), m_evt_handler->m_curr->pos);
5658 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY|RKCL|RVAL|RNXT|QMRK), m_evt_handler->m_curr->pos);
5659 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, 1 == (has_any(RKEY) + has_any(RKCL) + has_any(RVAL) + has_any(RNXT) + has_any(QMRK)), m_evt_handler->m_curr->pos);
5660
5661 if(m_evt_handler->m_curr->at_line_beginning())
5662 {
5663 _handle_flow_line_beginning();
5664 }
5665
5666 _handle_flow_skip_whitespace();
5667 if(!m_evt_handler->m_curr->line_contents.rem.len)
5668 goto mapflow_again;
5669
5670 if(has_any(RKEY))
5671 {
5672 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5673 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5674 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5675 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5676 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
5677 _c4dbgpf("mapflow[RKEY]: '{}'", first);
5678 ScannedScalar sc;
5679 if(first == '\'')
5680 {
5681 _c4dbgp("mapflow[RKEY]: scanning single-quoted scalar");
5682 sc = _scan_scalar_squot();
5683 csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc);
5684 _handle_annotations_before_blck_key_scalar();
5685 m_evt_handler->set_key_scalar_squoted(maybe_filtered);
5686 addrem_flags(RKCL, RKEY|QMRK);
5687 }
5688 else if(first == '"')
5689 {
5690 _c4dbgp("mapflow[RKEY]: scanning double-quoted scalar");
5691 sc = _scan_scalar_dquot();
5692 csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc);
5693 _handle_annotations_before_blck_key_scalar();
5694 m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
5695 addrem_flags(RKCL, RKEY|QMRK);
5696 }
5697 // block scalars (ie | and >) cannot appear in flow containers
5698 else if(_scan_scalar_plain_map_flow(&sc))
5699 {
5700 _c4dbgp("mapflow[RKEY]: plain scalar");
5701 csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, m_evt_handler->m_curr->indref);
5702 _handle_annotations_before_blck_key_scalar();
5703 m_evt_handler->set_key_scalar_plain(maybe_filtered);
5704 addrem_flags(RKCL, RKEY|QMRK);
5705 }
5706 else if(first == '?')
5707 {
5708 _c4dbgp("mapflow[RKEY]: explicit key");
5709 _handle_annotations_before_blck_key_scalar();
5710 addrem_flags(QMRK, RKEY);
5711 _line_progressed(1);
5712 _maybe_skip_whitespace_tokens();
5713 }
5714 else if(first == ':')
5715 {
5716 _c4dbgp("mapflow[RKEY]: setting empty key");
5717 _handle_annotations_before_blck_key_scalar();
5718 m_evt_handler->set_key_scalar_plain_empty();
5719 addrem_flags(RVAL, RKEY|QMRK);
5720 _line_progressed(1);
5721 _maybe_skip_whitespace_tokens();
5722 }
5723 else if(first == ',')
5724 {
5725 _c4dbgp("mapflow[RKEY]: comma!");
5726 if(!_handle_annotations_before_unexpected_flow_token_rkey())
5727 _c4err("unexpected comma");
5728 addrem_flags(RNXT, RKEY|QMRK);
5729 // keep going in this function
5730 }
5731 else if(first == '}') // this happens on a trailing comma like ", }"
5732 {
5733 _c4dbgp("mapflow[RKEY]: end!");
5734 (void)_handle_annotations_before_unexpected_flow_token_rkey();
5735 _line_progressed(1);
5736 _end_map_flow();
5737 goto mapflow_finish;
5738 }
5739 else if(first == '&')
5740 {
5741 csubstr anchor = _scan_anchor();
5742 _c4dbgpf("mapflow[RKEY]: key anchor! {}", prs_(anchor));
5743 _add_annotation(&m_pending_anchors, anchor);
5744 }
5745 else if(first == '!')
5746 {
5747 csubstr tag = _scan_tag();
5748 _c4dbgpf("mapflow[RKEY]: tag! {}", prs_(tag));
5749 _add_annotation(&m_pending_tags, tag);
5750 }
5751 else if(first == '*')
5752 {
5753 csubstr ref = _scan_ref_map();
5754 _c4dbgpf("mapflow[RKEY]: key ref! {}", prs_(ref));
5755 _handle_keyref(ref);
5756 addrem_flags(RKCL, RKEY);
5757 }
5758 else if(first == '[')
5759 {
5760 // RYML's tree cannot store container keys, but that's
5761 // handled inside the tree event handler. Other handler
5762 // types may be able to handle it.
5763 _c4dbgp("mapflow[RKEY]: start child seqflow (!)");
5764 _handle_annotations_before_blck_key_scalar();
5765 addrem_flags(RKCL, RKEY);
5766 m_evt_handler->begin_seq_key_flow();
5767 addrem_flags(RSEQ|RVAL, RMAP|RKCL);
5768 _set_indentation(m_evt_handler->m_parent->indref);
5769 _line_progressed(1);
5770 goto mapflow_finish;
5771 }
5772 else if(first == '{')
5773 {
5774 // RYML's tree cannot store container keys, but that's
5775 // handled inside the tree event handler. Other handler
5776 // types may be able to handle it.
5777 _c4dbgp("mapflow[RKEY]: start child mapflow (!)");
5778 _handle_annotations_before_blck_key_scalar();
5779 addrem_flags(RKCL, RKEY);
5780 m_evt_handler->begin_map_key_flow();
5781 addrem_flags(RKEY, RVAL|RKCL);
5782 _set_indentation(m_evt_handler->m_parent->indref);
5783 _line_progressed(1);
5784 // keep going in this function
5785 }
5786 else
5787 {
5788 _c4err("parse error"); // LCOV_EXCL_LINE
5789 }
5790 }
5791 else if(has_any(RKCL)) // read the key colon
5792 {
5793 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5794 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5795 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5796 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5797 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
5798 _c4dbgpf("mapflow[RKCL]: '{}'", first);
5799 if(first == ':')
5800 {
5801 _c4dbgp("mapflow[RKCL]: found the colon");
5802 addrem_flags(RVAL, RKCL);
5803 _line_progressed(1);
5804 }
5805 else if(first == '}')
5806 {
5807 _c4dbgp("mapflow[RKCL]: end with missing val!");
5808 addrem_flags(RVAL, RKCL);
5809 m_evt_handler->set_val_scalar_plain_empty();
5810 _line_progressed(1);
5811 _end_map_flow();
5812 goto mapflow_finish;
5813 }
5814 else if(first == ',')
5815 {
5816 _c4dbgp("mapflow[RKCL]: got comma. val is missing");
5817 m_evt_handler->set_val_scalar_plain_empty();
5818 m_evt_handler->add_sibling();
5819 addrem_flags(RKEY, RKCL);
5820 _line_progressed(1);
5821 if(m_evt_handler->m_curr->line_contents.rem.begins_with('#'))
5822 {
5823 _c4err("parse error: invalid comment after comma");
5824 }
5825 }
5826 else
5827 {
5828 _c4err("parse error");
5829 }
5830 }
5831 else if(has_any(RVAL))
5832 {
5833 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5834 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5835 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5836 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5837 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
5838 _c4dbgpf("mapflow[RVAL]: '{}'", first);
5839 ScannedScalar sc;
5840 if(first == '\'')
5841 {
5842 _c4dbgp("mapflow[RVAL]: scanning single-quoted scalar");
5843 sc = _scan_scalar_squot();
5844 csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
5845 _handle_annotations_before_blck_val_scalar();
5846 m_evt_handler->set_val_scalar_squoted(maybe_filtered);
5847 addrem_flags(RNXT, RVAL);
5848 }
5849 else if(first == '"')
5850 {
5851 _c4dbgp("mapflow[RVAL]: scanning double-quoted scalar");
5852 sc = _scan_scalar_dquot();
5853 csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
5854 _handle_annotations_before_blck_val_scalar();
5855 m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
5856 addrem_flags(RNXT, RVAL);
5857 }
5858 // block scalars (ie | and >) cannot appear in flow containers
5859 else if(_scan_scalar_plain_map_flow(&sc))
5860 {
5861 _c4dbgp("mapflow[RVAL]: plain scalar.");
5862 csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref);
5863 _handle_annotations_before_blck_val_scalar();
5864 m_evt_handler->set_val_scalar_plain(maybe_filtered);
5865 addrem_flags(RNXT, RVAL);
5866 }
5867 else if(first == '[')
5868 {
5869 _c4dbgp("mapflow[RVAL]: start val seqflow");
5870 addrem_flags(RNXT, RVAL);
5871 _handle_annotations_before_blck_val_scalar();
5872 m_evt_handler->begin_seq_val_flow();
5873 _set_indentation(m_evt_handler->m_parent->indref);
5874 addrem_flags(RSEQ|RVAL, RMAP|RNXT);
5875 _line_progressed(1);
5876 goto mapflow_finish;
5877 }
5878 else if(first == '{')
5879 {
5880 _c4dbgp("mapflow[RVAL]: start val mapflow");
5881 addrem_flags(RNXT, RVAL);
5882 _handle_annotations_before_blck_val_scalar();
5883 m_evt_handler->begin_map_val_flow();
5884 _set_indentation(m_evt_handler->m_parent->indref);
5885 addrem_flags(RKEY, RNXT);
5886 _line_progressed(1);
5887 // keep going in this function
5888 }
5889 else if(first == '}')
5890 {
5891 _c4dbgp("mapflow[RVAL]: end!");
5892 _handle_annotations_before_blck_val_scalar();
5893 m_evt_handler->set_val_scalar_plain_empty();
5894 _line_progressed(1);
5895 _end_map_flow();
5896 goto mapflow_finish;
5897 }
5898 else if(first == ',')
5899 {
5900 _c4dbgp("mapflow[RVAL]: empty val!");
5901 _handle_annotations_before_blck_val_scalar();
5902 m_evt_handler->set_val_scalar_plain_empty();
5903 addrem_flags(RNXT, RVAL);
5904 // keep going in this function
5905 }
5906 else if(first == '*')
5907 {
5908 csubstr ref = _scan_ref_map();
5909 _c4dbgpf("mapflow[RVAL]: key ref! {}", prs_(ref));
5910 _handle_valref(ref);
5911 addrem_flags(RNXT, RVAL);
5912 }
5913 else if(first == '&')
5914 {
5915 csubstr anchor = _scan_anchor();
5916 _c4dbgpf("mapflow[RVAL]: key anchor! {}", prs_(anchor));
5917 _add_annotation(&m_pending_anchors, anchor);
5918 }
5919 else if(first == '!')
5920 {
5921 csubstr tag = _scan_tag();
5922 _c4dbgpf("mapflow[RVAL]: tag! {}", prs_(tag));
5923 _add_annotation(&m_pending_tags, tag);
5924 }
5925 else
5926 {
5927 _c4err("parse error");
5928 }
5929 }
5930 else if(has_any(RNXT))
5931 {
5932 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5933 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5934 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5935 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
5936 _c4dbgpf("mapflow[RNXT]: '{}'", m_evt_handler->m_curr->line_contents.rem.str[0]);
5937 if(m_evt_handler->m_curr->line_contents.rem.begins_with(','))
5938 {
5939 _c4dbgp("mapflow[RNXT]: expect next keyval");
5940 m_evt_handler->add_sibling();
5941 addrem_flags(RKEY, RNXT);
5942 _line_progressed(1);
5943 if(m_evt_handler->m_curr->line_contents.rem.begins_with('#'))
5944 {
5945 _c4err("parse error: invalid comment after comma");
5946 }
5947 }
5948 else if(m_evt_handler->m_curr->line_contents.rem.begins_with('}'))
5949 {
5950 _c4dbgp("mapflow[RNXT]: end!");
5951 _line_progressed(1);
5952 _end_map_flow();
5953 goto mapflow_finish;
5954 }
5955 else
5956 {
5957 _c4err("parse error");
5958 }
5959 }
5960 else if(has_any(QMRK))
5961 {
5962 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
5963 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
5964 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
5965 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
5966 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
5967 _c4dbgpf("mapflow[QMRK]: '{}'", first);
5968 ScannedScalar sc;
5969 if(first == '\'')
5970 {
5971 _c4dbgp("mapflow[QMRK]: scanning single-quoted scalar");
5972 sc = _scan_scalar_squot();
5973 csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc);
5974 _handle_annotations_before_blck_key_scalar();
5975 m_evt_handler->set_key_scalar_squoted(maybe_filtered);
5976 addrem_flags(RKCL, QMRK);
5977 }
5978 else if(first == '"')
5979 {
5980 _c4dbgp("mapflow[QMRK]: scanning double-quoted scalar");
5981 sc = _scan_scalar_dquot();
5982 csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc);
5983 _handle_annotations_before_blck_key_scalar();
5984 m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
5985 addrem_flags(RKCL, QMRK);
5986 }
5987 // block scalars (ie | and >) cannot appear in flow containers
5988 else if(_scan_scalar_plain_map_flow(&sc))
5989 {
5990 _c4dbgp("mapflow[QMRK]: plain scalar");
5991 csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, m_evt_handler->m_curr->indref);
5992 _handle_annotations_before_blck_key_scalar();
5993 m_evt_handler->set_key_scalar_plain(maybe_filtered);
5994 addrem_flags(RKCL, QMRK);
5995 }
5996 else if(first == ':')
5997 {
5998 _c4dbgp("mapflow[QMRK]: setting empty key");
5999 _handle_annotations_before_blck_key_scalar();
6000 m_evt_handler->set_key_scalar_plain_empty();
6001 addrem_flags(RVAL, QMRK);
6002 _line_progressed(1);
6003 _maybe_skip_whitespace_tokens();
6004 }
6005 else if(first == '}') // this happens on a trailing comma like ", }"
6006 {
6007 _c4dbgp("mapflow[QMRK]: end!");
6008 _handle_annotations_before_blck_key_scalar();
6009 m_evt_handler->set_key_scalar_plain_empty();
6010 m_evt_handler->set_val_scalar_plain_empty();
6011 _end_map_flow();
6012 _line_progressed(1);
6013 goto mapflow_finish;
6014 }
6015 else if(first == ',')
6016 {
6017 _c4dbgp("mapflow[QMRK]: empty key+val!");
6018 _handle_annotations_before_blck_key_scalar();
6019 m_evt_handler->set_key_scalar_plain_empty();
6020 m_evt_handler->set_val_scalar_plain_empty();
6021 addrem_flags(RNXT, QMRK);
6022 }
6023 else if(first == '&')
6024 {
6025 csubstr anchor = _scan_anchor();
6026 _c4dbgpf("mapflow[QMRK]: key anchor! {}", prs_(anchor));
6027 _add_annotation(&m_pending_anchors, anchor);
6028 }
6029 else if(first == '*')
6030 {
6031 csubstr ref = _scan_ref_map();
6032 _c4dbgpf("mapflow[QMRK]: key ref! {}", prs_(ref));
6033 _handle_keyref(ref);
6034 addrem_flags(RKCL, QMRK);
6035 }
6036 else if(first == '[')
6037 {
6038 // RYML's tree cannot store container keys, but that's
6039 // handled inside the tree sink. Other sink types may be
6040 // able to handle it.
6041 _c4dbgp("mapflow[QMRK]: start child seqflow (!)");
6042 addrem_flags(RKCL, QMRK);
6043 _handle_annotations_before_blck_key_scalar();
6044 m_evt_handler->begin_seq_key_flow();
6045 addrem_flags(RSEQ|RVAL, RMAP|RKCL);
6046 _set_indentation(m_evt_handler->m_parent->indref);
6047 _line_progressed(1);
6048 goto mapflow_finish;
6049 }
6050 else if(first == '{')
6051 {
6052 // RYML's tree cannot store container keys, but that's
6053 // handled inside the tree sink. Other sink types may be
6054 // able to handle it.
6055 _c4dbgp("mapflow[QMRK]: start child mapflow (!)");
6056 addrem_flags(RKCL, QMRK);
6057 _handle_annotations_before_blck_key_scalar();
6058 m_evt_handler->begin_map_key_flow();
6059 _set_indentation(m_evt_handler->m_parent->indref);
6060 addrem_flags(RKEY, RKCL);
6061 _line_progressed(1);
6062 // keep going in this function
6063 }
6064 else if(first == '!')
6065 {
6066 csubstr tag = _scan_tag();
6067 _c4dbgpf("mapflow[QMRK]: tag! {}", prs_(tag));
6068 _add_annotation(&m_pending_tags, tag);
6069 }
6070 else
6071 {
6072 _c4err("parse error"); // LCOV_EXCL_LINE
6073 }
6074 }
6075
6076 mapflow_again:
6077 _c4dbgt("mapflow: go again", 0);
6078 if(_finished_line())
6079 {
6080 if C4_LIKELY(!_finished_file())
6081 {
6082 _line_ended();
6083 _scan_line();
6084 _c4dbgnextline();
6085 }
6086 else
6087 {
6088 _c4err("missing terminating }");
6089 }
6090 }
6091 goto mapflow_start;
6092
6093 mapflow_finish:
6094 _c4dbgp("mapflow: finish");
6095}
6096
6097
6098//-----------------------------------------------------------------------------
6099
6100template<class EventHandler>
6101void ParseEngine<EventHandler>::_handle_seq_block()
6102{
6103seqblck_start:
6104 _c4dbgpf("handle_seq_block: seq_id={} node_id={} level={} indent={}", m_evt_handler->m_parent->node_id, m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
6105
6106 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RSEQ), m_evt_handler->m_curr->pos);
6107 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RBLCK), m_evt_handler->m_curr->pos);
6108 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RVAL|RNXT), m_evt_handler->m_curr->pos);
6109 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, 1 == (has_any(RVAL) + has_any(RNXT)), m_evt_handler->m_curr->pos);
6110
6111 _maybe_skip_comment_strict();
6112 if(!m_evt_handler->m_curr->line_contents.rem.len)
6113 goto seqblck_again;
6114
6115 if(has_any(RVAL))
6116 {
6117 _c4dbgpf("seqblck[RVAL]: col={}", m_evt_handler->m_curr->pos.col);
6118 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
6119 if(m_evt_handler->m_curr->at_line_beginning())
6120 {
6121 _c4dbgpf("seqblck[RVAL]: indref={} indentation={}", m_evt_handler->m_curr->indref+1, m_evt_handler->m_curr->line_contents.indentation);
6122 if(m_evt_handler->m_curr->indentation_ge_extra())
6123 {
6124 _c4dbgpf("seqblck[RVAL]: skip {} from indentation", m_evt_handler->m_curr->line_contents.indentation);
6125 _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
6126 if(!m_evt_handler->m_curr->line_contents.rem.len)
6127 goto seqblck_again;
6128 }
6129 else if(m_evt_handler->m_curr->indentation_lt_extra())
6130 {
6131 _c4dbgp("seqblck[RVAL]: smaller indentation than RVAL!");
6132 if(m_evt_handler->m_curr->indentation_eq())
6133 {
6134 _c4dbgp("seqblck[RVAL]: smaller indentation than RVAL!");
6135 _handle_annotations_before_blck_val_scalar();
6136 m_evt_handler->set_val_scalar_plain_empty();
6137 addrem_flags(RNXT, RVAL);
6138 goto seqblck_again;
6139 }
6140 else
6141 {
6142 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indentation_lt(), m_evt_handler->m_curr->pos);
6143 _c4dbgp("seqblck[RVAL]: smaller indentation!");
6144 _handle_indentation_pop_from_block_seq();
6145 goto seqblck_finish;
6146 }
6147 }
6148 else if(m_evt_handler->m_curr->line_contents.indentation == npos)
6149 {
6150 _c4dbgp("seqblck[RVAL]: empty line!");
6151 _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
6152 goto seqblck_again;
6153 }
6154 }
6155 RYML_ASSERT_PARSE_CB_(callbacks(), m_evt_handler->m_curr->line_contents.rem.len, m_evt_handler->m_curr->pos);
6156 const size_t startmark = _handle_block_skip_leading_whitespace();
6157 _c4dbgpf("seqblck[RVAL]: startmark={}", startmark);
6158 if(startmark == npos)
6159 {
6160 _c4dbgp("seqblck[RVAL]: whitespace only");
6161 goto seqblck_again;
6162 }
6163 const size_t tabmark = _handle_block_get_whitespace_mark();
6164 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
6165 _c4dbgpf("seqblck[RVAL]: first='{}' currcol={}", first, m_evt_handler->m_curr->pos.col - 1);
6166 const size_t startline = m_evt_handler->m_curr->pos.line;
6167 _c4assert(m_evt_handler->m_curr->line_contents.current_col() >= m_bom_len);
6168 const size_t startindent = m_evt_handler->m_curr->line_contents.current_col() - m_bom_len;
6169 ScannedScalar sc;
6170 if(first == '\'')
6171 {
6172 _c4dbgp("seqblck[RVAL]: single-quoted scalar");
6173 sc = _scan_scalar_squot();
6174 if(!_maybe_scan_following_colon())
6175 {
6176 _c4dbgp("seqblck[RVAL]: set as val");
6177 _handle_annotations_before_blck_val_scalar();
6178 csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc); // VAL!
6179 m_evt_handler->set_val_scalar_squoted(maybe_filtered);
6180 addrem_flags(RNXT, RVAL);
6181 }
6182 else
6183 {
6184 _c4dbgp("seqblck[RVAL]: start mapblck, set scalar as key");
6185 _handle_block_check_leading_tabs(startmark);
6186 addrem_flags(RNXT, RVAL);
6187 _handle_annotations_before_start_mapblck(startline);
6188 _handle_colon();
6189 m_evt_handler->begin_map_val_block();
6190 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6191 csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc); // KEY!
6192 m_evt_handler->set_key_scalar_squoted(maybe_filtered);
6193 addrem_flags(RMAP|RVAL, RSEQ|RNXT);
6194 _maybe_skip_whitespace_tokens();
6195 goto seqblck_finish;
6196 }
6197 }
6198 else if(first == '"')
6199 {
6200 _c4dbgp("seqblck[RVAL]: double-quoted scalar");
6201 sc = _scan_scalar_dquot();
6202 if(!_maybe_scan_following_colon())
6203 {
6204 _c4dbgp("seqblck[RVAL]: set as val");
6205 _handle_annotations_before_blck_val_scalar();
6206 csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc); // VAL!
6207 m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
6208 addrem_flags(RNXT, RVAL);
6209 }
6210 else
6211 {
6212 _c4dbgp("seqblck[RVAL]: start mapblck, set scalar as key");
6213 addrem_flags(RNXT, RVAL);
6214 _handle_block_check_leading_tabs(startmark);
6215 _handle_annotations_before_start_mapblck(startline);
6216 _handle_colon();
6217 m_evt_handler->begin_map_val_block();
6218 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6219 csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc); // KEY!
6220 m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
6221 addrem_flags(RMAP|RVAL, RSEQ|RNXT);
6222 _maybe_skip_whitespace_tokens();
6223 goto seqblck_finish;
6224 }
6225 }
6226 // block scalars can only appear as keys when in QMRK scope
6227 // (ie, after ? tokens), so no need to scan following colon in
6228 // here.
6229 else if(first == '|')
6230 {
6231 _c4dbgp("seqblck[RVAL]: block-literal scalar");
6232 ScannedBlock sb;
6233 _scan_block(&sb, m_evt_handler->m_curr->indref + 1);
6234 _handle_annotations_before_blck_val_scalar();
6235 csubstr maybe_filtered = _maybe_filter_val_scalar_literal(sb);
6236 m_evt_handler->set_val_scalar_literal(maybe_filtered);
6237 addrem_flags(RNXT, RVAL);
6238 }
6239 else if(first == '>')
6240 {
6241 _c4dbgp("seqblck[RVAL]: block-folded scalar");
6242 ScannedBlock sb;
6243 _scan_block(&sb, m_evt_handler->m_curr->indref + 1);
6244 _handle_annotations_before_blck_val_scalar();
6245 csubstr maybe_filtered = _maybe_filter_val_scalar_folded(sb);
6246 m_evt_handler->set_val_scalar_folded(maybe_filtered);
6247 addrem_flags(RNXT, RVAL);
6248 }
6249 else if(_scan_scalar_plain_seq_blck(&sc))
6250 {
6251 _c4dbgp("seqblck[RVAL]: plain scalar.");
6252 if(!_maybe_scan_following_colon())
6253 {
6254 _c4dbgp("seqblck[RVAL]: set as val");
6255 _handle_annotations_before_blck_val_scalar();
6256 csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref); // VAL!
6257 m_evt_handler->set_val_scalar_plain(maybe_filtered);
6258 addrem_flags(RNXT, RVAL);
6259 }
6260 else
6261 {
6262 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indref != npos, m_evt_handler->m_curr->pos);
6263 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, startindent > m_evt_handler->m_curr->indref, m_evt_handler->m_curr->pos);
6264 _c4dbgp("seqblck[RVAL]: start mapblck, set scalar as key");
6265 _handle_block_check_leading_tabs(startmark, tabmark);
6266 addrem_flags(RNXT, RVAL);
6267 _handle_annotations_before_start_mapblck(startline);
6268 _handle_colon();
6269 m_evt_handler->begin_map_val_block();
6270 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6271 csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, m_evt_handler->m_curr->indref); // KEY!
6272 m_evt_handler->set_key_scalar_plain(maybe_filtered);
6273 addrem_flags(RMAP|RVAL, RSEQ|RNXT);
6274 _maybe_skip_whitespace_tokens();
6275 goto seqblck_finish;
6276 }
6277 }
6278 else if(first == '[')
6279 {
6280 _c4dbgp("seqblck[RVAL]: start child seqflow");
6281 addrem_flags(RNXT, RVAL);
6282 _handle_annotations_before_blck_val_scalar();
6283 m_evt_handler->begin_seq_val_flow();
6284 addrem_flags(RFLOW|RVAL, RBLCK|RNXT);
6285 _line_progressed(1);
6286 _set_indentation(m_evt_handler->m_parent->indref + 1u);
6287 goto seqblck_finish;
6288 }
6289 else if(first == '{')
6290 {
6291 _c4dbgp("seqblck[RVAL]: start child mapflow");
6292 addrem_flags(RNXT, RVAL);
6293 _handle_annotations_before_blck_val_scalar();
6294 m_evt_handler->begin_map_val_flow();
6295 addrem_flags(RMAP|RKEY|RFLOW, RBLCK|RSEQ|RVAL|RNXT);
6296 _line_progressed(1);
6297 _set_indentation(m_evt_handler->m_parent->indref + 1u);
6298 goto seqblck_finish;
6299 }
6300 else if(first == '-')
6301 {
6302 _c4dbgp("seqblck[RVAL]: dash");
6303 _handle_block_check_leading_tabs(startmark);
6304 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indref != npos, m_evt_handler->m_curr->pos);
6305 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, startindent > m_evt_handler->m_curr->indref, m_evt_handler->m_curr->pos);
6306 _c4dbgp("seqblck[RVAL]: start child seqblck");
6307 RYML_ASSERT_PARSE_CB_(this->callbacks(), startindent > m_evt_handler->m_curr->indref, m_evt_handler->m_curr->pos);
6308 addrem_flags(RNXT, RVAL);
6309 _handle_annotations_before_blck_val_scalar();
6310 m_evt_handler->begin_seq_val_block();
6311 addrem_flags(RVAL, RNXT);
6312 _set_indentation(startindent);
6313 // keep going on inside this function
6314 _line_progressed(1);
6315 }
6316 else if(first == ':')
6317 {
6318 _c4dbgp("seqblck[RVAL]: start child mapblck with empty key");
6319 addrem_flags(RNXT, RVAL);
6320 _handle_annotations_before_start_mapblck(startline);
6321 _handle_colon();
6322 m_evt_handler->begin_map_val_block();
6323 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6324 m_evt_handler->set_key_scalar_plain_empty();
6325 addrem_flags(RMAP|RVAL, RSEQ|RNXT);
6326 _line_progressed(1);
6327 _maybe_skip_whitespace_tokens();
6328 goto seqblck_finish;
6329 }
6330 else if(first == '&')
6331 {
6332 const csubstr anchor = _scan_anchor();
6333 _c4dbgpf("seqblck[RVAL]: anchor! {}", prs_(anchor));
6334 // we need to buffer the anchors, as there may be two
6335 // consecutive anchors in here
6336 _add_annotation(&m_pending_anchors, anchor, startindent, startline);
6337 }
6338 else if(first == '*')
6339 {
6340 csubstr ref = _scan_ref_seq();
6341 _c4dbgpf("seqblck[RVAL]: ref! {}", prs_(ref));
6342 if(!_maybe_scan_following_colon())
6343 {
6344 _c4dbgp("seqblck[RVAL]: set ref as val!");
6345 _handle_valref(ref);
6346 addrem_flags(RNXT, RVAL);
6347 }
6348 else
6349 {
6350 _c4dbgp("seqblck[RVAL]: ref is key of map");
6351 addrem_flags(RNXT, RVAL);
6352 _handle_annotations_before_start_mapblck(startline);
6353 m_evt_handler->begin_map_val_block();
6354 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6355 _handle_keyref(ref);
6356 addrem_flags(RMAP|RVAL, RSEQ|RNXT);
6357 _set_indentation(startindent);
6358 _maybe_skip_whitespace_tokens();
6359 goto seqblck_finish;
6360 }
6361 }
6362 else if(first == '!')
6363 {
6364 csubstr tag = _scan_tag();
6365 _c4dbgpf("seqblck[RVAL]: val tag! {}", prs_(tag));
6366 // we need to buffer the tags, as there may be two
6367 // consecutive tags in here
6368 _add_annotation(&m_pending_tags, tag, startindent, startline);
6369 }
6370 else if(first == '?')
6371 {
6372 _c4dbgp("seqblck[RVAL]: start child mapblck, explicit key");
6373 addrem_flags(RNXT, RVAL);
6374 m_evt_handler->begin_map_val_block();
6375 addrem_flags(RMAP|QMRK, RSEQ|RNXT);
6376 _set_indentation(startindent);
6377 _line_progressed(1);
6378 _maybe_skipchars(' ');
6379 if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
6380 {
6381 _c4dbgp("seqblck[RVAL]: seqblck starts after ?");
6382 addrem_flags(RKCL, QMRK);
6383 m_evt_handler->begin_seq_key_block();
6384 addrem_flags(RSEQ|RVAL, RMAP|RKCL);
6385 _save_indentation();
6386 _line_progressed(1);
6387 _maybe_skipchars(' ');
6388 }
6389 goto seqblck_finish;
6390 }
6391 else
6392 {
6393 _c4err("parse error");
6394 }
6395 }
6396 else // RNXT
6397 {
6398 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RNXT), m_evt_handler->m_curr->pos);
6399 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
6400 //
6401 // handle indentation
6402 //
6403 _c4dbgpf("seqblck[RNXT]: indref={} indentation={}", m_evt_handler->m_curr->indref, m_evt_handler->m_curr->line_contents.indentation);
6404 if C4_LIKELY(m_evt_handler->m_curr->at_line_beginning())
6405 {
6406 _c4dbgp("seqblck[RNXT]: at line begin");
6407 if(m_evt_handler->m_curr->indentation_ge())
6408 {
6409 _c4dbgpf("seqblck[RNXT]: skip {} from indref", m_evt_handler->m_curr->indref);
6410 _line_progressed(m_evt_handler->m_curr->indref);
6411 if(!m_evt_handler->m_curr->line_contents.rem.len)
6412 goto seqblck_again;
6413 }
6414 else if(m_evt_handler->m_curr->indentation_lt())
6415 {
6416 _c4dbgp("seqblck[RNXT]: smaller indentation!");
6417 _handle_indentation_pop_from_block_seq();
6418 if(has_all(RSEQ|RBLCK))
6419 {
6420 _c4dbgp("seqblck[RNXT]: still seqblck!");
6421 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RNXT), m_evt_handler->m_curr->pos);
6422 _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
6423 if(!m_evt_handler->m_curr->line_contents.rem.len)
6424 goto seqblck_again; // LCOV_EXCL_LINE
6425 }
6426 else
6427 {
6428 _c4dbgp("seqblck[RNXT]: no longer seqblck!");
6429 goto seqblck_finish;
6430 }
6431 }
6432 else if(m_evt_handler->m_curr->line_contents.indentation == npos)
6433 {
6434 _c4dbgpf("seqblck[RNXT]: blank line, len={}", m_evt_handler->m_curr->line_contents.rem);
6435 _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
6436 if(!m_evt_handler->m_curr->line_contents.rem.len)
6437 goto seqblck_again; // LCOV_EXCL_LINE
6438 }
6439 }
6440 else
6441 {
6442 _c4dbgp("seqblck[RNXT]: NOT at line begin");
6443 if(!m_evt_handler->m_curr->line_contents.rem.begins_with_any(" \t"))
6444 {
6445 _c4err("parse error");
6446 }
6447 else
6448 {
6449 _skipchars(" \t");
6450 if(!m_evt_handler->m_curr->line_contents.rem.len)
6451 {
6452 _c4dbgp("seqblck[RNXT]: again");
6453 goto seqblck_again; // LCOV_EXCL_LINE
6454 }
6455 }
6456 }
6457 //
6458 // now handle the tokens
6459 //
6460 _c4assert(m_evt_handler->m_curr->line_contents.rem.len > 0);
6461 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
6462 _c4dbgpf("seqblck[RNXT]: '{}' node_id={}", _c4prc(first), m_evt_handler->m_curr->node_id);
6463 if(first == '-')
6464 {
6465 if(m_evt_handler->m_curr->indref > 0
6466 || m_evt_handler->m_curr->line_contents.indentation > 0
6467 || !_is_doc_begin_token(m_evt_handler->m_curr->line_contents.rem))
6468 {
6469 if C4_LIKELY(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
6470 {
6471 _c4dbgp("seqblck[RNXT]: expect next val");
6472 addrem_flags(RVAL, RNXT);
6473 m_evt_handler->add_sibling();
6474 _line_progressed(1);
6475 }
6476 else
6477 {
6478 _c4err("parse error");
6479 }
6480 }
6481 else
6482 {
6483 _c4dbgp("seqblck[RNXT]: start doc");
6484 _start_doc_suddenly();
6485 _line_progressed(3);
6486 _maybe_skip_whitespace_tokens();
6487 goto seqblck_finish;
6488 }
6489 }
6490 else if(first == ':')
6491 {
6492 // This happens for example in `- [a: b]: c` (after
6493 // terminating the seq, ie, after `]`). All other cases
6494 // (ie colon after scalars) are caught elsewhere (ie, in
6495 // RVAL state).
6496 if C4_LIKELY(m_evt_handler->m_parent && (m_evt_handler->m_parent->flags & RMAP))
6497 {
6498 _c4dbgp("seqblck[RNXT]: actually this seq was '?' key of parent map");
6499 m_evt_handler->end_seq_block();
6500 goto seqblck_finish;
6501 }
6502 else
6503 {
6504 _c4err("parse error");
6505 }
6506 }
6507 else if(first == '.')
6508 {
6509 _c4dbgp("seqblck[RNXT]: maybe doc?");
6510 if(_is_doc_end_token(m_evt_handler->m_curr->line_contents.rem))
6511 {
6512 _c4dbgp("seqblck[RNXT]: end doc");
6513 _end_doc_suddenly();
6514 _line_progressed(3);
6515 _maybe_skip_whitespace_tokens();
6516 _check_doc_end_tokens();
6517 goto seqblck_finish;
6518 }
6519 else
6520 {
6521 _c4err("parse error");
6522 }
6523 }
6524 else
6525 {
6526 // may be an indentless sequence nested in a map...
6527 #ifdef RYML_DBG
6528 _print_state_stack();
6529 #endif
6530 if(m_evt_handler->m_parent
6531 && has_all(RMAP|RBLCK, m_evt_handler->m_parent)
6532 && m_evt_handler->m_curr->indref == m_evt_handler->m_parent->indref)
6533 {
6534 _c4dbgpf("seqblck[RNXT]: end indentless seq, go to parent={}. node={}", m_evt_handler->m_parent->node_id, m_evt_handler->m_curr->node_id);
6535 RYML_ASSERT_PARSE_CB_(this->callbacks(), m_evt_handler->m_curr != m_evt_handler->m_parent, m_evt_handler->m_curr->pos);
6536 _handle_indentation_pop(m_evt_handler->m_parent);
6537 RYML_ASSERT_PARSE_CB_(this->callbacks(), has_all(RMAP|RBLCK), m_evt_handler->m_curr->pos);
6538 m_evt_handler->add_sibling();
6539 addrem_flags(RKEY, RNXT);
6540 goto seqblck_finish;
6541 }
6542 else if(first == '\t')
6543 {
6544 size_t pos = m_evt_handler->m_curr->line_contents.rem.first_not_of('\t');
6545 if(pos == npos)
6546 {
6547 _line_progressed(m_evt_handler->m_curr->line_contents.rem.len);
6548 goto seqblck_again;
6549 }
6550 }
6551 _c4err("parse error");
6552 }
6553 }
6554
6555 seqblck_again:
6556 _c4dbgt("seqblck: go again", 0);
6557 if(_finished_line())
6558 {
6559 m_bom_len = 0;
6560 _line_ended();
6561 _scan_line();
6562 if(_finished_file())
6563 {
6564 _c4dbgp("seqblck: finish!");
6565 _end_seq_blck();
6566 goto seqblck_finish;
6567 }
6568 _c4dbgnextline();
6569 }
6570 goto seqblck_start;
6571
6572 seqblck_finish:
6573 _c4dbgp("seqblck: finish");
6574}
6575
6576
6577//-----------------------------------------------------------------------------
6578
6579template<class EventHandler>
6580void ParseEngine<EventHandler>::_handle_map_block()
6581{
6582mapblck_start:
6583 _c4dbgpf("handle_map_block: map_id={} node_id={} level={} indref={}", m_evt_handler->m_parent->node_id, m_evt_handler->m_curr->node_id, m_evt_handler->m_curr->level, m_evt_handler->m_curr->indref);
6584
6585 // states: RKEY -> RVAL -> RNXT
6586 // states: QMRK -> RKCL -> RVAL -> RNXT
6587 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RMAP), m_evt_handler->m_curr->pos);
6588 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RBLCK), m_evt_handler->m_curr->pos);
6589 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY|RKCL|RVAL|RNXT|QMRK), m_evt_handler->m_curr->pos);
6590 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, 1 == (has_any(RKEY) + has_any(RKCL) + has_any(RVAL) + has_any(RNXT) + has_any(QMRK)), m_evt_handler->m_curr->pos);
6591
6592 _maybe_skip_comment();
6593 if(!m_evt_handler->m_curr->line_contents.rem.len)
6594 goto mapblck_again;
6595
6596 if(has_any(RKEY))
6597 {
6598 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
6599 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
6600 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
6601 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
6602 //
6603 // handle indentation
6604 //
6605 if(m_evt_handler->m_curr->at_line_beginning())
6606 {
6607 if(m_evt_handler->m_curr->indentation_eq())
6608 {
6609 _c4dbgpf("mapblck[RKEY]: skip {} from indref", m_evt_handler->m_curr->indref);
6610 _line_progressed(m_evt_handler->m_curr->indref);
6611 if(!m_evt_handler->m_curr->line_contents.rem.len)
6612 goto mapblck_again;
6613 }
6614 else if(m_evt_handler->m_curr->indentation_lt())
6615 {
6616 _c4dbgp("mapblck[RKEY]: smaller indentation!");
6617 _handle_indentation_pop_from_block_map();
6618 _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
6619 if(has_all(RMAP|RBLCK))
6620 {
6621 _c4dbgp("mapblck[RKEY]: still mapblck!");
6622 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_any(RKEY), m_evt_handler->m_curr->pos);
6623 if(!m_evt_handler->m_curr->line_contents.rem.len)
6624 goto mapblck_again;
6625 }
6626 else
6627 {
6628 _c4dbgp("mapblck[RKEY]: no longer mapblck!");
6629 goto mapblck_finish;
6630 }
6631 }
6632 else
6633 {
6634 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indentation_gt(), m_evt_handler->m_curr->pos);
6635 _c4err("invalid indentation");
6636 }
6637 }
6638 //
6639 // now handle the tokens
6640 //
6641 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
6642 const size_t startline = m_evt_handler->m_curr->pos.line;
6643 const size_t startindent = m_evt_handler->m_curr->line_contents.current_col();
6644 _c4dbgpf("mapblck[RKEY]: '{}'", _c4prc(first));
6645 ScannedScalar sc;
6646 if(first == '\'')
6647 {
6648 _c4dbgp("mapblck[RKEY]: scanning single-quoted scalar");
6649 sc = _scan_scalar_squot();
6650 csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
6651 _handle_annotations_before_blck_key_scalar();
6652 m_evt_handler->set_key_scalar_squoted(maybe_filtered);
6653 addrem_flags(RVAL, RKEY);
6654 if(!_maybe_scan_following_colon())
6655 _c4err("could not find ':' colon after key");
6656 _handle_colon();
6657 _maybe_skip_whitespace_tokens();
6658 }
6659 else if(first == '"')
6660 {
6661 _c4dbgp("mapblck[RKEY]: scanning double-quoted scalar");
6662 sc = _scan_scalar_dquot();
6663 csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
6664 _handle_annotations_before_blck_key_scalar();
6665 m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
6666 addrem_flags(RVAL, RKEY);
6667 if(!_maybe_scan_following_colon())
6668 _c4err("could not find ':' colon after key");
6669 _handle_colon();
6670 _maybe_skip_whitespace_tokens();
6671 }
6672 // block scalars (| and >) can not be used as keys unless they
6673 // appear in an explicit QMRK scope (ie, after the ? token),
6674 else if C4_UNLIKELY(first == '|')
6675 {
6676 _c4err("block map: literal keys must be enclosed in '?'");
6677 }
6678 else if C4_UNLIKELY(first == '>')
6679 {
6680 _c4err("block map: folded keys must be enclosed in '?'");
6681 }
6682 else if(_scan_scalar_plain_map_blck(&sc))
6683 {
6684 _c4dbgp("mapblck[RKEY]: plain scalar");
6685 csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref);
6686 _handle_annotations_before_blck_key_scalar();
6687 m_evt_handler->set_key_scalar_plain(maybe_filtered);
6688 addrem_flags(RVAL, RKEY);
6689 if(!_maybe_scan_following_colon())
6690 _c4err("could not find ':' colon after key");
6691 _handle_colon();
6692 _maybe_skip_whitespace_tokens();
6693 }
6694 else if(first == '?')
6695 {
6696 _c4dbgp("mapblck[RKEY]: key token!");
6697 addrem_flags(QMRK, RKEY);
6698 _line_progressed(1);
6699 _maybe_skipchars(' ');
6700 if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
6701 {
6702 _c4dbgp("mapblck[RKEY]: seqblck starts after ?");
6703 addrem_flags(RKCL, QMRK);
6704 m_evt_handler->begin_seq_key_block();
6705 addrem_flags(RSEQ|RVAL, RMAP|RKCL);
6706 _save_indentation();
6707 _line_progressed(1);
6708 _maybe_skipchars(' ');
6709 goto mapblck_finish;
6710 }
6711 goto mapblck_again;
6712 }
6713 else if(first == ':')
6714 {
6715 _c4dbgp("mapblck[RKEY]: setting empty key");
6716 _handle_annotations_before_blck_key_scalar();
6717 m_evt_handler->set_key_scalar_plain_empty();
6718 addrem_flags(RVAL, RKEY);
6719 _line_progressed(1);
6720 _handle_colon();
6721 _maybe_skip_whitespace_tokens();
6722 }
6723 else if(first == '*')
6724 {
6725 csubstr ref = _scan_ref_map();
6726 _c4dbgpf("mapblck[RKEY]: key ref! {}", prs_(ref));
6727 _handle_keyref(ref);
6728 addrem_flags(RVAL, RKEY);
6729 if(!_maybe_scan_following_colon())
6730 _c4err("could not find ':' colon after key");
6731 _handle_colon();
6732 _maybe_skip_whitespace_tokens();
6733 }
6734 else if(first == '&')
6735 {
6736 csubstr anchor = _scan_anchor();
6737 _c4dbgpf("mapblck[RKEY]: key anchor! {}", prs_(anchor));
6738 _add_annotation(&m_pending_anchors, anchor, startindent, startline);
6739 }
6740 else if(first == '!')
6741 {
6742 csubstr tag = _scan_tag();
6743 _c4dbgpf("mapblck[RKEY]: key tag! {}", prs_(tag));
6744 _add_annotation(&m_pending_tags, tag, startindent, startline);
6745 }
6746 else if(first == '[')
6747 {
6748 // RYML's tree cannot store container keys, but that's
6749 // handled inside the tree handler. Other handlers may be
6750 // able to handle it.
6751 _c4dbgp("mapblck[RKEY]: start child seqflow (!)");
6752 _handle_annotations_before_blck_key_scalar();
6753 m_evt_handler->begin_seq_key_flow();
6754 addrem_flags(RSEQ|RFLOW|RVAL, RKEY|RMAP|RBLCK);
6755 _line_progressed(1);
6756 _set_indentation(startindent);
6757 goto mapblck_finish;
6758 }
6759 else if(first == '{')
6760 {
6761 // RYML's tree cannot store container keys, but that's
6762 // handled inside the tree handler. Other handlers may be
6763 // able to handle it.
6764 _c4dbgp("mapblck[RKEY]: start child mapflow (!)");
6765 _handle_annotations_before_blck_key_scalar();
6766 m_evt_handler->begin_map_key_flow();
6767 addrem_flags(RFLOW|RKEY, RBLCK);
6768 _line_progressed(1);
6769 _set_indentation(startindent);
6770 goto mapblck_finish;
6771 }
6772 else if(first == '-')
6773 {
6774 _c4dbgp("mapblck[RKEY]: maybe doc?");
6775 if(m_evt_handler->m_curr->line_contents.indentation == 0 && _is_doc_begin_token(m_evt_handler->m_curr->line_contents.rem))
6776 {
6777 _c4dbgp("mapblck[RKEY]: end+start doc");
6778 _start_doc_suddenly();
6779 _line_progressed(3);
6780 _maybe_skip_whitespace_tokens();
6781 goto mapblck_finish;
6782 }
6783 else
6784 {
6785 _c4err("parse error");
6786 }
6787 }
6788 else if(first == '.')
6789 {
6790 _c4dbgp("mapblck[RKEY]: maybe end doc?");
6791 if(m_evt_handler->m_curr->line_contents.indentation == 0 && _is_doc_end_token(m_evt_handler->m_curr->line_contents.rem))
6792 {
6793 _c4dbgp("mapblck[RKEY]: end doc");
6794 _end_doc_suddenly();
6795 _line_progressed(3);
6796 _maybe_skip_whitespace_tokens();
6797 _check_doc_end_tokens();
6798 goto mapblck_finish;
6799 }
6800 else
6801 {
6802 _c4err("parse error"); // LCOV_EXCL_LINE
6803 }
6804 }
6805 else
6806 {
6807 _c4err("parse error");
6808 }
6809 }
6810 else if(has_any(RVAL))
6811 {
6812 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
6813 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
6814 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
6815 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
6816 //
6817 // handle indentation
6818 //
6819 if(m_evt_handler->m_curr->at_line_beginning())
6820 {
6821 _c4dbgpf("mapblck[RVAL]: indref={} indentation={}", m_evt_handler->m_curr->indref+1, m_evt_handler->m_curr->line_contents.indentation);
6822 m_evt_handler->m_curr->more_indented = false;
6823 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indref != npos, m_evt_handler->m_curr->pos);
6824 if(m_evt_handler->m_curr->indentation_eq_extra())
6825 {
6826 _c4dbgp("mapblck[RVAL]: skip indentation!");
6827 _line_progressed(m_evt_handler->m_curr->indref + 1);
6828 if(!m_evt_handler->m_curr->line_contents.rem.len)
6829 goto mapblck_again;
6830 }
6831 else if(m_evt_handler->m_curr->indentation_gt_extra())
6832 {
6833 _c4dbgp("mapblck[RVAL]: more indented!");
6834 m_evt_handler->m_curr->more_indented = true;
6835 _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
6836 if(!m_evt_handler->m_curr->line_contents.rem.len)
6837 goto mapblck_again; // LCOV_EXCL_LINE
6838 }
6839 else if(m_evt_handler->m_curr->indentation_lt_extra())
6840 {
6841 if(m_evt_handler->m_curr->indentation_eq())
6842 {
6843 _c4dbgp("mapblck[RVAL]: smaller indentation than RVAL!");
6844 // watchout for indentless seqs
6845 if(!_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem.sub(m_evt_handler->m_curr->line_contents.indentation)))
6846 {
6847 _c4dbgp("mapblck[RVAL]: smaller indentation than RVAL!");
6848 _handle_annotations_before_blck_val_scalar();
6849 m_evt_handler->set_val_scalar_plain_empty();
6850 addrem_flags(RNXT, RVAL);
6851 goto mapblck_again;
6852 }
6853 }
6854 else
6855 {
6856 _c4dbgp("mapblck[RVAL]: smaller indentation than RKEY!");
6857 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indentation_lt(), m_evt_handler->m_curr->pos);
6858 _handle_indentation_pop_from_block_map();
6859 if(has_all(RMAP|RBLCK))
6860 {
6861 _c4dbgp("mapblck[RVAL]: still mapblck!");
6862 _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
6863 if(has_any(RNXT))
6864 {
6865 _c4dbgp("mapblck[RVAL]: speculatively expect next keyval");
6866 m_evt_handler->add_sibling();
6867 addrem_flags(RKEY, RNXT);
6868 }
6869 goto mapblck_again;
6870 }
6871 else
6872 {
6873 _c4dbgp("mapblck[RVAL]: no longer mapblck!");
6874 goto mapblck_finish;
6875 }
6876 }
6877 }
6878 }
6879 const size_t startcol = _handle_block_skip_leading_whitespace();
6880 if(startcol == npos)
6881 {
6882 _c4dbgp("mapblck[RVAL]: whitespace only");
6883 goto mapblck_again; // LCOV_EXCL_LINE
6884 }
6885 const size_t tabmark = _handle_block_get_whitespace_mark();
6886 //
6887 // now handle the tokens
6888 //
6889 _c4assert(m_evt_handler->m_curr->line_contents.rem.len);
6890 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
6891 const size_t startline = m_evt_handler->m_curr->pos.line;
6892 const size_t startindent = m_evt_handler->m_curr->line_contents.current_col();
6893 _c4dbgpf("mapblck[RVAL]: '{}'", _c4prc(first));
6894 ScannedScalar sc;
6895 if(first == '\'')
6896 {
6897 _c4dbgp("mapblck[RVAL]: scanning single-quoted scalar");
6898 sc = _scan_scalar_squot();
6899 if(!_maybe_scan_following_colon())
6900 {
6901 _c4dbgp("mapblck[RVAL]: set as val");
6902 _handle_annotations_before_blck_val_scalar();
6903 csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc); // VAL!
6904 m_evt_handler->set_val_scalar_squoted(maybe_filtered);
6905 addrem_flags(RNXT, RVAL);
6906 }
6907 else
6908 {
6909 _c4assert(m_evt_handler->m_curr->indref != npos);
6910 _c4assert(startindent > m_evt_handler->m_curr->indref);
6911 _c4dbgp("mapblck[RVAL]: start new block map, set scalar as key");
6912 _handle_block_check_leading_tabs(startcol);
6913 _handle_annotations_before_start_mapblck(startline);
6914 addrem_flags(RNXT, RVAL);
6915 _handle_colon();
6916 m_evt_handler->begin_map_val_block();
6917 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6918 csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc); // KEY!
6919 m_evt_handler->set_key_scalar_squoted(maybe_filtered);
6920 _maybe_skip_whitespace_tokens();
6921 // keep the child state on RVAL
6922 addrem_flags(RVAL, RNXT);
6923 }
6924 }
6925 else if(first == '"')
6926 {
6927 _c4dbgp("mapblck[RVAL]: scanning double-quoted scalar");
6928 sc = _scan_scalar_dquot();
6929 if(!_maybe_scan_following_colon())
6930 {
6931 _c4dbgp("mapblck[RVAL]: set as val");
6932 _handle_annotations_before_blck_val_scalar();
6933 csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc); // VAL!
6934 m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
6935 addrem_flags(RNXT, RVAL);
6936 }
6937 else
6938 {
6939 _c4assert(m_evt_handler->m_curr->indref != npos);
6940 _c4assert(startindent > m_evt_handler->m_curr->indref);
6941 _c4dbgp("mapblck[RVAL]: start new block map, set scalar as key");
6942 _handle_block_check_leading_tabs(startcol);
6943 _handle_annotations_before_start_mapblck(startline);
6944 addrem_flags(RNXT, RVAL);
6945 _handle_colon();
6946 m_evt_handler->begin_map_val_block();
6947 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6948 csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc); // KEY!
6949 m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
6950 _maybe_skip_whitespace_tokens();
6951 // keep the child state on RVAL
6952 addrem_flags(RVAL, RNXT);
6953 }
6954 }
6955 // block scalars can only appear as keys when in QMRK scope
6956 // (ie, after ? tokens), so no need to scan following colon
6957 else if(first == '|')
6958 {
6959 _c4dbgp("mapblck[RVAL]: scanning block-literal scalar");
6960 ScannedBlock sb;
6961 _scan_block(&sb, m_evt_handler->m_curr->indref + 1);
6962 _handle_annotations_before_blck_val_scalar();
6963 csubstr maybe_filtered = _maybe_filter_val_scalar_literal(sb);
6964 m_evt_handler->set_val_scalar_literal(maybe_filtered);
6965 addrem_flags(RNXT, RVAL);
6966 }
6967 else if(first == '>')
6968 {
6969 _c4dbgp("mapblck[RVAL]: scanning block-folded scalar");
6970 ScannedBlock sb;
6971 _scan_block(&sb, m_evt_handler->m_curr->indref + 1);
6972 _handle_annotations_before_blck_val_scalar();
6973 csubstr maybe_filtered = _maybe_filter_val_scalar_folded(sb);
6974 m_evt_handler->set_val_scalar_folded(maybe_filtered);
6975 addrem_flags(RNXT, RVAL);
6976 }
6977 else if(_scan_scalar_plain_map_blck(&sc))
6978 {
6979 _c4dbgp("mapblck[RVAL]: plain scalar.");
6980 if(!_maybe_scan_following_colon())
6981 {
6982 _c4dbgp("mapblck[RVAL]: set as val");
6983 _handle_annotations_before_blck_val_scalar();
6984 csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, m_evt_handler->m_curr->indref); // VAL!
6985 m_evt_handler->set_val_scalar_plain(maybe_filtered);
6986 addrem_flags(RNXT, RVAL);
6987 }
6988 else
6989 {
6990 _c4assert(m_evt_handler->m_curr->indref != npos);
6991 _c4assert(startindent > m_evt_handler->m_curr->indref);
6992 _c4dbgpf("mapblck[RVAL]: start new block map, set scalar as key {}", m_evt_handler->m_curr->indref);
6993 _handle_block_check_leading_tabs(startcol, tabmark);
6994 addrem_flags(RNXT, RVAL);
6995 _handle_annotations_before_start_mapblck(startline);
6996 _handle_colon();
6997 m_evt_handler->begin_map_val_block();
6998 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
6999 csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, m_evt_handler->m_curr->indref); // KEY!
7000 m_evt_handler->set_key_scalar_plain(maybe_filtered);
7001 _maybe_skip_whitespace_tokens();
7002 // keep the child state on RVAL
7003 addrem_flags(RVAL, RNXT);
7004 }
7005 }
7006 else if(first == '-' && _is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7007 {
7008 if C4_UNLIKELY(!m_evt_handler->m_curr->at_first_token())
7009 _c4err("parse error");
7010 _c4dbgp("mapblck[RVAL]: start val seqblck");
7011 _handle_block_check_leading_tabs(startcol);
7012 addrem_flags(RNXT, RVAL);
7013 _handle_annotations_before_blck_val_scalar();
7014 m_evt_handler->begin_seq_val_block();
7015 addrem_flags(RSEQ|RVAL, RMAP|RNXT);
7016 _set_indentation(startindent);
7017 _line_progressed(1);
7018 _maybe_skip_whitespace_tokens();
7019 goto mapblck_finish;
7020 }
7021 else if(first == '[')
7022 {
7023 _c4dbgp("mapblck[RVAL]: start val seqflow");
7024 addrem_flags(RNXT, RVAL);
7025 _handle_annotations_before_blck_val_scalar();
7026 m_evt_handler->begin_seq_val_flow();
7027 addrem_flags(RSEQ|RFLOW|RVAL, RMAP|RBLCK|RNXT);
7028 _set_indentation(m_evt_handler->m_parent->indref + 1u);
7029 _line_progressed(1);
7030 goto mapblck_finish;
7031 }
7032 else if(first == '{')
7033 {
7034 _c4dbgp("mapblck[RVAL]: start val mapflow");
7035 addrem_flags(RNXT, RVAL);
7036 _handle_annotations_before_blck_val_scalar();
7037 m_evt_handler->begin_map_val_flow();
7038 addrem_flags(RKEY|RFLOW, RBLCK|RVAL|RNXT);
7039 m_evt_handler->m_curr->scalar_col = m_evt_handler->m_curr->line_contents.indentation;
7040 _set_indentation(m_evt_handler->m_parent->indref + 1u);
7041 _line_progressed(1);
7042 goto mapblck_finish;
7043 }
7044 else if(first == '*')
7045 {
7046 csubstr ref = _scan_ref_map();
7047 _c4dbgpf("mapblck[RVAL]: ref! {}", prs_(ref));
7048 if(_maybe_scan_following_colon())
7049 {
7050 _c4dbgp("mapblck[RVAL]: start child map, block");
7051 addrem_flags(RNXT, RVAL);
7052 _handle_annotations_before_blck_val_scalar();
7053 m_evt_handler->begin_map_val_block();
7054 _handle_keyref(ref);
7055 _set_indentation(startindent);
7056 // keep going in RVAL
7057 addrem_flags(RVAL, RNXT);
7058 }
7059 else
7060 {
7061 _c4dbgp("mapblck[RVAL]: was val ref");
7062 _handle_valref(ref);
7063 addrem_flags(RNXT, RVAL);
7064 }
7065 _maybe_skip_whitespace_tokens();
7066 }
7067 else if(first == '&')
7068 {
7069 csubstr anchor = _scan_anchor();
7070 _c4dbgpf("mapblck[RVAL]: anchor! {}", prs_(anchor));
7071 // we need to buffer the anchors, as there may be two
7072 // consecutive anchors in here
7073 _add_annotation(&m_pending_anchors, anchor, startindent, startline);
7074 }
7075 else if(first == '!')
7076 {
7077 csubstr tag = _scan_tag();
7078 _c4dbgpf("mapblck[RVAL]: tag! {}", prs_(tag));
7079 // we need to buffer the tags, as there may be two
7080 // consecutive tags in here
7081 _add_annotation(&m_pending_tags, tag, startindent, startline);
7082 }
7083 else if(first == '?')
7084 {
7085 if C4_UNLIKELY(!m_evt_handler->m_curr->at_first_token())
7086 _c4err("parse error");
7087 _c4dbgp("mapblck[RVAL]: start val mapblck");
7088 addrem_flags(RNXT, RVAL);
7089 _handle_annotations_before_blck_val_scalar();
7090 m_evt_handler->begin_map_val_block();
7091 addrem_flags(QMRK, RNXT);
7092 _set_indentation(startindent);
7093 _line_progressed(1);
7094 _maybe_skipchars(' ');
7095 if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7096 {
7097 _c4dbgp("mapblck[RVAL]: seqblck starts after ?");
7098 addrem_flags(RKCL, QMRK);
7099 m_evt_handler->begin_seq_key_block();
7100 addrem_flags(RSEQ|RVAL, RMAP|RKCL);
7101 _save_indentation();
7102 _line_progressed(1);
7103 _maybe_skipchars(' ');
7104 goto mapblck_finish;
7105 }
7106 goto mapblck_again;
7107 }
7108 else if(first == ':')
7109 {
7110 _c4dbgp("mapblck[RVAL]: start val mapblck");
7111 addrem_flags(RNXT, RVAL);
7112 _handle_annotations_before_start_mapblck(startline);
7113 _handle_colon();
7114 m_evt_handler->begin_map_val_block();
7115 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7116 m_evt_handler->set_key_scalar_plain_empty();
7117 // keep the child state on RVAL
7118 addrem_flags(RVAL, RNXT);
7119 _line_progressed(1);
7120 _maybe_skip_whitespace_tokens();
7121 goto mapblck_again;
7122 }
7123 else
7124 {
7125 _c4err("parse error"); // LCOV_EXCL_LINE
7126 }
7127 }
7128 else if(has_any(RNXT))
7129 {
7130 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
7131 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
7132 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
7133 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
7134 //
7135 // handle indentation
7136 //
7137 if(m_evt_handler->m_curr->at_line_beginning())
7138 {
7139 _c4dbgpf("mapblck[RNXT]: indref={} indentation={}", m_evt_handler->m_curr->indref, m_evt_handler->m_curr->line_contents.indentation);
7140 if(m_evt_handler->m_curr->indentation_eq())
7141 {
7142 _c4dbgpf("mapblck[RNXT]: skip {} from indref", m_evt_handler->m_curr->indref);
7143 _line_progressed(m_evt_handler->m_curr->indref);
7144 _c4dbgp("mapblck[RNXT]: speculatively expect next keyval");
7145 m_evt_handler->add_sibling();
7146 addrem_flags(RKEY, RNXT);
7147 goto mapblck_again;
7148 }
7149 else if(m_evt_handler->m_curr->indentation_lt())
7150 {
7151 _c4dbgp("mapblck[RNXT]: smaller indentation!");
7152 _handle_indentation_pop_from_block_map();
7153 if(has_all(RMAP|RBLCK))
7154 {
7155 _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
7156 if(!has_any(RKCL))
7157 {
7158 _c4dbgp("mapblck[RNXT]: speculatively expect next keyval");
7159 m_evt_handler->add_sibling();
7160 addrem_flags(RKEY, RNXT);
7161 }
7162 goto mapblck_again;
7163 }
7164 else
7165 {
7166 goto mapblck_finish;
7167 }
7168 }
7169 }
7170 else
7171 {
7172 _c4dbgp("mapblck[RNXT]: NOT at line begin");
7173 if(!m_evt_handler->m_curr->line_contents.rem.begins_with_any(" \t"))
7174 {
7175 _c4err("parse error");
7176 }
7177 else
7178 {
7179 _skipchars(" \t");
7180 if(!m_evt_handler->m_curr->line_contents.rem.len)
7181 {
7182 _c4dbgp("seqblck[RNXT]: again");
7183 goto mapblck_again; // LCOV_EXCL_LINE
7184 }
7185 }
7186 }
7187 //
7188 // handle tokens
7189 //
7190 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->line_contents.rem.len > 0, m_evt_handler->m_curr->pos);
7191 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
7192 _c4dbgpf("mapblck[RNXT]: '{}'", _c4prc(first));
7193 if(first == ' ')
7194 {
7195 _c4dbgp("mapblck[RNXT]: skip spaces");
7196 _maybe_skip_whitespace_tokens();
7197 }
7198 else
7199 {
7200 _c4err("parse error");
7201 }
7202 }
7203 else if(has_any(QMRK))
7204 {
7205 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
7206 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKCL), m_evt_handler->m_curr->pos);
7207 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
7208 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
7209 if(_handle_map_block_qmrk())
7210 goto mapblck_again;
7211 else
7212 goto mapblck_finish;
7213 }
7214 else if(has_any(RKCL)) // read the key colon (after QMRK)
7215 {
7216 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RKEY), m_evt_handler->m_curr->pos);
7217 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RVAL), m_evt_handler->m_curr->pos);
7218 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT), m_evt_handler->m_curr->pos);
7219 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(QMRK), m_evt_handler->m_curr->pos);
7220 if(_handle_map_block_rkcl())
7221 goto mapblck_again;
7222 else
7223 goto mapblck_finish;
7224 }
7225
7226 mapblck_again:
7227 _c4dbgt("mapblck: again", 0);
7228 if(_finished_line())
7229 {
7230 _line_ended();
7231 _scan_line();
7232 if(_finished_file())
7233 {
7234 _c4dbgp("mapblck: file finished!");
7235 _end_map_blck();
7236 goto mapblck_finish;
7237 }
7238 _c4dbgnextline();
7239 }
7240 goto mapblck_start;
7241
7242 mapblck_finish:
7243 _c4dbgp("mapblck: finish");
7244}
7245
7246
7247//-----------------------------------------------------------------------------
7248
7249// return true if we should remain in map_block
7250template<class EventHandler>
7251bool ParseEngine<EventHandler>::_handle_map_block_qmrk()
7252{
7253 //
7254 // handle indentation
7255 //
7256 if(m_evt_handler->m_curr->at_line_beginning())
7257 {
7258 _c4dbgpf("mapblck[QMRK]: at line beginning. ind={} indref={}", m_evt_handler->m_curr->line_contents.indentation, m_evt_handler->m_curr->indref);
7259 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->line_contents.indentation != npos, m_evt_handler->m_curr->pos);
7260 if(m_evt_handler->m_curr->indentation_eq_extra())
7261 {
7262 _c4dbgpf("mapblck[QMRK]: skip {} from indref", m_evt_handler->m_curr->indref + 1);
7263 _line_progressed(m_evt_handler->m_curr->indref + 1);
7264 if(!m_evt_handler->m_curr->line_contents.rem.len)
7265 return true; // go again
7266 }
7267 // indentation can be larger in QMRK state
7268 else if(m_evt_handler->m_curr->indentation_gt_extra())
7269 {
7270 _c4dbgp("mapblck[QMRK]: larger indentation !");
7271 _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
7272 if(!m_evt_handler->m_curr->line_contents.rem.len)
7273 return true; // go again
7274 }
7275 else
7276 {
7277 _c4dbgp("mapblck[QMRK]: smaller indentation!");
7278 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->indentation_lt_extra(), m_evt_handler->m_curr->pos);
7279 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_curr->line_contents.rem.len > 0, m_evt_handler->m_curr->pos);
7280 if(m_evt_handler->m_curr->indentation_eq()
7281 // defend against docs or indentless seqs
7282 && m_evt_handler->m_curr->line_contents.rem.str[0] != '-')
7283 {
7284 _c4dbgp("mapblck[QMRK]: QMRK finished!");
7285 _handle_annotations_before_blck_key_scalar();
7286 m_evt_handler->set_key_scalar_plain_empty();
7287 addrem_flags(RKCL, QMRK);
7288 return true; // go again
7289 }
7290 else if(m_evt_handler->m_curr->indentation_lt())
7291 {
7292 _c4dbgp("mapblck[QMRK]: indentation pop!");
7293 _handle_indentation_pop_from_block_map();
7294 _line_progressed(m_evt_handler->m_curr->line_contents.indentation);
7295 if(has_all(RMAP|RBLCK))
7296 {
7297 _c4dbgp("mapblck[QMRK]: still mapblck!");
7298 return true; // go again
7299 }
7300 else
7301 {
7302 _c4dbgp("mapblck[QMRK]: no longer mapblck!");
7303 return false; // finish mapblck
7304 }
7305 }
7306 }
7307 }
7308 //
7309 // now handle the tokens
7310 //
7311 _c4assert(m_evt_handler->m_curr->line_contents.rem.len);
7312 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
7313 const size_t startline = m_evt_handler->m_curr->pos.line;
7314 const size_t startindent = m_evt_handler->m_curr->line_contents.current_col();
7315 _c4dbgpf("mapblck[QMRK]: '{}'", first);
7316 ScannedScalar sc;
7317 if(first == '\'')
7318 {
7319 _c4dbgp("mapblck[QMRK]: scanning single-quoted scalar");
7320 sc = _scan_scalar_squot();
7321 csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc); // KEY!
7322 addrem_flags(RKCL, QMRK);
7323 if(!_maybe_scan_following_colon())
7324 {
7325 _c4dbgp("mapblck[QMRK]: set as key");
7326 _handle_annotations_before_blck_key_scalar();
7327 m_evt_handler->set_key_scalar_squoted(maybe_filtered);
7328 }
7329 else
7330 {
7331 _c4dbgp("mapblck[QMRK]: start new block map as key (!), set scalar as key");
7332 _handle_annotations_before_start_mapblck_as_key();
7333 m_evt_handler->begin_map_key_block();
7334 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7335 m_evt_handler->set_key_scalar_squoted(maybe_filtered);
7336 _maybe_skip_whitespace_tokens();
7337 _set_indentation(startindent);
7338 // keep the child state on RVAL
7339 addrem_flags(RVAL, RKCL);
7340 }
7341 }
7342 else if(first == '"')
7343 {
7344 _c4dbgp("mapblck[QMRK]: scanning double-quoted scalar");
7345 sc = _scan_scalar_dquot();
7346 csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc); // KEY!
7347 addrem_flags(RKCL, QMRK);
7348 if(!_maybe_scan_following_colon())
7349 {
7350 _c4dbgp("mapblck[QMRK]: set as key");
7351 _handle_annotations_before_blck_key_scalar();
7352 m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
7353 }
7354 else
7355 {
7356 _c4dbgp("mapblck[QMRK]: start new block map as key (!), set scalar as key");
7357 _handle_annotations_before_start_mapblck_as_key();
7358 m_evt_handler->begin_map_key_block();
7359 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7360 m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
7361 _maybe_skip_whitespace_tokens();
7362 _set_indentation(startindent);
7363 // keep the child state on RVAL
7364 addrem_flags(RVAL, RKCL);
7365 }
7366 }
7367 else if(first == '|')
7368 {
7369 _c4dbgp("mapblck[QMRK]: scanning block-literal scalar");
7370 ScannedBlock sb;
7371 _scan_block(&sb, m_evt_handler->m_curr->indref + 1);
7372 csubstr maybe_filtered = _maybe_filter_key_scalar_literal(sb); // KEY!
7373 _handle_annotations_before_blck_key_scalar();
7374 m_evt_handler->set_key_scalar_literal(maybe_filtered);
7375 addrem_flags(RKCL, QMRK);
7376 }
7377 else if(first == '>')
7378 {
7379 _c4dbgp("mapblck[QMRK]: scanning block-literal scalar");
7380 ScannedBlock sb;
7381 _scan_block(&sb, m_evt_handler->m_curr->indref + 1);
7382 csubstr maybe_filtered = _maybe_filter_key_scalar_folded(sb); // KEY!
7383 _handle_annotations_before_blck_key_scalar();
7384 m_evt_handler->set_key_scalar_folded(maybe_filtered);
7385 addrem_flags(RKCL, QMRK);
7386 }
7387 else if(_scan_scalar_plain_map_blck(&sc))
7388 {
7389 _c4dbgp("mapblck[QMRK]: plain scalar");
7390 csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, m_evt_handler->m_curr->indref); // KEY!
7391 addrem_flags(RKCL, QMRK);
7392 if(!_maybe_scan_following_colon())
7393 {
7394 _c4dbgp("mapblck[QMRK]: set as key");
7395 _handle_annotations_before_blck_key_scalar();
7396 m_evt_handler->set_key_scalar_plain(maybe_filtered);
7397 }
7398 else
7399 {
7400 _c4dbgp("mapblck[QMRK]: start new block map as key (!), set scalar as key");
7401 _handle_annotations_before_start_mapblck_as_key();
7402 m_evt_handler->begin_map_key_block();
7403 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7404 m_evt_handler->set_key_scalar_plain(maybe_filtered);
7405 _maybe_skip_whitespace_tokens();
7406 _set_indentation(startindent);
7407 // keep the child state on RVAL
7408 addrem_flags(RVAL, RKCL);
7409 }
7410 }
7411 else if(first == ':')
7412 {
7413 _c4dbgp("mapblck[QMRK]: start new block map as key (!), empty key");
7414 addrem_flags(RKCL, QMRK);
7415 _handle_annotations_before_start_mapblck_as_key();
7416 m_evt_handler->begin_map_key_block();
7417 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7418 m_evt_handler->set_key_scalar_plain_empty();
7419 _line_progressed(1);
7420 _maybe_skip_whitespace_tokens();
7421 _set_indentation(startindent);
7422 // keep the child state on RVAL
7423 addrem_flags(RVAL, RKCL);
7424 }
7425 else if(first == '*')
7426 {
7427 csubstr ref = _scan_ref_map();
7428 _c4dbgpf("mapblck[QMRK]: key ref! {}", prs_(ref));
7429 addrem_flags(RKCL, QMRK);
7430 if(!_maybe_scan_following_colon())
7431 {
7432 _c4dbgp("mapblck[QMRK]: set ref as key");
7433 _handle_keyref(ref);
7434 }
7435 else
7436 {
7437 _c4dbgp("mapblck[QMRK]: start new block map as key (!), set ref as key");
7438 _handle_annotations_before_start_mapblck_as_key();
7439 m_evt_handler->begin_map_key_block();
7440 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7441 _handle_keyref(ref);
7442 _set_indentation(startindent);
7443 // keep the child state on RVAL
7444 addrem_flags(RVAL, RKCL|QMRK);
7445 }
7446 _maybe_skip_whitespace_tokens();
7447 }
7448 else if(first == '&')
7449 {
7450 csubstr anchor = _scan_anchor();
7451 _c4dbgpf("mapblck[QMRK]: key anchor! {}", prs_(anchor));
7452 _add_annotation(&m_pending_anchors, anchor, startindent, startline);
7453 }
7454 else if(first == '!')
7455 {
7456 csubstr tag = _scan_tag();
7457 _c4dbgpf("mapblck[QMRK]: key tag! {}", prs_(tag));
7458 _add_annotation(&m_pending_tags, tag, startindent, startline);
7459 }
7460 else if(first == '-')
7461 {
7462 _c4dbgp("mapblck[QMRK]: maybe seq or doc?");
7463 if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7464 {
7465 _c4dbgp("mapblck[QMRK]: start child seqblck (!)");
7466 addrem_flags(RKCL, QMRK);
7467 _handle_annotations_before_blck_key_scalar();
7468 m_evt_handler->begin_seq_key_block();
7469 addrem_flags(RVAL|RSEQ, RMAP|RKCL);
7470 _set_indentation(startindent);
7471 _line_progressed(1);
7472 }
7473 else
7474 {
7475 _c4dbgp("mapblck[QMRK]: end+start doc");
7476 _c4assert(_is_doc_begin_token(m_evt_handler->m_curr->line_contents.rem));
7477 _start_doc_suddenly();
7478 _line_progressed(3);
7479 }
7480 _maybe_skip_whitespace_tokens();
7481 return false; // finish mapblck
7482 }
7483 else if(first == '[')
7484 {
7485 _c4dbgp("mapblck[QMRK]: start child seqflow (!)");
7486 addrem_flags(RKCL, QMRK);
7487 _handle_annotations_before_blck_key_scalar();
7488 m_evt_handler->begin_seq_key_flow();
7489 addrem_flags(RVAL|RSEQ|RFLOW, RMAP|RKCL|RBLCK);
7490 _set_indentation(m_evt_handler->m_parent->indref + 1);
7491 _line_progressed(1);
7492 return false; // finish mapblck
7493 }
7494 else if(first == '{')
7495 {
7496 _c4dbgp("mapblck[QMRK]: start child mapflow (!)");
7497 addrem_flags(RKCL, QMRK);
7498 _handle_annotations_before_blck_key_scalar();
7499 m_evt_handler->begin_map_key_flow();
7500 addrem_flags(RKEY|RFLOW, RVAL|RKCL|RBLCK);
7501 _set_indentation(m_evt_handler->m_parent->indref + 1);
7502 _line_progressed(1);
7503 return false; // finish mapblck
7504 }
7505 else if(first == '?')
7506 {
7507 _c4dbgpf("mapblck[QMRK]: another QMRK '?'. ind={} indref={}", startindent, m_evt_handler->m_curr->indref);
7508 RYML_ASSERT_PARSE_CB_(callbacks(), startindent > m_evt_handler->m_curr->indref, m_evt_handler->m_curr->pos);
7509 _c4dbgp("mapblck[QMRK]: ? indent gt - start child mapblck (!)");
7510 addrem_flags(RKCL, QMRK);
7511 _handle_annotations_before_blck_key_scalar();
7512 m_evt_handler->begin_map_key_block();
7513 addrem_flags(QMRK, RKCL);
7514 _set_indentation(startindent);
7515 // indentation_lt() should be handled elsewhere
7516 _line_progressed(1);
7517 _maybe_skipchars(' ');
7518 if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7519 {
7520 _c4dbgp("mapblck[RVAL]: seqblck starts after ?");
7521 addrem_flags(RKCL, QMRK);
7522 m_evt_handler->begin_seq_key_block();
7523 addrem_flags(RSEQ|RVAL, RMAP|RKCL);
7524 _save_indentation();
7525 _line_progressed(1);
7526 _maybe_skipchars(' ');
7527 return false;
7528 }
7529 }
7530 else
7531 {
7532 _c4err("parse error");
7533 }
7534 return true; // continue in mapblck
7535}
7536
7537
7538//-----------------------------------------------------------------------------
7539
7540// return true if we should remain in map_block
7541template<class EventHandler>
7542bool ParseEngine<EventHandler>::_handle_map_block_rkcl()
7543{
7544 //
7545 // handle indentation
7546 //
7547 if(m_evt_handler->m_curr->at_line_beginning())
7548 {
7549 if(m_evt_handler->m_curr->indentation_eq())
7550 {
7551 _c4dbgpf("mapblck[RKCL]: skip {} from indref", m_evt_handler->m_curr->indref);
7552 _line_progressed(m_evt_handler->m_curr->indref);
7553 if(!m_evt_handler->m_curr->line_contents.rem.len)
7554 return true; // continue in mapblck
7555 }
7556 else if C4_UNLIKELY(m_evt_handler->m_curr->indentation_lt())
7557 {
7558 _c4err("invalid indentation");
7559 }
7560 }
7561 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
7562 _c4dbgpf("mapblck[RKCL]: '{}'", first);
7563 if(first == ':')
7564 {
7565 _c4dbgp("mapblck[RKCL]: found the colon");
7566 _line_progressed(1);
7567 _maybe_skipchars(' ');
7568 #if defined(__GNUC__) && ( \
7569 ((__GNUC__ >= 12) && ((C4_WORDSIZE == 4) || defined(C4_CPU_S390_X) || defined(C4_CPU_PPC64))) \
7570 || \
7571 (__GNUC__ == 16 && defined(C4_CPU_X86_64)))
7572 C4_DONT_OPTIMIZE(m_evt_handler->m_curr->line_contents.rem);
7573 #endif
7574 // sequence is valid after the RKCL ':'
7575 if(!_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7576 {
7577 addrem_flags(RVAL, RKCL);
7578 return true; // continue in mapblck
7579 }
7580 else
7581 {
7582 _c4dbgp("mapblck[RKCL]: start val seqblck");
7583 addrem_flags(RNXT, RKCL);
7584 m_evt_handler->begin_seq_val_block();
7585 addrem_flags(RSEQ|RVAL, RMAP|RNXT);
7586 _save_indentation();
7587 _line_progressed(1);
7588 _maybe_skipchars(' ');
7589 return false; // finish mapblck
7590 }
7591 }
7592 else if(first == '?')
7593 {
7594 _c4dbgp("mapblck[RKCL]: got '?'. val was empty");
7595 m_evt_handler->set_val_scalar_plain_empty();
7596 m_evt_handler->add_sibling();
7597 addrem_flags(QMRK, RKCL);
7598 _line_progressed(1);
7599 _maybe_skipchars(' ');
7600 if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7601 {
7602 _c4dbgp("mapblck[RKCL]: seqblck starts after ?");
7603 addrem_flags(RKCL, QMRK);
7604 m_evt_handler->begin_seq_key_block();
7605 addrem_flags(RSEQ|RVAL, RMAP|QMRK);
7606 _save_indentation();
7607 _line_progressed(1);
7608 _maybe_skipchars(' ');
7609 return false;
7610 }
7611 }
7612 else if(first == '-')
7613 {
7614 if(m_evt_handler->m_curr->indref == 0 || m_evt_handler->m_curr->line_contents.indentation == 0 || _is_doc_begin_token(m_evt_handler->m_curr->line_contents.rem))
7615 {
7616 _c4dbgp("mapblck[RKCL]: end+start doc");
7617 RYML_CHECK_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, _is_doc_begin_token(m_evt_handler->m_curr->line_contents.rem), m_evt_handler->m_curr->pos);
7618 _start_doc_suddenly();
7619 _line_progressed(3);
7620 _maybe_skip_whitespace_tokens();
7621 return false; // finish mapblck
7622 }
7623 else
7624 {
7625 _c4err("parse error"); // LCOV_EXCL_LINE
7626 }
7627 }
7628 else if(first == '.')
7629 {
7630 _c4dbgp("mapblck[RKCL]: maybe end doc?");
7631 csubstr rs = m_evt_handler->m_curr->line_contents.rem.sub(1);
7632 if(rs == ".." || rs.begins_with(".. "))
7633 {
7634 _c4dbgp("mapblck[RKCL]: end+start doc");
7635 _end_doc_suddenly();
7636 _line_progressed(3);
7637 _maybe_skip_whitespace_tokens();
7638 _check_doc_end_tokens();
7639 return false; // finish mapblck
7640 }
7641 else
7642 {
7643 _c4err("parse error"); // LCOV_EXCL_LINE
7644 }
7645 }
7646 else/* if(m_was_inside_qmrk) */
7647 {
7648 _c4dbgp("mapblck[RKCL]: missing :");
7649 if C4_UNLIKELY(!m_evt_handler->m_curr->indentation_eq())
7650 _c4err("parse error"); // LCOV_EXCL_LINE
7651 m_evt_handler->set_val_scalar_plain_empty();
7652 m_evt_handler->add_sibling();
7653 addrem_flags(RKEY, RKCL);
7654 }
7655 return true;
7656}
7657
7658
7659//-----------------------------------------------------------------------------
7660
7661template<class EventHandler>
7662void ParseEngine<EventHandler>::_handle_unk_json()
7663{
7664 _c4dbgpf("handle_unk_json indref={} target={}", m_evt_handler->m_curr->indref, m_evt_handler->m_curr->node_id);
7665
7666 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT|RSEQ|RMAP), m_evt_handler->m_curr->pos);
7667 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RTOP), m_evt_handler->m_curr->pos);
7668
7669 _maybe_skip_comment();
7670 csubstr rem = m_evt_handler->m_curr->line_contents.rem;
7671 if(!rem.len)
7672 return;
7673
7674 size_t pos = rem.first_not_of(" \t");
7675 if(pos)
7676 {
7677 pos = pos != npos ? pos : rem.len;
7678 _c4dbgpf("skipping indentation of {}", pos);
7679 _line_progressed(pos);
7680 rem = m_evt_handler->m_curr->line_contents.rem;
7681 if(!rem.len)
7682 return;
7683 _c4dbgpf("rem is now {}", prs_(rem));
7684 }
7685
7686 if(rem.begins_with('['))
7687 {
7688 _c4dbgp("it's a seq");
7689 _check_trailing_doc_token();
7690 _maybe_begin_doc();
7691 m_evt_handler->begin_seq_val_flow();
7692 addrem_flags(RSEQ|RFLOW|RVAL, RUNK|RTOP|RDOC);
7693 _set_indentation(m_evt_handler->m_curr->line_contents.current_col(rem));
7694 m_doc_empty = false;
7695 _line_progressed(1);
7696 }
7697 else if(rem.begins_with('{'))
7698 {
7699 _c4dbgp("it's a map");
7700 _check_trailing_doc_token();
7701 _maybe_begin_doc();
7702 m_evt_handler->begin_map_val_flow();
7703 addrem_flags(RMAP|RFLOW|RKEY, RVAL|RTOP|RUNK|RDOC);
7704 m_doc_empty = false;
7705 _set_indentation(m_evt_handler->m_curr->line_contents.current_col(rem));
7706 _line_progressed(1);
7707 }
7708 else if(_handle_bom())
7709 {
7710 _c4dbgp("byte order mark");
7711 }
7712 else
7713 {
7714 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(SSCL), m_evt_handler->m_curr->pos);
7715 _maybe_skip_whitespace_tokens();
7716 csubstr s = m_evt_handler->m_curr->line_contents.rem;
7717 if(!s.len)
7718 return;
7719 const size_t startindent = m_evt_handler->m_curr->line_contents.indentation; // save
7720 const char first = s.str[0];
7721 ScannedScalar sc;
7722 if(first == '"')
7723 {
7724 _c4dbgp("runk_json: scanning double-quoted scalar");
7725 _check_trailing_doc_token();
7726 _maybe_begin_doc();
7727 add_flags(RDOC);
7728 m_doc_empty = false;
7729 sc = _scan_scalar_dquot();
7730 csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
7731 if(!_maybe_scan_following_colon())
7732 {
7733 _c4dbgp("runk_json: set as val");
7734 _handle_annotations_before_blck_val_scalar();
7735 m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
7736 }
7737 else
7738 {
7739 _c4err("parse error");
7740 }
7741 }
7742 else if(_scan_scalar_plain_unk(&sc))
7743 {
7744 _c4dbgp("runk_json: got a plain scalar");
7745 _check_trailing_doc_token();
7746 _maybe_begin_doc();
7747 add_flags(RDOC);
7748 m_doc_empty = false;
7749 if(!_maybe_scan_following_colon())
7750 {
7751 _c4dbgp("runk_json: set as val");
7752 _handle_annotations_before_blck_val_scalar();
7753 csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, startindent);
7754 m_evt_handler->set_val_scalar_plain(maybe_filtered);
7755 }
7756 else
7757 {
7758 _c4err("parse error"); // LCOV_EXCL_LINE
7759 }
7760 }
7761 else
7762 {
7763 _c4err("parse error"); // LCOV_EXCL_LINE
7764 }
7765 }
7766}
7767
7768
7769//-----------------------------------------------------------------------------
7770
7771template<class EventHandler>
7772void ParseEngine<EventHandler>::_handle_unk()
7773{
7774 _c4dbgpf("handle_unk indref={} target={}", m_evt_handler->m_curr->indref, m_evt_handler->m_curr->node_id);
7775
7776 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RNXT|RSEQ|RMAP), m_evt_handler->m_curr->pos);
7777 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RTOP), m_evt_handler->m_curr->pos);
7778
7779 _maybe_skipchars(' ');
7780 _maybe_skip_comment();
7781
7782 if(!m_evt_handler->m_curr->line_contents.rem.len)
7783 return;
7784
7785 _c4dbgpf("runk: rem is now {}", prs_(m_evt_handler->m_curr->line_contents.rem));
7786
7787 if(m_evt_handler->m_curr->line_contents.indentation == 0u && (m_evt_handler->m_curr->at_line_beginning() || (m_bom_len && (m_evt_handler->m_curr->pos.line == m_bom_line))))
7788 {
7789 _c4dbgpf("runk: rtop: zero indent + at line begin. offset={}", m_evt_handler->m_curr->pos.offset);
7790 _c4dbgp("runk: check BOM");
7791 if(_handle_bom())
7792 {
7793 m_bom_line = m_evt_handler->m_curr->pos.line;
7794 _c4dbgpf("runk: byte order mark! line={} offset={}", m_bom_line, m_evt_handler->m_curr->pos.offset);
7795 return;
7796 }
7797 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
7798 _c4dbgpf("runk: rtop: first={}", _c4prc(first));
7799 if(first == '-')
7800 {
7801 _c4dbgp("runk: rtop: suspecting doc");
7802 if(_is_doc_begin_token(m_evt_handler->m_curr->line_contents.rem))
7803 {
7804 _c4dbgp("runk: rtop: begin doc");
7805 _maybe_end_doc();
7806 _begin2_doc_expl();
7807 _set_indentation(0);
7808 addrem_flags(RDOC|RUNK, NDOC);
7809 _line_progressed(3u);
7810 _maybe_skip_whitespace_tokens();
7811 return;
7812 }
7813 }
7814 else if(first == '.')
7815 {
7816 _c4dbgp("runk: rtop: suspecting doc end");
7817 if(_is_doc_end_token(m_evt_handler->m_curr->line_contents.rem))
7818 {
7819 _c4dbgp("runk: rtop: end doc");
7820 if(has_any(RDOC))
7821 {
7822 _end2_doc_expl();
7823 }
7824 else
7825 {
7826 _c4dbgp("runk: rtop: ignore end doc");
7827 }
7828 addrem_flags(NDOC|RUNK, RDOC);
7829 _line_progressed(3u);
7830 _maybe_skip_whitespace_tokens();
7831 _check_doc_end_tokens();
7832 return;
7833 }
7834 }
7835 else if(first == '%')
7836 {
7837 _c4dbgpf("directive: {}", m_evt_handler->m_curr->line_contents.rem);
7838 if C4_UNLIKELY(has_any(RDOC) || (!m_doc_empty && has_none(NDOC)))
7839 _c4err("need document footer before directives");
7840 _handle_directive(m_evt_handler->m_curr->line_contents.rem);
7841 return;
7842 }
7843 }
7844
7845 /* no else-if! */
7846
7847 size_t startindent = m_evt_handler->m_curr->line_contents.indentation;
7848 size_t remindent = m_evt_handler->m_curr->line_contents.current_col(m_evt_handler->m_curr->line_contents.rem);
7849 if(m_bom_len)
7850 {
7851 _c4dbgpf("runk: prev BOMlen={}", m_bom_len);
7852 if(m_evt_handler->m_curr->pos.line == m_bom_line)
7853 {
7854 _c4dbgpf("runk: BOM remindent={} offset={}", remindent, m_evt_handler->m_curr->pos.offset);
7855 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, remindent >= m_bom_len, m_evt_handler->m_curr->pos);
7856 remindent -= m_bom_len;
7857 }
7858 else
7859 {
7860 m_bom_len = 0;
7861 }
7862 }
7863
7864 size_t startcol = _handle_block_skip_leading_whitespace();
7865 const char first = m_evt_handler->m_curr->line_contents.rem.str[0];
7866
7867 if(first == '[')
7868 {
7869 _c4dbgp("runk: flow seq?");
7870 _handle_unk_begin_doc();
7871 if C4_LIKELY( ! _annotations_require_key_container())
7872 {
7873 _c4dbgp("runk: it's a seq, flow");
7874 _handle_annotations_before_blck_val_scalar();
7875 m_evt_handler->begin_seq_val_flow();
7876 addrem_flags(RSEQ|RFLOW|RVAL, RUNK|RTOP|RDOC);
7877 _set_indentation(0);
7878 }
7879 else
7880 {
7881 _c4dbgp("runk: start new block map, set flow seq as key (!)");
7882 _handle_annotations_before_start_mapblck(m_evt_handler->m_curr->pos.line);
7883 m_evt_handler->begin_map_val_block();
7884 addrem_flags(RMAP|RBLCK|RKEY, RUNK|RTOP|RDOC);
7885 _handle_annotations_and_indentation_after_start_mapblck(remindent, m_evt_handler->m_curr->pos.line);
7886 m_evt_handler->begin_seq_key_flow();
7887 addrem_flags(RSEQ|RFLOW|RVAL, RMAP|RBLCK|RKEY);
7888 _set_indentation(0);
7889 }
7890 _line_progressed(1);
7891 }
7892 else if(first == '{')
7893 {
7894 _c4dbgp("runk: flow map?");
7895 _handle_unk_begin_doc();
7896 if C4_LIKELY( ! _annotations_require_key_container())
7897 {
7898 _c4dbgp("runk: it's a map, flow");
7899 _handle_annotations_before_blck_val_scalar();
7900 m_evt_handler->begin_map_val_flow();
7901 addrem_flags(RMAP|RFLOW|RKEY, RVAL|RTOP|RUNK|RDOC);
7902 _set_indentation(0);
7903 }
7904 else
7905 {
7906 _c4dbgp("runk: start new block map, set flow map as key (!)");
7907 _handle_annotations_before_start_mapblck(m_evt_handler->m_curr->pos.line);
7908 m_evt_handler->begin_map_val_block();
7909 addrem_flags(RMAP|RBLCK|RKEY, RUNK|RTOP|RDOC);
7910 _handle_annotations_and_indentation_after_start_mapblck(remindent, m_evt_handler->m_curr->pos.line);
7911 m_evt_handler->begin_map_key_flow();
7912 addrem_flags(RMAP|RFLOW, RBLCK);
7913 _set_indentation(0);
7914 }
7915 _line_progressed(1);
7916 }
7917 else if(first == '-' && _is_blck_token(m_evt_handler->m_curr->line_contents.rem))
7918 {
7919 _c4dbgp("runk: it's a seq, block");
7920 if C4_UNLIKELY(!m_evt_handler->m_curr->at_first_token())
7921 startindent = _handle_unk_check_left_tokens(startindent, m_evt_handler->m_curr->pos.col, /*skip_annotations*/false);
7922 _handle_unk_begin_doc();
7923 _handle_annotations_before_blck_val_scalar();
7924 m_evt_handler->begin_seq_val_block();
7925 addrem_flags(RSEQ|RBLCK|RVAL, RNXT|RTOP|RUNK|RDOC);
7926 _set_indentation(startindent);
7927 _line_progressed(1);
7928 _maybe_skipchars(' ');
7929 }
7930 else if(first == '?' && _is_blck_token(m_evt_handler->m_curr->line_contents.rem))
7931 {
7932 _c4dbgp("runk: it's a map + this key is complex");
7933 if C4_UNLIKELY(!m_evt_handler->m_curr->at_first_token())
7934 startindent = _handle_unk_check_left_tokens(startindent, m_evt_handler->m_curr->pos.col, /*skip_annotations*/false);
7935 _handle_block_check_leading_tabs(startcol);
7936 _handle_unk_begin_doc();
7937 _handle_annotations_before_blck_val_scalar();
7938 m_evt_handler->begin_map_val_block();
7939 addrem_flags(RMAP|RBLCK|QMRK, RKEY|RVAL|RTOP|RUNK|RDOC);
7940 _set_indentation(startindent);
7941 _line_progressed(1);
7942 _maybe_skipchars(' ');
7943 if(_is_blck_seq_token_maybe(m_evt_handler->m_curr->line_contents.rem))
7944 {
7945 _c4dbgp("runk: seqblck key starts after ?");
7946 addrem_flags(RKCL, QMRK);
7947 m_evt_handler->begin_seq_key_block();
7948 addrem_flags(RSEQ|RVAL, RMAP|RKCL);
7949 _save_indentation();
7950 _line_progressed(1);
7951 _maybe_skipchars(' ');
7952 }
7953 }
7954 else if(first == ':' && _is_blck_token(m_evt_handler->m_curr->line_contents.rem))
7955 {
7956 if(m_doc_empty || (m_pending_anchors.num_entries | m_pending_tags.num_entries))
7957 {
7958 _c4dbgp("runk: it's a map with an empty key");
7959 if C4_UNLIKELY(!m_evt_handler->m_curr->at_first_token())
7960 startindent = _handle_unk_check_left_tokens(startindent, m_evt_handler->m_curr->pos.col);
7961 _handle_block_check_leading_tabs(startcol);
7962 const size_t startline = m_evt_handler->m_curr->pos.line; // save
7963 _handle_unk_begin_doc();
7964 _handle_annotations_before_start_mapblck(startline);
7965 _handle_colon();
7966 m_evt_handler->begin_map_val_block();
7967 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
7968 m_evt_handler->set_key_scalar_plain_empty();
7969 _set_indentation(startindent);
7970 }
7971 else
7972 {
7973 _c4err("block colon cannot occur on a new line unless ? is used");
7974 }
7975 addrem_flags(RMAP|RBLCK|RVAL, RTOP|RUNK|RDOC);
7976 _line_progressed(1);
7977 _maybe_skip_whitespace_tokens();
7978 }
7979 else if(first == '&')
7980 {
7981 csubstr anchor = _scan_anchor();
7982 _c4dbgpf("anchor! {}", prs_(anchor));
7983 const size_t line = m_evt_handler->m_curr->pos.line;
7984 _handle_unk_begin_doc();
7985 _add_annotation(&m_pending_anchors, anchor, remindent, line);
7986 _set_indentation(0);
7987 }
7988 else if(first == '*')
7989 {
7990 csubstr ref = _scan_ref_map();
7991 _c4dbgpf("runk: ref! {}", prs_(ref));
7992 _handle_unk_begin_doc();
7993 if(!_maybe_scan_following_colon())
7994 {
7995 _c4dbgp("runk: set val ref");
7996 _handle_valref(ref);
7997 }
7998 else
7999 {
8000 _c4dbgp("runk: start new block map, set ref as key");
8001 _handle_block_check_leading_tabs(startcol);
8002 const size_t startline = m_evt_handler->m_curr->pos.line; // save
8003 _handle_annotations_before_start_mapblck(startline);
8004 m_evt_handler->begin_map_val_block();
8005 _handle_keyref(ref);
8006 _maybe_skip_whitespace_tokens();
8007 _set_indentation(0);
8008 addrem_flags(RMAP|RBLCK|RVAL, RTOP|RUNK|RDOC);
8009 }
8010 }
8011 else if(first == '!')
8012 {
8013 csubstr tag_orig;
8014 csubstr tag = _scan_tag(&tag_orig);
8015 _c4dbgpf("runk: val tag! {}", prs_(tag));
8016 // we need to buffer the tags, as there may be two
8017 // consecutive tags in here
8018 const size_t indentation = m_evt_handler->m_curr->line_contents.current_col(m_evt_handler->m_curr->line_contents.rem);
8019 const size_t line = m_evt_handler->m_curr->pos.line;
8020 _add_annotation(&m_pending_tags, tag, indentation, line, tag_orig);
8021 }
8022 else
8023 {
8024 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(SSCL), m_evt_handler->m_curr->pos);
8025 const size_t startscalar = _handle_block_get_whitespace_mark();
8026 const size_t startline = m_evt_handler->m_curr->pos.line; // save
8027 auto beginmap = [&](size_t startindent_){
8028 if C4_UNLIKELY(m_evt_handler->m_curr->pos.line > startline)
8029 _c4err("multiline scalars cannot be used as implicit keys");
8030 _handle_block_check_leading_tabs(startcol, startscalar);
8031 _handle_annotations_before_start_mapblck(startline);
8032 _handle_colon();
8033 m_evt_handler->begin_map_val_block();
8034 _handle_annotations_and_indentation_after_start_mapblck(startindent_, startline);
8035 };
8036 auto after_beginmap = [&](size_t startindent_){
8037 _maybe_skip_whitespace_tokens();
8038 _set_indentation(startindent_);
8039 addrem_flags(RMAP|RBLCK|RVAL, RTOP|RUNK|RDOC);
8040 };
8041 if(first == '|')
8042 {
8043 _c4dbgp("runk: block-literal scalar");
8044 _handle_unk_begin_doc();
8045 ScannedBlock sb;
8046 _scan_block(&sb, startindent);
8047 _handle_annotations_before_blck_val_scalar();
8048 csubstr maybe_filtered = _maybe_filter_val_scalar_literal(sb);
8049 m_evt_handler->set_val_scalar_literal(maybe_filtered);
8050 }
8051 else if(first == '>')
8052 {
8053 _c4dbgp("runk: block-folded scalar");
8054 _handle_unk_begin_doc();
8055 ScannedBlock sb;
8056 _scan_block(&sb, startindent);
8057 _handle_annotations_before_blck_val_scalar();
8058 csubstr maybe_filtered = _maybe_filter_val_scalar_folded(sb);
8059 m_evt_handler->set_val_scalar_folded(maybe_filtered);
8060 }
8061 else if(first == '\'')
8062 {
8063 _c4dbgp("runk: single-quoted scalar");
8064 _handle_unk_begin_doc();
8065 bool firsttoken = m_evt_handler->m_curr->at_first_token();
8066 size_t col = m_evt_handler->m_curr->pos.col;
8067 ScannedScalar sc = _scan_scalar_squot();
8068 if(!_maybe_scan_following_colon())
8069 {
8070 _c4dbgp("runk: set as val");
8071 _handle_annotations_before_blck_val_scalar();
8072 csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
8073 m_evt_handler->set_val_scalar_squoted(maybe_filtered);
8074 }
8075 else
8076 {
8077 _c4dbgp("runk: start new block map, set single-quoted scalar as key");
8078 if C4_UNLIKELY(m_evt_handler->m_curr->pos.line > startline)
8079 _c4err("multiline key");
8080 if(!firsttoken)
8081 startindent = _handle_unk_check_left_tokens(startindent, col);
8082 beginmap(startindent);
8083 csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
8084 m_evt_handler->set_key_scalar_squoted(maybe_filtered);
8085 after_beginmap(startindent);
8086 }
8087 }
8088 else if(first == '"')
8089 {
8090 _c4dbgp("runk: double-quoted scalar");
8091 _handle_unk_begin_doc();
8092 bool firsttoken = m_evt_handler->m_curr->at_first_token();
8093 size_t col = m_evt_handler->m_curr->pos.col;
8094 ScannedScalar sc = _scan_scalar_dquot();
8095 if(!_maybe_scan_following_colon())
8096 {
8097 _c4dbgp("runk: set as val");
8098 _handle_annotations_before_blck_val_scalar();
8099 csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
8100 m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
8101 }
8102 else
8103 {
8104 _c4dbgp("runk: start new block map, set double-quoted scalar as key");
8105 if C4_UNLIKELY(m_evt_handler->m_curr->pos.line > startline)
8106 _c4err("multiline key");
8107 if(!firsttoken)
8108 startindent = _handle_unk_check_left_tokens(startindent, col);
8109 beginmap(startindent);
8110 csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
8111 m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
8112 after_beginmap(startindent);
8113 }
8114 }
8115 else
8116 {
8117 bool firsttoken = m_evt_handler->m_curr->at_first_token();
8118 size_t col = m_evt_handler->m_curr->pos.col;
8119 ScannedScalar sc;
8120 if(_scan_scalar_plain_unk(&sc))
8121 {
8122 _c4dbgp("runk: plain scalar");
8123 _handle_unk_begin_doc();
8124 if(!_maybe_scan_following_colon())
8125 {
8126 _c4dbgp("runk: set as val");
8127 _handle_annotations_before_blck_val_scalar();
8128 csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, startindent);
8129 m_evt_handler->set_val_scalar_plain(maybe_filtered);
8130 }
8131 else
8132 {
8133 _c4dbgp("runk: start new block map, set plain scalar as key");
8134 // there is already a check to multiline inside
8135 // _scan_scalar_plain_unk(), so we don't need to
8136 // throw an error here. but let's be safe by
8137 // asserting the assumption:
8138 _c4assert(m_evt_handler->m_curr->pos.line == startline);
8139 if(!firsttoken)
8140 startindent = _handle_unk_check_left_tokens(startindent, col);
8141 beginmap(startindent);
8142 csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, startindent);
8143 m_evt_handler->set_key_scalar_plain(maybe_filtered);
8144 after_beginmap(startindent);
8145 }
8146 }
8147 else
8148 {
8149 _c4err("parse error"); // LCOV_EXCL_LINE
8150 }
8151 }
8152 }
8153
8154 if(m_bom_len && has_none(RUNK))
8155 {
8156 _c4dbgpf("runk: BOMlen={} BOMline={} now={} at_end={}", m_bom_len, m_bom_line, m_evt_handler->m_curr->pos.line, !m_evt_handler->m_curr->line_contents.rem.len);
8157 if(m_evt_handler->m_curr->pos.line != m_bom_line || !m_evt_handler->m_curr->line_contents.rem.len)
8158 {
8159 _c4dbgp("runk: clear BOMlen");
8160 m_bom_len = 0;
8161 }
8162 }
8163}
8164
8165template<class EventHandler>
8166void ParseEngine<EventHandler>::_handle_unk_begin_doc()
8167{
8168 _c4dbgp("runk: begin doc");
8169 _check_trailing_doc_token();
8170 _maybe_begin_doc();
8171 add_flags(RDOC);
8172 m_doc_empty = false;
8173}
8174
8175template<class EventHandler>
8176size_t ParseEngine<EventHandler>::_handle_unk_check_left_tokens(size_t realindent, size_t col, bool skip_annotations)
8177{
8178 _c4assert(col >= 1);
8179 col -= 1;
8180 _c4assert(col >= m_bom_len);
8181 csubstr s = m_evt_handler->m_curr->line_contents.full.range(m_bom_len, col);
8182 size_t pos = 0;
8183 _c4dbgpf("runk: check left tokens: s={}", prs_(s, /*escape*/true));
8184 if(skip_annotations)
8185 {
8186 _handle_unk_get_first_non_pending_token_pos(s, &realindent, &pos);
8187 _c4dbgpf("runk: skip annotations: realindent={} pos={}", realindent, pos);
8188 }
8189 size_t firstns = s.first_not_of(' ', pos);
8190 if(firstns == npos)
8191 firstns = s.len;
8192 _c4dbgpf("runk: check left tokens:\n"
8193 " tokens={} skipped={}\n"
8194 " bomlen={} first={} col={}\n"
8195 " (bomlen+first)={} vs {}=col\n"
8196 " startindent={} lineindent={}"
8197 , prs_(s, /*escape*/true), prs_(s.sub(firstns), /*escape*/true)
8198 , m_bom_len, firstns, col
8199 , m_bom_len+firstns, col,
8200 realindent, m_evt_handler->m_curr->line_contents.indentation);
8201 if(m_bom_len + firstns != col)
8202 _c4err("parse error");
8203 if(!skip_annotations)
8204 realindent = firstns;
8205 _c4dbgpf("runk: pos={} firstns={} -> realindent={}", pos, firstns, realindent);
8206 return realindent;
8207}
8208
8209
8210/** skip annotations which are pending on the same line */
8211template<class EventHandler>
8212void ParseEngine<EventHandler>::_handle_unk_get_first_non_pending_token_pos(csubstr s, size_t *indent, size_t *first_non_token_pos)
8213{
8214 csubstr first, second;
8215 uint32_t total = _get_annotations_same_line(s, &first, &second);
8216 _c4dbgpf("runk: before skip: {}", prs_(s, true));
8217 size_t pos = s.first_not_of(" \t");
8218 if(pos == npos)
8219 pos = s.len;
8220 if(!total)
8221 {
8222 *indent = *first_non_token_pos = pos;
8223 return;
8224 }
8225 _c4assert(!s.sub(pos).begins_with_any(" \t"));
8226 _c4dbgpf("runk: after skip leading {} whitespace: {}", pos, prs_(s.sub(pos), true));
8227 _c4dbgpf("runk: first annotation: {}", first);
8228 _c4assert(first.len);
8229 _c4assert(first.is_sub(s));
8230 _c4assert(first.is_sub(s.sub(pos)));
8231 _c4assert(s.sub(pos).begins_with(first));
8232 *indent = pos;
8233 pos += first.len;
8234 _c4dbgpf("runk: after skip first annotation: pos={} {}", pos, prs_(s.sub(pos), true));
8235 if(total > 1)
8236 {
8237 _c4dbgpf("runk: second annotation: {}", second);
8238 _c4assert(total == 2);
8239 _c4assert(second.len);
8240 _c4assert(second.is_sub(s));
8241 _c4assert(second.is_sub(s.sub(pos)));
8242 csubstr spos = s.sub(pos);
8243 size_t more = spos.first_not_of(" \t");
8244 _c4assert(more != npos); // because the annotations are on the same line
8245 _c4dbgpf("runk: next nonspace: {}", pos + more);
8246 pos += more;
8247 _c4dbgpf("runk: after skip annotation whitespace: pos={} {}", pos, prs_(s.sub(pos), true));
8248 _c4assert(s.sub(pos).begins_with(second));
8249 pos += second.len;
8250 _c4dbgpf("runk: after skip annotation 2: pos={} {}", pos, prs_(s.sub(pos), true));
8251 }
8252 *first_non_token_pos = pos;
8253}
8254
8255
8256template<class EventHandler>
8257uint32_t ParseEngine<EventHandler>::_get_annotations_same_line(csubstr token_soup, csubstr *first_, csubstr *second_) const
8258{
8259 _c4assert(!m_evt_handler->m_curr->at_first_token());
8260 (void)token_soup;
8261 using EntryPtr = typename Annotation::Entry const* C4_RESTRICT;
8262 EntryPtr first = nullptr;
8263 EntryPtr second = nullptr;
8264 uint32_t total = (uint32_t)(m_pending_anchors.num_entries + m_pending_tags.num_entries);
8265 if(total)
8266 {
8267 _c4dbgpf("there are {} pending annotations: {} anchors + {} tags", total, m_pending_anchors.num_entries, m_pending_tags.num_entries);
8268 auto valid_if_same_line = [this](EntryPtr entry){
8269 _c4dbgpf("pending: {} indent={} line={} vs currline={}", maybe_null_str_(entry->str), entry->indentation, entry->line, m_evt_handler->m_curr->pos.line);
8270 return (entry->line == m_evt_handler->m_curr->pos.line) ? entry : nullptr;
8271 };
8272 // now select annotations only on the same line
8273 total = 0;
8274 for(size_t i = 0; i < m_pending_anchors.num_entries; ++i)
8275 total += !!valid_if_same_line(&m_pending_anchors.annotations[i]);
8276 for(size_t i = 0; i < m_pending_tags.num_entries; ++i)
8277 total += !!valid_if_same_line(&m_pending_tags.annotations[i]);
8278 _c4dbgpf("{} annotations on same line", total);
8279 _c4assert(total > 0); // because this function is only called
8280 // while not at the first token. That
8281 // means we must have same-line
8282 // annotations.
8283 auto get_first_on_same_line = [this](EntryPtr not_this_one){
8284 for(size_t i = 0; i < m_pending_anchors.num_entries; ++i)
8285 if(&m_pending_anchors.annotations[i] != not_this_one
8286 && m_pending_anchors.annotations[i].line == m_evt_handler->m_curr->pos.line)
8287 return &m_pending_anchors.annotations[i];
8288 for(size_t i = 0; i < m_pending_tags.num_entries; ++i)
8289 if(&m_pending_tags.annotations[i] != not_this_one
8290 && m_pending_tags.annotations[i].line == m_evt_handler->m_curr->pos.line)
8291 return &m_pending_tags.annotations[i];
8292 C4_UNREACHABLE(); // LCOV_EXCL_LINE
8293 return (EntryPtr)nullptr; // LCOV_EXCL_LINE
8294 };
8295 _c4assert(total >= 1);
8296 // assign to first
8297 first = get_first_on_same_line(nullptr);
8298 _c4assert(first);
8299 _c4dbgpf("first annotation: {} indent={} line={}", maybe_null_str_(first->str), first->indentation, first->line);
8300 if(total > 1)
8301 {
8302 _c4assert(total == 2);
8303 // assign to second
8304 second = get_first_on_same_line(first);
8305 _c4assert(second);
8306 _c4dbgpf("second annotation: {} indent={} line={}", maybe_null_str_(second->str), second->indentation, second->line);
8307 }
8308 auto extract_string = [&](EntryPtr e){
8309 // tags can be null when the arena ran out of space
8310 if(!e->str.str || e->str.begins_with_any("!<"))
8311 {
8312 csubstr tag = e->orig;
8313 _c4assert(tag.str);
8314 _c4assert(tag.len);
8315 _c4assert(tag.is_sub(token_soup));
8316 _c4dbgpf("tag: {} -> {}", maybe_null_str_(e->str), tag);
8317 return tag;
8318 }
8319 csubstr anchor = e->str;
8320 _c4assert(anchor.len);
8321 _c4assert(anchor.str);
8322 _c4assert(anchor.is_sub(token_soup));
8323 _c4assert(!anchor.begins_with('&'));
8324 _c4assert(anchor.str - token_soup.str > 0);
8325 // add back the anchor's &
8326 --anchor.str;
8327 ++anchor.len;
8328 _c4assert(anchor.begins_with('&'));
8329 _c4dbgpf("anchor: {} -> {}", e->str, anchor);
8330 return anchor;
8331 };
8332 *first_ = first ? extract_string(first) : nullptr;
8333 *second_ = second ? extract_string(second) : nullptr;
8334 if(total > 1 && (first_->str > second_->str))
8335 {
8336 csubstr tmp = *first_;
8337 *first_ = *second_;
8338 *second_ = tmp;
8339 _c4dbgpf("swap first and second: {} -> {}", *first_, *second_);
8340 }
8341 }
8342 return total;
8343}
8344
8345
8346//-----------------------------------------------------------------------------
8347
8348template<class EventHandler>
8349C4_COLD void ParseEngine<EventHandler>::_handle_usty()
8350{
8351 _c4dbgpf("handle_usty target={}", m_evt_handler->m_curr->indref, m_evt_handler->m_curr->node_id);
8352
8353 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_none(RBLCK|RFLOW), m_evt_handler->m_curr->pos);
8354
8355 #ifdef RYML_NO_COVERAGE__TO_BE_DELETED
8356 if(has_any(RNXT))
8357 {
8358 _c4dbgp("usty[RNXT]: finishing!");
8359 _end_stream();
8360 }
8361 #endif
8362
8363 _maybe_skip_comment();
8364 csubstr rem = m_evt_handler->m_curr->line_contents.rem;
8365 if(!rem.len)
8366 return;
8367
8368 size_t pos = rem.first_not_of(" \t");
8369 if(pos)
8370 {
8371 pos = pos != npos ? pos : rem.len;
8372 _c4dbgpf("skipping indentation of {}", pos);
8373 _line_progressed(pos);
8374 rem = m_evt_handler->m_curr->line_contents.rem;
8375 if(!rem.len)
8376 return;
8377 _c4dbgpf("rem is now {}", prs_(rem));
8378 }
8379
8380 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, rem.len > 0, m_evt_handler->m_curr->pos);
8381 size_t startindent = m_evt_handler->m_curr->line_contents.indentation; // save
8382 char first = rem.str[0];
8383 if(has_any(RSEQ)) // destination is a sequence
8384 {
8385 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(RMAP), m_evt_handler->m_curr->pos);
8386 _c4dbgpf("usty[RSEQ]: first='{}'", _c4prc(first));
8387 if(first == '[')
8388 {
8389 _c4dbgp("usty[RSEQ]: it's a flow seq. merging it");
8390 add_flags(RNXT);
8391 m_evt_handler->_push();
8392 addrem_flags(RFLOW|RVAL, RNXT|USTY);
8393 _set_indentation(startindent);
8394 _line_progressed(1);
8395 _maybe_skip_whitespace_tokens();
8396 }
8397 else if(first == '-' && _is_blck_token(rem))
8398 {
8399 _c4dbgp("usty[RSEQ]: it's a block seq. merging it");
8400 add_flags(RNXT);
8401 m_evt_handler->_push();
8402 addrem_flags(RBLCK|RVAL, RNXT|USTY);
8403 _set_indentation(startindent);
8404 _line_progressed(1);
8405 _maybe_skip_whitespace_tokens();
8406 }
8407 else
8408 {
8409 _c4err("can only parse a seq into an existing seq");
8410 }
8411 }
8412 else if(has_any(RMAP)) // destination is a map
8413 {
8414 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(RSEQ), m_evt_handler->m_curr->pos);
8415 _c4dbgpf("usty[RMAP]: first='{}'", _c4prc(first));
8416 if(first == '{')
8417 {
8418 _c4dbgp("usty[RMAP]: it's a flow map. merging it");
8419 add_flags(RNXT);
8420 _handle_annotations_before_blck_val_scalar();
8421 m_evt_handler->_push();
8422 addrem_flags(RMAP|RFLOW|RKEY, RNXT|USTY);
8423 _set_indentation(startindent);
8424 _line_progressed(1);
8425 _maybe_skip_whitespace_tokens();
8426 }
8427 else if(first == '?' && _is_blck_token(rem))
8428 {
8429 _c4dbgp("usty[RMAP]: it's a block map + this key is complex");
8430 add_flags(RNXT);
8431 _handle_annotations_before_blck_val_scalar();
8432 m_evt_handler->_push();
8433 addrem_flags(RMAP|RBLCK|QMRK, RNXT|USTY);
8434 _save_indentation();
8435 _line_progressed(1);
8436 _maybe_skip_whitespace_tokens();
8437 }
8438 else if(first == ':' && _is_blck_token(rem))
8439 {
8440 _c4dbgp("usty[RMAP]: it's a map with an empty key");
8441 add_flags(RNXT);
8442 _handle_annotations_before_blck_val_scalar();
8443 m_evt_handler->_push();
8444 m_evt_handler->set_key_scalar_plain_empty();
8445 addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8446 _save_indentation();
8447 _line_progressed(1);
8448 _maybe_skip_whitespace_tokens();
8449 }
8450 else if(rem.begins_with('&'))
8451 {
8452 csubstr anchor = _scan_anchor();
8453 _c4dbgpf("usty[RMAP]: anchor! {}", prs_(anchor));
8454 const size_t indentation = m_evt_handler->m_curr->line_contents.current_col(rem);
8455 const size_t line = m_evt_handler->m_curr->pos.line;
8456 _add_annotation(&m_pending_anchors, anchor, indentation, line);
8457 _set_indentation(m_evt_handler->m_curr->line_contents.current_col(rem));
8458 }
8459 else if(first == '*')
8460 {
8461 csubstr ref = _scan_ref_map();
8462 _c4dbgpf("usty[RMAP]: ref! {}", prs_(ref));
8463 if(!_maybe_scan_following_colon())
8464 {
8465 _c4err("cannot read a VAL to a map");
8466 }
8467 else
8468 {
8469 _c4dbgp("usty[RMAP]: start new block map, set ref as key");
8470 const size_t startline = m_evt_handler->m_curr->pos.line; // save
8471 add_flags(RNXT);
8472 _handle_annotations_before_start_mapblck(startline);
8473 m_evt_handler->_push();
8474 _handle_keyref(ref);
8475 _maybe_skip_whitespace_tokens();
8476 _set_indentation(startindent);
8477 addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8478 }
8479 }
8480 else if(first == '!')
8481 {
8482 csubstr tag = _scan_tag();
8483 _c4dbgpf("usty[RMAP]: val tag! {}", prs_(tag));
8484 // we need to buffer the tags, as there may be two
8485 // consecutive tags in here
8486 const size_t indentation = m_evt_handler->m_curr->line_contents.current_col(rem);
8487 const size_t line = m_evt_handler->m_curr->pos.line;
8488 _add_annotation(&m_pending_tags, tag, indentation, line);
8489 }
8490 else if(first == '[' || (first == '-' && _is_blck_token(rem)))
8491 {
8492 _c4err("cannot parse a seq into an existing map");
8493 }
8494 else
8495 {
8496 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(SSCL), m_evt_handler->m_curr->pos);
8497 startindent = m_evt_handler->m_curr->line_contents.indentation; // save
8498 const size_t startline = m_evt_handler->m_curr->pos.line; // save
8499 ScannedScalar sc;
8500 _c4dbgpf("usty[RMAP]: maybe scalar. first='{}'", _c4prc(first));
8501 if(first == '\'')
8502 {
8503 _c4dbgp("usty[RMAP]: scanning single-quoted scalar");
8504 sc = _scan_scalar_squot();
8505 if(!_maybe_scan_following_colon())
8506 {
8507 _c4err("cannot read a VAL to a map");
8508 }
8509 else
8510 {
8511 _c4dbgp("usty[RMAP]: start new block map, set scalar as key");
8512 add_flags(RNXT);
8513 _handle_annotations_before_start_mapblck(startline);
8514 m_evt_handler->_push();
8515 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
8516 csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc);
8517 m_evt_handler->set_key_scalar_squoted(maybe_filtered);
8518 _set_indentation(startindent);
8519 addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8520 _maybe_skip_whitespace_tokens();
8521 }
8522 }
8523 else if(first == '"')
8524 {
8525 _c4dbgp("usty[RMAP]: scanning double-quoted scalar");
8526 sc = _scan_scalar_dquot();
8527 if(!_maybe_scan_following_colon())
8528 {
8529 _c4err("cannot read a VAL to a map");
8530 }
8531 else
8532 {
8533 _c4dbgp("usty[RMAP]: start new block map, set double-quoted scalar as key");
8534 add_flags(RNXT);
8535 _handle_annotations_before_start_mapblck(startline);
8536 m_evt_handler->_push();
8537 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
8538 csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc);
8539 m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
8540 _set_indentation(startindent);
8541 addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8542 _maybe_skip_whitespace_tokens();
8543 }
8544 }
8545 else if(first == '|')
8546 {
8547 _c4err("block literal keys must be enclosed in '?'");
8548 }
8549 else if(first == '>')
8550 {
8551 _c4err("block literal keys must be enclosed in '?'");
8552 }
8553 else if(_scan_scalar_plain_unk(&sc))
8554 {
8555 _c4dbgp("usty[RMAP]: got a plain scalar");
8556 if(!_maybe_scan_following_colon())
8557 {
8558 _c4err("cannot read a VAL to a map");
8559 }
8560 else
8561 {
8562 _c4dbgp("usty[RMAP]: start new block map, set scalar as key");
8563 add_flags(RNXT);
8564 _handle_annotations_before_start_mapblck(startline);
8565 m_evt_handler->_push();
8566 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
8567 csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, startindent);
8568 m_evt_handler->set_key_scalar_plain(maybe_filtered);
8569 _set_indentation(startindent);
8570 addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8571 _maybe_skip_whitespace_tokens();
8572 }
8573 }
8574 else
8575 {
8576 _c4err("parse error"); // LCOV_EXCL_LINE
8577 }
8578 }
8579 }
8580 else // destination is unknown
8581 {
8582 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(RSEQ), m_evt_handler->m_curr->pos);
8583 _c4dbgpf("usty[UNK]: first='{}'", _c4prc(first));
8584 if(first == '[')
8585 {
8586 _c4dbgp("usty[UNK]: it's a flow seq");
8587 add_flags(RNXT);
8588 _handle_annotations_before_blck_val_scalar();
8589 m_evt_handler->begin_seq_val_flow();
8590 addrem_flags(RSEQ|RFLOW|RVAL, RNXT|USTY);
8591 _set_indentation(startindent);
8592 _line_progressed(1);
8593 _maybe_skip_whitespace_tokens();
8594 }
8595 else if(first == '-' && _is_blck_token(rem))
8596 {
8597 _c4dbgp("usty[UNK]: it's a block seq");
8598 add_flags(RNXT);
8599 _handle_annotations_before_blck_val_scalar();
8600 m_evt_handler->begin_seq_val_block();
8601 addrem_flags(RSEQ|RBLCK|RVAL, RNXT|USTY);
8602 _set_indentation(startindent);
8603 _line_progressed(1);
8604 _maybe_skip_whitespace_tokens();
8605 }
8606 else if(first == '{')
8607 {
8608 _c4dbgp("usty[UNK]: it's a flow map");
8609 add_flags(RNXT);
8610 _handle_annotations_before_blck_val_scalar();
8611 m_evt_handler->begin_map_val_flow();
8612 addrem_flags(RMAP|RFLOW|RKEY, RNXT|USTY);
8613 _set_indentation(startindent);
8614 _line_progressed(1);
8615 _maybe_skip_whitespace_tokens();
8616 }
8617 else if(first == '?' && _is_blck_token(rem))
8618 {
8619 _c4dbgp("usty[UNK]: it's a map + this key is complex");
8620 add_flags(RNXT);
8621 _handle_annotations_before_blck_val_scalar();
8622 m_evt_handler->begin_map_val_block();
8623 addrem_flags(RMAP|RBLCK|QMRK, RNXT|USTY);
8624 _save_indentation();
8625 _line_progressed(1);
8626 _maybe_skip_whitespace_tokens();
8627 }
8628 else if(first == ':' && _is_blck_token(rem))
8629 {
8630 _c4dbgp("usty[UNK]: it's a map with an empty key");
8631 add_flags(RNXT);
8632 _handle_annotations_before_blck_val_scalar();
8633 m_evt_handler->begin_map_val_block();
8634 m_evt_handler->set_key_scalar_plain_empty();
8635 addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8636 _save_indentation();
8637 _line_progressed(1);
8638 _maybe_skip_whitespace_tokens();
8639 }
8640 else if(first == '&')
8641 {
8642 csubstr anchor = _scan_anchor();
8643 _c4dbgpf("usty[UNK]: anchor! {}", prs_(anchor));
8644 const size_t indentation = m_evt_handler->m_curr->line_contents.current_col(rem);
8645 const size_t line = m_evt_handler->m_curr->pos.line;
8646 _add_annotation(&m_pending_anchors, anchor, indentation, line);
8647 _set_indentation(m_evt_handler->m_curr->line_contents.current_col(rem));
8648 }
8649 else if(first == '*')
8650 {
8651 csubstr ref = _scan_ref_map();
8652 _c4dbgpf("usty[UNK]: ref! {}", prs_(ref));
8653 if(!_maybe_scan_following_colon())
8654 {
8655 _c4dbgp("usty[UNK]: set val ref");
8656 _handle_valref(ref);
8657 }
8658 else
8659 {
8660 _c4dbgp("usty[UNK]: start new block map, set ref as key");
8661 const size_t startline = m_evt_handler->m_curr->pos.line; // save
8662 add_flags(RNXT);
8663 _handle_annotations_before_start_mapblck(startline);
8664 m_evt_handler->begin_map_val_block();
8665 _handle_keyref(ref);
8666 _maybe_skip_whitespace_tokens();
8667 _set_indentation(startindent);
8668 addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8669 }
8670 }
8671 else if(first == '!')
8672 {
8673 csubstr tag = _scan_tag();
8674 _c4dbgpf("usty[UNK]: val tag! {}", prs_(tag));
8675 // we need to buffer the tags, as there may be two
8676 // consecutive tags in here
8677 const size_t indentation = m_evt_handler->m_curr->line_contents.current_col(rem);
8678 const size_t line = m_evt_handler->m_curr->pos.line;
8679 _add_annotation(&m_pending_tags, tag, indentation, line);
8680 }
8681 else
8682 {
8683 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! has_any(SSCL), m_evt_handler->m_curr->pos);
8684 startindent = m_evt_handler->m_curr->line_contents.indentation; // save
8685 const size_t startline = m_evt_handler->m_curr->pos.line; // save
8686 first = rem.str[0];
8687 ScannedScalar sc;
8688 _c4dbgpf("usty[UNK]: maybe scalar. first='{}'", _c4prc(first));
8689 if(first == '\'')
8690 {
8691 _c4dbgp("usty[UNK]: scanning single-quoted scalar");
8692 sc = _scan_scalar_squot();
8693 if(!_maybe_scan_following_colon())
8694 {
8695 _c4dbgp("usty[UNK]: set as val");
8696 _handle_annotations_before_blck_val_scalar();
8697 csubstr maybe_filtered = _maybe_filter_val_scalar_squot(sc);
8698 m_evt_handler->set_val_scalar_squoted(maybe_filtered);
8699 _end_stream();
8700 }
8701 else
8702 {
8703 _c4dbgp("usty[UNK]: start new block map, set scalar as key");
8704 add_flags(RNXT);
8705 _handle_annotations_before_start_mapblck(startline);
8706 m_evt_handler->begin_map_val_block();
8707 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
8708 csubstr maybe_filtered = _maybe_filter_key_scalar_squot(sc);
8709 m_evt_handler->set_key_scalar_squoted(maybe_filtered);
8710 _set_indentation(startindent);
8711 addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8712 _maybe_skip_whitespace_tokens();
8713 }
8714 }
8715 else if(first == '"')
8716 {
8717 _c4dbgp("usty[UNK]: scanning double-quoted scalar");
8718 sc = _scan_scalar_dquot();
8719 if(!_maybe_scan_following_colon())
8720 {
8721 _c4dbgp("usty[UNK]: set as val");
8722 _handle_annotations_before_blck_val_scalar();
8723 csubstr maybe_filtered = _maybe_filter_val_scalar_dquot(sc);
8724 m_evt_handler->set_val_scalar_dquoted(maybe_filtered);
8725 _end_stream();
8726 }
8727 else
8728 {
8729 _c4dbgp("usty[UNK]: start new block map, set double-quoted scalar as key");
8730 add_flags(RNXT);
8731 _handle_annotations_before_start_mapblck(startline);
8732 m_evt_handler->begin_map_val_block();
8733 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
8734 csubstr maybe_filtered = _maybe_filter_key_scalar_dquot(sc);
8735 m_evt_handler->set_key_scalar_dquoted(maybe_filtered);
8736 _set_indentation(startindent);
8737 addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8738 _maybe_skip_whitespace_tokens();
8739 }
8740 }
8741 else if(first == '|')
8742 {
8743 _c4dbgp("usty[UNK]: scanning block-literal scalar");
8744 ScannedBlock sb;
8745 _scan_block(&sb, startindent);
8746 _c4dbgp("usty[UNK]: set as val");
8747 _handle_annotations_before_blck_val_scalar();
8748 csubstr maybe_filtered = _maybe_filter_val_scalar_literal(sb);
8749 m_evt_handler->set_val_scalar_literal(maybe_filtered);
8750 _end_stream();
8751 }
8752 else if(first == '>')
8753 {
8754 _c4dbgp("usty[UNK]: scanning block-folded scalar");
8755 ScannedBlock sb;
8756 _scan_block(&sb, startindent);
8757 _c4dbgp("usty[UNK]: set as val");
8758 _handle_annotations_before_blck_val_scalar();
8759 csubstr maybe_filtered = _maybe_filter_val_scalar_folded(sb);
8760 m_evt_handler->set_val_scalar_folded(maybe_filtered);
8761 _end_stream();
8762 }
8763 else if(_scan_scalar_plain_unk(&sc))
8764 {
8765 _c4dbgp("usty[UNK]: got a plain scalar");
8766 if(!_maybe_scan_following_colon())
8767 {
8768 _c4dbgp("usty[UNK]: set as val");
8769 _handle_annotations_before_blck_val_scalar();
8770 csubstr maybe_filtered = _maybe_filter_val_scalar_plain(sc, startindent);
8771 m_evt_handler->set_val_scalar_plain(maybe_filtered);
8772 _end_stream();
8773 }
8774 else
8775 {
8776 _c4dbgp("usty[UNK]: start new block map, set scalar as key");
8777 add_flags(RNXT);
8778 _handle_annotations_before_start_mapblck(startline);
8779 m_evt_handler->begin_map_val_block();
8780 _handle_annotations_and_indentation_after_start_mapblck(startindent, startline);
8781 csubstr maybe_filtered = _maybe_filter_key_scalar_plain(sc, startindent);
8782 m_evt_handler->set_key_scalar_plain(maybe_filtered);
8783 _set_indentation(startindent);
8784 addrem_flags(RMAP|RBLCK|RVAL, RNXT|USTY);
8785 _maybe_skip_whitespace_tokens();
8786 }
8787 }
8788 else
8789 {
8790 _c4err("parse error"); // LCOV_EXCL_LINE
8791 }
8792 }
8793 }
8794}
8795
8796
8797//-----------------------------------------------------------------------------
8798
8799template<class EventHandler>
8800void ParseEngine<EventHandler>::parse_json_in_place_ev(csubstr filename, substr src)
8801{
8802 RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_stack.size() >= 1);
8803 RYML_SAVE_TEST_JSON_(filename, src);
8804 m_evt_handler->start_parse(filename.str, src);
8805 m_evt_handler->begin_stream();
8806 _reset();
8807 while( ! _finished_file())
8808 {
8809 _scan_line();
8810 while( ! _finished_line())
8811 {
8812 _c4dbgnextline();
8813 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! m_evt_handler->m_curr->line_contents.rem.empty(), m_evt_handler->m_curr->pos);
8814 if(has_any(RSEQ))
8815 {
8816 _handle_seq_json();
8817 }
8818 else if(has_any(RMAP))
8819 {
8820 _handle_map_json();
8821 }
8822 else if(has_any(RUNK))
8823 {
8824 _handle_unk_json();
8825 }
8826 else
8827 {
8828 _c4err("internal error"); // LCOV_EXCL_LINE
8829 }
8830 }
8831 if(_finished_file())
8832 break; // it may have finished because of multiline blocks
8833 _line_ended();
8834 }
8835 _end_stream();
8836 m_evt_handler->finish_parse();
8837}
8838
8839
8840//-----------------------------------------------------------------------------
8841
8842template<class EventHandler>
8843void ParseEngine<EventHandler>::parse_in_place_ev(csubstr filename, substr src)
8844{
8845 RYML_ASSERT_BASIC_CB_(m_evt_handler->m_stack.m_callbacks, m_evt_handler->m_stack.size() >= 1);
8846 RYML_SAVE_TEST_YAML_(filename, src);
8847 m_evt_handler->start_parse(filename.str, src);
8848 m_evt_handler->begin_stream();
8849 _reset();
8850 while( ! _finished_file())
8851 {
8852 _scan_line();
8853 while( ! _finished_line())
8854 {
8855 _c4dbgnextline();
8856 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, ! m_evt_handler->m_curr->line_contents.rem.empty(), m_evt_handler->m_curr->pos);
8857 if(has_any(RFLOW))
8858 {
8859 if(has_none(RSEQIMAP))
8860 {
8861 if(has_any(RSEQ))
8862 {
8863 _handle_seq_flow();
8864 }
8865 else
8866 {
8867 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RMAP), m_evt_handler->m_curr->pos);
8868 _handle_map_flow();
8869 }
8870 }
8871 else
8872 {
8873 _handle_seq_imap();
8874 }
8875 }
8876 else if(has_any(RBLCK))
8877 {
8878 if(has_any(RSEQ))
8879 {
8880 _handle_seq_block();
8881 }
8882 else
8883 {
8884 RYML_ASSERT_PARSE_CB_(m_evt_handler->m_stack.m_callbacks, has_all(RMAP), m_evt_handler->m_curr->pos);
8885 _handle_map_block();
8886 }
8887 }
8888 else if(has_any(RUNK))
8889 {
8890 _handle_unk();
8891 }
8892 else if(has_any(USTY))
8893 {
8894 _handle_usty();
8895 }
8896 else
8897 {
8898 _c4err("internal error"); // LCOV_EXCL_LINE
8899 }
8900 }
8901 if(_finished_file())
8902 break; // it may have finished because of multiline blocks
8903 _line_ended();
8904 }
8905 _end_stream();
8906 m_evt_handler->finish_parse();
8907}
8908/** @endcond */
8909
8910} // namespace yml
8911} // namespace c4
8912
8913// NOLINTEND(hicpp-signed-bitwise,cppcoreguidelines-avoid-goto,hicpp-avoid-goto,hicpp-multiway-paths-covered,modernize-avoid-c-style-cast)
8914
8915#undef _c4dbgnextline
8916#undef _c4assert
8917#undef _c4err
8918
8919C4_SUPPRESS_WARNING_MSVC_POP
8920C4_SUPPRESS_WARNING_GCC_CLANG_POP
8921
8922#endif // C4_YML_PARSE_ENGINE_DEF_HPP_
charconv.hpp
Lightweight generic type-safe wrappers for converting individual values to/from strings.
c4::yml::ParseEngine
This is the main driver of parsing logic: it scans the YAML or JSON source for tokens,...
Definition parse_engine.hpp:291
c4::yml::ParseEngine::filter_scalar_plain
FilterResult filter_scalar_plain(csubstr scalar, substr dst, size_t indentation)
filter a plain scalar
c4::yml::ParseEngine::location_contents
csubstr location_contents(Location const &loc) const
Get the string starting at a particular location, to the end of the parsed source buffer.
c4::yml::ParseEngine::filter_scalar_squoted
FilterResult filter_scalar_squoted(csubstr scalar, substr dst)
filter a single-quoted scalar
c4::yml::ParseEngine::filter_scalar_dquoted
FilterResult filter_scalar_dquoted(csubstr scalar, substr dst)
filter a double-quoted scalar
c4::yml::ParseEngine::parse_json_in_place_ev
void parse_json_in_place_ev(csubstr filename, substr src)
parse JSON in place, emitting events to the current handler
c4::yml::ParseEngine::val_location
Location val_location(const char *val) const
Given a pointer to a buffer position, get the location.
c4::yml::ParseEngine::filter_scalar_plain_in_place
FilterResult filter_scalar_plain_in_place(substr scalar, size_t cap, size_t indentation)
filter a plain scalar in place
c4::yml::ParseEngine::filter_scalar_squoted_in_place
FilterResult filter_scalar_squoted_in_place(substr scalar, size_t cap)
filter a single-quoted scalar in place
c4::yml::ParseEngine::filter_scalar_dquoted_in_place
FilterResultExtending filter_scalar_dquoted_in_place(substr scalar, size_t cap)
filter a double-quoted scalar in place
c4::yml::ParseEngine::parse_in_place_ev
void parse_in_place_ev(csubstr filename, substr src)
parse YAML in place, emitting events to the current handler
c4::yml::ParseEngine::filter_scalar_block_literal_in_place
FilterResult filter_scalar_block_literal_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp)
filter a block-literal scalar in place
c4::yml::ParseEngine::ParseEngine
ParseEngine(EventHandler *evt_handler, ParserOptions const &opts={})
Definition parse_engine.def.hpp:277
c4::yml::ParseEngine::filter_scalar_block_literal
FilterResult filter_scalar_block_literal(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp)
filter a block-literal scalar
c4::yml::ParseEngine::filter_scalar_block_folded_in_place
FilterResult filter_scalar_block_folded_in_place(substr scalar, size_t cap, size_t indentation, BlockChomp_e chomp)
filter a block-folded scalar in place
c4::yml::ParseEngine::operator=
ParseEngine & operator=(ParseEngine &&) noexcept
Definition parse_engine.def.hpp:340
c4::yml::ParseEngine::filter_scalar_block_folded
FilterResult filter_scalar_block_folded(csubstr scalar, substr dst, size_t indentation, BlockChomp_e chomp)
filter a block-folded scalar
RYML_LOCATIONS_SMALL_THRESHOLD
#define RYML_LOCATIONS_SMALL_THRESHOLD
threshold at which a location search will revert from linear to binary search.
Definition common.hpp:39
RYML_NOEXCEPT
#define RYML_NOEXCEPT
Conditionally expands to noexcept when RYML_USE_ASSERT is 0 and is empty otherwise.
Definition doxy_common.hpp:290
filter_processor.hpp
c4::atou
bool atou(csubstr str, T *v) noexcept
Convert a trimmed string to an unsigned integral value.
Definition charconv.hpp:1558
c4::yml::err_parse
void err_parse(ErrorDataParse const &errdata, const char *msg)
trigger a parse error to its respective handler, with a non-formatted error message.
Definition common.cpp:210
c4::read_hex
bool read_hex(csubstr s, I *v) noexcept
read an hexadecimal integer from a string.
Definition charconv.hpp:902
c4::substr
basic_substring< char > substr
a mutable string view
Definition substr.hpp:2355
c4::csubstr
basic_substring< const char > csubstr
an immutable string view
Definition substr.hpp:2356
c4::yml::is_valid_tag_handle
bool is_valid_tag_handle(csubstr handle)
Definition tag.cpp:210
c4::yml::is_custom_tag
bool is_custom_tag(csubstr tag)
is a tag of the form !handle!tag?
Definition tag.cpp:9
c4::decode_code_point
substr decode_code_point(substr out, csubstr code_point)
decode the given code_point, writing into the output string in out.
Definition utf.cpp:42
c4::yml::NONE
@ NONE
an index to none
Definition common.hpp:131
c4::yml::ParserFlag_t
int ParserFlag_t
data type for ParserState_e
Definition parser_state.hpp:14
c4::yml::RTOP
@ RTOP
reading at top level
Definition parser_state.hpp:18
c4::yml::RSET
@ RSET
the (implicit) map being read is a !!set.
Definition parser_state.hpp:31
c4::yml::RSEQ
@ RSEQ
reading a seq
Definition parser_state.hpp:21
c4::yml::RNXT
@ RNXT
read next sibling
Definition parser_state.hpp:28
c4::yml::RUNK
@ RUNK
reading unknown state (when starting): must determine whether scalar, map or seq
Definition parser_state.hpp:19
c4::yml::RKEY
@ RKEY
reading a key
Definition parser_state.hpp:25
c4::yml::RKCL
@ RKCL
reading the key colon (ie the : after the key in the map)
Definition parser_state.hpp:27
c4::yml::NDOC
@ NDOC
no document mode. a document has ended and another has not started yet.
Definition parser_state.hpp:33
c4::yml::RDOC
@ RDOC
reading a document
Definition parser_state.hpp:32
c4::yml::QSCL
@ QSCL
stored scalar was quoted
Definition parser_state.hpp:30
c4::yml::RBLCK
@ RBLCK
reading in block mode
Definition parser_state.hpp:23
c4::yml::RMAP
@ RMAP
reading a map
Definition parser_state.hpp:20
c4::yml::USTY
@ USTY
reading in unknown style mode - must determine FLOW or BLCK reading an implicit map nested in an expl...
Definition parser_state.hpp:34
c4::yml::QMRK
@ QMRK
reading an explicit key (? key)
Definition parser_state.hpp:24
c4::yml::SSCL
@ SSCL
there's a stored scalar
Definition parser_state.hpp:29
c4::yml::RVAL
@ RVAL
reading a val
Definition parser_state.hpp:26
c4::yml::RFLOW
@ RFLOW
reading is inside explicit flow chars: [] or {}
Definition parser_state.hpp:22
c4::yml::adjust_pos_with_escapes
size_t adjust_pos_with_escapes(csubstr scalar, size_t pos, bool keep_newlines=false)
Adjust a position in a scalar, increasing it to account for any escaped characters.
Definition escape_scalar.hpp:172
c4::yml::to_chars
size_t to_chars(substr buf, escaped_scalar e)
formatting implementation to escape a scalar with escape_scalar()
Definition escape_scalar.hpp:264
c4::yml::npos
@ npos
a null string position
Definition common.hpp:138
c4::yml::id_type
RYML_ID_TYPE id_type
The type of a node id in the YAML tree; to override the default type, define the macro RYML_ID_TYPE t...
Definition common.hpp:124
c4::yml::UTF8
@ UTF8
UTF8.
Definition common.hpp:144
c4::yml::NOBOM
@ NOBOM
No Byte Order Mark was found.
Definition common.hpp:143
c4::yml::Encoding_e
enum c4::yml::Encoding_ Encoding_e
c4::yml::version
csubstr version()
Definition version.cpp:6
node_type.hpp
_c4dbgnextline
#define _c4dbgnextline()
Definition parse_engine.def.hpp:53
_ryml_relocate
#define _ryml_relocate(s)
_c4err
#define _c4err(...)
Definition parse_engine.def.hpp:34
_c4assert
#define _c4assert(...)
Definition parse_engine.def.hpp:37
RYML_WITH_TAB_TOKENS_
#define RYML_WITH_TAB_TOKENS_(...)
Definition parse_engine.def.hpp:46
RYML_WITH_OR_WITHOUT_TAB_TOKENS_
#define RYML_WITH_OR_WITHOUT_TAB_TOKENS_(with, without)
Definition parse_engine.def.hpp:48
parse_engine.hpp
c4::basic_substring::range
basic_substring range(size_t first, size_t last=npos) const noexcept
return [first,last[.
Definition substr.hpp:519
c4::basic_substring::first_not_of
size_t first_not_of(const C c) const
Definition substr.hpp:993
c4::basic_substring::triml
basic_substring triml(const C c) const
trim left
Definition substr.hpp:629
c4::basic_substring::first_of
size_t first_of(const C c, size_t start=0) const
Definition substr.hpp:934
c4::basic_substring::first
basic_substring first(size_t num) const noexcept
return the first num elements: [0,num[
Definition substr.hpp:529
c4::basic_substring< const char >::npos
@ npos
Definition substr.hpp:238
c4::basic_substring::sub
basic_substring sub(size_t first) const noexcept
return [first,len[
Definition substr.hpp:502
c4::basic_substring::trimr
basic_substring trimr(const C c) const
trim the character c from the right
Definition substr.hpp:653
c4::basic_substring::str
C * str
a restricted pointer to the first character of the substring
Definition substr.hpp:216
c4::yml::ErrorDataParse
Data for a parse error.
Definition common.hpp:269
c4::yml::FilterProcessorSrcDst
Filters an input string into a different output string.
Definition filter_processor.hpp:49
c4::yml::FilterResultExtending
Result for filtering a scalar which not fit in the intended memory.
Definition filter_processor.hpp:36
c4::yml::FilterResult
Result for filtering a scalar which not fit in the intended memory.
Definition filter_processor.hpp:27
c4::yml::LineContents
Helper to control the line contents while parsing a buffer.
Definition parser_state.hpp:57
c4::yml::LineContents::reset_with_next_line
void reset_with_next_line(substr buf, size_t start) RYML_NOEXCEPT
Definition parser_state.hpp:66
c4::yml::Location
holds a source or yaml file position, for example when an error is detected; See also location_format...
Definition common.hpp:229
c4::yml::Location::name
csubstr name
name of the file
Definition common.hpp:233
c4::yml::ParserOptions
Options to give to the ParseEngine to control its behavior.
Definition parse_options.hpp:22
c4::yml::TagCache
Accelerator structure to reduce memory requirements by enabling reuse of resolved tags.
Definition tag.hpp:71
c4::yml::escaped_scalar
formatting helper to escape a scalar with escape_scalar_fn()
Definition escape_scalar.hpp:257
utf.hpp
utilities for UTF and Byte Order Mark