From b5c2daf23d0fad299eaacdd5fadef0e591944dad Mon Sep 17 00:00:00 2001 From: Yuki Izumi Date: Wed, 9 Aug 2017 17:13:16 +1000 Subject: [PATCH 1/6] Emphasis sourcepos broken when in block --- api_test/main.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api_test/main.c b/api_test/main.c index c10601a9b..ad64437b5 100644 --- a/api_test/main.c +++ b/api_test/main.c @@ -936,7 +936,7 @@ static void test_feed_across_line_ending(test_batch_runner *runner) { static void source_pos(test_batch_runner *runner) { static const char markdown[] = - "Hi *there*.\n" + "# Hi *there*.\n" "\n" "Hello “ \n" "there `hi` -- [okay](www.google.com (ok)).\n" @@ -952,13 +952,13 @@ static void source_pos(test_batch_runner *runner) { STR_EQ(runner, xml, "\n" "\n" "\n" - " \n" - " Hi \n" - " \n" - " there\n" + " \n" + " Hi \n" + " \n" + " there\n" " \n" - " .\n" - " \n" + " .\n" + " \n" " \n" " Hello “ \n" " \n" From a759aa7074ff8f61cda49296c0bdc9edd3745770 Mon Sep 17 00:00:00 2001 From: Yuki Izumi Date: Wed, 9 Aug 2017 17:13:50 +1000 Subject: [PATCH 2/6] internal_offset for blocks containing inlines --- api_test/main.c | 12 ++++++------ extensions/table.c | 8 +++++++- src/blocks.c | 1 + src/inlines.c | 2 +- src/node.h | 1 + 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/api_test/main.c b/api_test/main.c index ad64437b5..057ee51ce 100644 --- a/api_test/main.c +++ b/api_test/main.c @@ -1048,26 +1048,26 @@ static void ext_source_pos(test_batch_runner *runner) { " \n" " \n" " \n" - " \n" + " \n" " a\n" " \n" - " \n" + " \n" " b\n" " \n" - " \n" + " \n" " \n" " c\n" " \n" " \n" " \n" " \n" - " \n" + " \n" " 1\n" " \n" - " \n" + " \n" " 2\n" " \n" - " \n" + " \n" " \n" " 3\n" " \n" diff --git a/extensions/table.c b/extensions/table.c index 73de5031d..f6d3e7a07 100644 --- a/extensions/table.c +++ b/extensions/table.c @@ -28,7 +28,7 @@ typedef struct { typedef struct { cmark_strbuf *buf; - int start_offset, end_offset; + int start_offset, end_offset, internal_offset; } node_cell; static void free_table_cell(cmark_mem *mem, void *data) { @@ -133,6 +133,10 @@ static table_row *row_from_string(cmark_syntax_extension *self, cell->buf = cell_buf; cell->start_offset = offset; cell->end_offset = offset + cell_matched - 1; + while (cell->start_offset > 0 && string[cell->start_offset - 1] != '|') { + --cell->start_offset; + ++cell->internal_offset; + } row->n_columns += 1; row->cells = cmark_llist_append(parser->mem, row->cells, cell); } @@ -249,6 +253,7 @@ static cmark_node *try_opening_table_header(cmark_syntax_extension *self, cmark_node *header_cell = cmark_parser_add_child(parser, table_header, CMARK_NODE_TABLE_CELL, parent_container->start_column + cell->start_offset); header_cell->start_line = header_cell->end_line = parent_container->start_line; + header_cell->internal_offset = cell->internal_offset; header_cell->end_column = parent_container->start_column + cell->end_offset; cmark_node_set_string_content(header_cell, (char *) cell->buf->ptr); cmark_node_set_syntax_extension(header_cell, self); @@ -292,6 +297,7 @@ static cmark_node *try_opening_table_row(cmark_syntax_extension *self, node_cell *cell = (node_cell *) tmp->data; cmark_node *node = cmark_parser_add_child(parser, table_row_block, CMARK_NODE_TABLE_CELL, parent_container->start_column + cell->start_offset); + node->internal_offset = cell->internal_offset; node->end_column = parent_container->start_column + cell->end_offset; cmark_node_set_string_content(node, (char *) cell->buf->ptr); cmark_node_set_syntax_extension(node, self); diff --git a/src/blocks.c b/src/blocks.c index 0c2222056..723abcc32 100644 --- a/src/blocks.c +++ b/src/blocks.c @@ -980,6 +980,7 @@ static void open_new_blocks(cmark_parser *parser, cmark_node **container, (*container)->as.heading.level = level; (*container)->as.heading.setext = false; + (*container)->internal_offset = matched; } else if (!indented && (matched = scan_open_code_fence( input, parser->first_nonspace))) { diff --git a/src/inlines.c b/src/inlines.c index 4faf873b1..623d6544e 100644 --- a/src/inlines.c +++ b/src/inlines.c @@ -1240,7 +1240,7 @@ void cmark_parse_inlines(cmark_parser *parser, cmark_reference_map *refmap, int options) { subject subj; - subject_from_buf(parser->mem, parent->start_line, parent->start_column - 1, &subj, &parent->content, refmap); + subject_from_buf(parser->mem, parent->start_line, parent->start_column - 1 + parent->internal_offset, &subj, &parent->content, refmap); cmark_chunk_rtrim(&subj.input); while (!is_eof(&subj) && parse_inline(parser, &subj, parent, options)) diff --git a/src/node.h b/src/node.h index e32814bcf..1d8aa50df 100644 --- a/src/node.h +++ b/src/node.h @@ -68,6 +68,7 @@ struct cmark_node { int start_column; int end_line; int end_column; + int internal_offset; uint16_t type; uint16_t flags; From dafff145d5e547de900bc5ee336d871dd8226204 Mon Sep 17 00:00:00 2001 From: Yuki Izumi Date: Thu, 10 Aug 2017 12:16:28 +1000 Subject: [PATCH 3/6] Really awkward newline handling --- src/inlines.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/inlines.c b/src/inlines.c index 623d6544e..90aaa8a8c 100644 --- a/src/inlines.c +++ b/src/inlines.c @@ -236,7 +236,8 @@ static CMARK_INLINE cmark_chunk take_while(subject *subj, int (*f)(int)) { // backticks, otherwise return the position in the subject // after the closing backticks. static bufsize_t scan_to_closing_backticks(subject *subj, - bufsize_t openticklength) { + bufsize_t openticklength, + int *newlines, int *since_newline) { bool found = false; if (openticklength > MAXBACKTICKS) { @@ -251,8 +252,15 @@ static bufsize_t scan_to_closing_backticks(subject *subj, while (!found) { // read non backticks unsigned char c; + int nls = 0, since_nl = 0; while ((c = peek_char(subj)) && c != '`') { advance(subj); + if (c == '\n') { + ++nls; + since_nl = 0; + } else { + ++since_nl; + } } if (is_eof(subj)) { break; @@ -267,6 +275,10 @@ static bufsize_t scan_to_closing_backticks(subject *subj, subj->backticks[numticks] = subj->pos - numticks; } if (numticks == openticklength) { + *newlines = nls; + if (nls) { + *since_newline = since_nl; + } return (subj->pos); } } @@ -278,9 +290,10 @@ static bufsize_t scan_to_closing_backticks(subject *subj, // Parse backtick code section or raw backticks, return an inline. // Assumes that the subject has a backtick at the current position. static cmark_node *handle_backticks(subject *subj) { + int newlines, since_newline; cmark_chunk openticks = take_while(subj, isbacktick); bufsize_t startpos = subj->pos; - bufsize_t endpos = scan_to_closing_backticks(subj, openticks.len); + bufsize_t endpos = scan_to_closing_backticks(subj, openticks.len, &newlines, &since_newline); if (endpos == 0) { // not found subj->pos = startpos; // rewind @@ -293,7 +306,14 @@ static cmark_node *handle_backticks(subject *subj) { cmark_strbuf_trim(&buf); cmark_strbuf_normalize_whitespace(&buf); - return make_code(subj, startpos, endpos - openticks.len - 1, cmark_chunk_buf_detach(&buf)); + cmark_node *node = make_code(subj, startpos, endpos - openticks.len - 1, cmark_chunk_buf_detach(&buf)); + if (newlines) { + subj->line += newlines; + node->end_line += newlines; + node->end_column = since_newline; + subj->column_offset = -subj->pos + since_newline + openticks.len; + } + return node; } } From ae5da0dc5186c507caffd97af689dac380410467 Mon Sep 17 00:00:00 2001 From: Yuki Izumi Date: Thu, 10 Aug 2017 12:48:22 +1000 Subject: [PATCH 4/6] Count newlines in every inline that could contain --- src/inlines.c | 63 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/src/inlines.c b/src/inlines.c index 90aaa8a8c..3ddece430 100644 --- a/src/inlines.c +++ b/src/inlines.c @@ -230,14 +230,44 @@ static CMARK_INLINE cmark_chunk take_while(subject *subj, int (*f)(int)) { return cmark_chunk_dup(&subj->input, startpos, len); } +static int count_newlines(subject *subj, bufsize_t from, bufsize_t len, int *since_newline) { + int nls = 0; + int since_nl = 0; + + while (len--) { + if (subj->input.data[from++] == '\n') { + ++nls; + since_nl = 0; + } else { + ++since_nl; + } + } + + if (!nls) + return 0; + + *since_newline = since_nl; + return nls; +} + +static void adjust_subj_node_newlines(subject *subj, cmark_node *node, int matchlen, int extra) { + int since_newline; + int newlines = count_newlines(subj, subj->pos - matchlen - extra, matchlen, &since_newline); + if (newlines) { + subj->line += newlines; + node->end_line += newlines; + node->end_column = since_newline; + subj->column_offset = -subj->pos + since_newline + extra; + } +} + // Try to process a backtick code span that began with a // span of ticks of length openticklength length (already // parsed). Return 0 if you don't find matching closing // backticks, otherwise return the position in the subject // after the closing backticks. static bufsize_t scan_to_closing_backticks(subject *subj, - bufsize_t openticklength, - int *newlines, int *since_newline) { + bufsize_t openticklength) { bool found = false; if (openticklength > MAXBACKTICKS) { @@ -252,15 +282,8 @@ static bufsize_t scan_to_closing_backticks(subject *subj, while (!found) { // read non backticks unsigned char c; - int nls = 0, since_nl = 0; while ((c = peek_char(subj)) && c != '`') { advance(subj); - if (c == '\n') { - ++nls; - since_nl = 0; - } else { - ++since_nl; - } } if (is_eof(subj)) { break; @@ -275,10 +298,6 @@ static bufsize_t scan_to_closing_backticks(subject *subj, subj->backticks[numticks] = subj->pos - numticks; } if (numticks == openticklength) { - *newlines = nls; - if (nls) { - *since_newline = since_nl; - } return (subj->pos); } } @@ -290,10 +309,9 @@ static bufsize_t scan_to_closing_backticks(subject *subj, // Parse backtick code section or raw backticks, return an inline. // Assumes that the subject has a backtick at the current position. static cmark_node *handle_backticks(subject *subj) { - int newlines, since_newline; cmark_chunk openticks = take_while(subj, isbacktick); bufsize_t startpos = subj->pos; - bufsize_t endpos = scan_to_closing_backticks(subj, openticks.len, &newlines, &since_newline); + bufsize_t endpos = scan_to_closing_backticks(subj, openticks.len); if (endpos == 0) { // not found subj->pos = startpos; // rewind @@ -307,12 +325,7 @@ static cmark_node *handle_backticks(subject *subj) { cmark_strbuf_normalize_whitespace(&buf); cmark_node *node = make_code(subj, startpos, endpos - openticks.len - 1, cmark_chunk_buf_detach(&buf)); - if (newlines) { - subj->line += newlines; - node->end_line += newlines; - node->end_column = since_newline; - subj->column_offset = -subj->pos + since_newline + openticks.len; - } + adjust_subj_node_newlines(subj, node, endpos - startpos, openticks.len); return node; } } @@ -824,7 +837,9 @@ static cmark_node *handle_pointy_brace(subject *subj, bool liberal_html_tag) { if (matchlen > 0) { contents = cmark_chunk_dup(&subj->input, subj->pos - 1, matchlen + 1); subj->pos += matchlen; - return make_raw_html(subj, subj->pos - 1 - matchlen, subj->pos - 1, contents); + cmark_node *node = make_raw_html(subj, subj->pos - matchlen - 1, subj->pos - 1, contents); + adjust_subj_node_newlines(subj, node, matchlen, 1); + return node; } if (liberal_html_tag) { @@ -832,7 +847,9 @@ static cmark_node *handle_pointy_brace(subject *subj, bool liberal_html_tag) { if (matchlen > 0) { contents = cmark_chunk_dup(&subj->input, subj->pos - 1, matchlen + 1); subj->pos += matchlen; - return make_raw_html(subj, subj->pos - 1 - matchlen, subj->pos - 1, contents); + cmark_node *node = make_raw_html(subj, subj->pos - matchlen - 1, subj->pos - 1, contents); + adjust_subj_node_newlines(subj, node, matchlen, 1); + return node; } } From 0c581db99648aacbb9e29b903d76dbcdecf9154b Mon Sep 17 00:00:00 2001 From: Yuki Izumi Date: Thu, 10 Aug 2017 13:25:55 +1000 Subject: [PATCH 5/6] Document --- src/inlines.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/inlines.c b/src/inlines.c index 3ddece430..0cf9eb907 100644 --- a/src/inlines.c +++ b/src/inlines.c @@ -230,6 +230,9 @@ static CMARK_INLINE cmark_chunk take_while(subject *subj, int (*f)(int)) { return cmark_chunk_dup(&subj->input, startpos, len); } +// Return the number of newlines in a given span of text in a subject. If +// the number is greater than zero, also return the number of characters +// between the last newline and the end of the span in `since_newline`. static int count_newlines(subject *subj, bufsize_t from, bufsize_t len, int *since_newline) { int nls = 0; int since_nl = 0; @@ -250,6 +253,9 @@ static int count_newlines(subject *subj, bufsize_t from, bufsize_t len, int *sin return nls; } +// Adjust `node`'s `end_line`, `end_column`, and `subj`'s `line` and +// `column_offset` according to the number of newlines in a just-matched span +// of text in `subj`. static void adjust_subj_node_newlines(subject *subj, cmark_node *node, int matchlen, int extra) { int since_newline; int newlines = count_newlines(subj, subj->pos - matchlen - extra, matchlen, &since_newline); From 2126ac2a387e6232108a3da05cfa939ff9332fac Mon Sep 17 00:00:00 2001 From: Yuki Izumi Date: Thu, 10 Aug 2017 13:39:36 +1000 Subject: [PATCH 6/6] Skip sourcepos tracking if disabled --- src/inlines.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/inlines.c b/src/inlines.c index 0cf9eb907..876bbde14 100644 --- a/src/inlines.c +++ b/src/inlines.c @@ -256,7 +256,11 @@ static int count_newlines(subject *subj, bufsize_t from, bufsize_t len, int *sin // Adjust `node`'s `end_line`, `end_column`, and `subj`'s `line` and // `column_offset` according to the number of newlines in a just-matched span // of text in `subj`. -static void adjust_subj_node_newlines(subject *subj, cmark_node *node, int matchlen, int extra) { +static void adjust_subj_node_newlines(subject *subj, cmark_node *node, int matchlen, int extra, int options) { + if (!(options & CMARK_OPT_SOURCEPOS)) { + return; + } + int since_newline; int newlines = count_newlines(subj, subj->pos - matchlen - extra, matchlen, &since_newline); if (newlines) { @@ -314,7 +318,7 @@ static bufsize_t scan_to_closing_backticks(subject *subj, // Parse backtick code section or raw backticks, return an inline. // Assumes that the subject has a backtick at the current position. -static cmark_node *handle_backticks(subject *subj) { +static cmark_node *handle_backticks(subject *subj, int options) { cmark_chunk openticks = take_while(subj, isbacktick); bufsize_t startpos = subj->pos; bufsize_t endpos = scan_to_closing_backticks(subj, openticks.len); @@ -331,7 +335,7 @@ static cmark_node *handle_backticks(subject *subj) { cmark_strbuf_normalize_whitespace(&buf); cmark_node *node = make_code(subj, startpos, endpos - openticks.len - 1, cmark_chunk_buf_detach(&buf)); - adjust_subj_node_newlines(subj, node, endpos - startpos, openticks.len); + adjust_subj_node_newlines(subj, node, endpos - startpos, openticks.len, options); return node; } } @@ -814,7 +818,7 @@ cmark_chunk cmark_clean_title(cmark_mem *mem, cmark_chunk *title) { // Parse an autolink or HTML tag. // Assumes the subject has a '<' character at the current position. -static cmark_node *handle_pointy_brace(subject *subj, bool liberal_html_tag) { +static cmark_node *handle_pointy_brace(subject *subj, int options) { bufsize_t matchlen = 0; cmark_chunk contents; @@ -844,17 +848,17 @@ static cmark_node *handle_pointy_brace(subject *subj, bool liberal_html_tag) { contents = cmark_chunk_dup(&subj->input, subj->pos - 1, matchlen + 1); subj->pos += matchlen; cmark_node *node = make_raw_html(subj, subj->pos - matchlen - 1, subj->pos - 1, contents); - adjust_subj_node_newlines(subj, node, matchlen, 1); + adjust_subj_node_newlines(subj, node, matchlen, 1, options); return node; } - if (liberal_html_tag) { + if (options & CMARK_OPT_LIBERAL_HTML_TAG) { matchlen = scan_liberal_html_tag(&subj->input, subj->pos); if (matchlen > 0) { contents = cmark_chunk_dup(&subj->input, subj->pos - 1, matchlen + 1); subj->pos += matchlen; cmark_node *node = make_raw_html(subj, subj->pos - matchlen - 1, subj->pos - 1, contents); - adjust_subj_node_newlines(subj, node, matchlen, 1); + adjust_subj_node_newlines(subj, node, matchlen, 1, options); return node; } } @@ -1212,7 +1216,7 @@ static int parse_inline(cmark_parser *parser, subject *subj, cmark_node *parent, new_inl = handle_newline(subj); break; case '`': - new_inl = handle_backticks(subj); + new_inl = handle_backticks(subj, options); break; case '\\': new_inl = handle_backslash(parser, subj); @@ -1221,7 +1225,7 @@ static int parse_inline(cmark_parser *parser, subject *subj, cmark_node *parent, new_inl = handle_entity(subj); break; case '<': - new_inl = handle_pointy_brace(subj, (options & CMARK_OPT_LIBERAL_HTML_TAG) != 0); + new_inl = handle_pointy_brace(subj, options); break; case '*': case '_':