Sourcepos fixes

api_test/main.c

@@ -936,7 +936,7 @@ static void test_feed_across_line_ending(test_batch_runner *runner) {
 
 static void source_pos(test_batch_runner *runner) {
   static const char markdown[] =
-    "Hi *there*.\n"
+    "# Hi *there*.\n"
     "\n"
     "Hello &ldquo; <http://www.google.com>\n"
     "there `hi` -- [okay](www.google.com (ok)).\n"
@@ -952,13 +952,13 @@ static void source_pos(test_batch_runner *runner) {
   STR_EQ(runner, xml, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                       "<!DOCTYPE document SYSTEM \"CommonMark.dtd\">\n"
                       "<document sourcepos=\"1:1-10:20\" xmlns=\"http://commonmark.org/xml/1.0\">\n"
-                      "  <paragraph sourcepos=\"1:1-1:11\">\n"
-                      "    <text sourcepos=\"1:1-1:3\">Hi </text>\n"
-                      "    <emph sourcepos=\"1:4-1:10\">\n"
-                      "      <text sourcepos=\"1:5-1:9\">there</text>\n"
+                      "  <heading sourcepos=\"1:1-1:13\" level=\"1\">\n"
+                      "    <text sourcepos=\"1:3-1:5\">Hi </text>\n"
+                      "    <emph sourcepos=\"1:6-1:12\">\n"
+                      "      <text sourcepos=\"1:7-1:11\">there</text>\n"
                       "    </emph>\n"
-                      "    <text sourcepos=\"1:11-1:11\">.</text>\n"
-                      "  </paragraph>\n"
+                      "    <text sourcepos=\"1:13-1:13\">.</text>\n"
+                      "  </heading>\n"
                       "  <paragraph sourcepos=\"3:1-4:42\">\n"
                       "    <text sourcepos=\"3:1-3:14\">Hello “ </text>\n"
                       "    <link sourcepos=\"3:15-3:37\" destination=\"http://www.google.com\" title=\"\">\n"
@@ -1048,26 +1048,26 @@ static void ext_source_pos(test_batch_runner *runner) {
 		      "    <item sourcepos=\"5:1-7:18\">\n"
                       "      <table sourcepos=\"5:4-7:18\">\n"
                       "        <table_header sourcepos=\"5:4-5:18\">\n"
-                      "          <table_cell sourcepos=\"5:6-5:7\">\n"
+                      "          <table_cell sourcepos=\"5:5-5:7\">\n"
                       "            <text sourcepos=\"5:6-5:6\">a</text>\n"
                       "          </table_cell>\n"
-                      "          <table_cell sourcepos=\"5:10-5:11\">\n"
+                      "          <table_cell sourcepos=\"5:9-5:11\">\n"
                       "            <text sourcepos=\"5:10-5:10\">b</text>\n"
                       "          </table_cell>\n"
-                      "          <table_cell sourcepos=\"5:14-5:17\">\n"
+                      "          <table_cell sourcepos=\"5:13-5:17\">\n"
                       "            <emph sourcepos=\"5:14-5:16\">\n"
                       "              <text sourcepos=\"5:15-5:15\">c</text>\n"
                       "            </emph>\n"
                       "          </table_cell>\n"
                       "        </table_header>\n"
                       "        <table_row sourcepos=\"7:4-7:18\">\n"
-                      "          <table_cell sourcepos=\"7:6-7:7\">\n"
+                      "          <table_cell sourcepos=\"7:5-7:7\">\n"
                       "            <text sourcepos=\"7:6-7:6\">1</text>\n"
                       "          </table_cell>\n"
-                      "          <table_cell sourcepos=\"7:10-7:11\">\n"
+                      "          <table_cell sourcepos=\"7:9-7:11\">\n"
                       "            <text sourcepos=\"7:10-7:10\">2</text>\n"
                       "          </table_cell>\n"
-                      "          <table_cell sourcepos=\"7:14-7:17\">\n"
+                      "          <table_cell sourcepos=\"7:13-7:17\">\n"
                       "            <strikethrough sourcepos=\"7:14-7:16\">\n"
                       "              <text sourcepos=\"7:15-7:15\">3</text>\n"
                       "            </strikethrough>\n"

extensions/table.c

@@ -28,7 +28,7 @@ typedef struct {
 
 typedef struct {
   cmark_strbuf *buf;
-  int start_offset, end_offset;
+  int start_offset, end_offset, internal_offset;
 } node_cell;
 
 static void free_table_cell(cmark_mem *mem, void *data) {
@@ -133,6 +133,10 @@ static table_row *row_from_string(cmark_syntax_extension *self,
       cell->buf = cell_buf;
       cell->start_offset = offset;
       cell->end_offset = offset + cell_matched - 1;
+      while (cell->start_offset > 0 && string[cell->start_offset - 1] != '|') {
+        --cell->start_offset;
+        ++cell->internal_offset;
+      }
       row->n_columns += 1;
       row->cells = cmark_llist_append(parser->mem, row->cells, cell);
     }
@@ -249,6 +253,7 @@ static cmark_node *try_opening_table_header(cmark_syntax_extension *self,
       cmark_node *header_cell = cmark_parser_add_child(parser, table_header,
           CMARK_NODE_TABLE_CELL, parent_container->start_column + cell->start_offset);
       header_cell->start_line = header_cell->end_line = parent_container->start_line;
+      header_cell->internal_offset = cell->internal_offset;
       header_cell->end_column = parent_container->start_column + cell->end_offset;
       cmark_node_set_string_content(header_cell, (char *) cell->buf->ptr);
       cmark_node_set_syntax_extension(header_cell, self);
@@ -292,6 +297,7 @@ static cmark_node *try_opening_table_row(cmark_syntax_extension *self,
       node_cell *cell = (node_cell *) tmp->data;
       cmark_node *node = cmark_parser_add_child(parser, table_row_block,
           CMARK_NODE_TABLE_CELL, parent_container->start_column + cell->start_offset);
+      node->internal_offset = cell->internal_offset;
       node->end_column = parent_container->start_column + cell->end_offset;
       cmark_node_set_string_content(node, (char *) cell->buf->ptr);
       cmark_node_set_syntax_extension(node, self);

src/blocks.c

@@ -980,6 +980,7 @@ static void open_new_blocks(cmark_parser *parser, cmark_node **container,
 
       (*container)->as.heading.level = level;
       (*container)->as.heading.setext = false;
+      (*container)->internal_offset = matched;
 
     } else if (!indented && (matched = scan_open_code_fence(
                                  input, parser->first_nonspace))) {

src/inlines.c

@@ -230,6 +230,47 @@ static CMARK_INLINE cmark_chunk take_while(subject *subj, int (*f)(int)) {
   return cmark_chunk_dup(&subj->input, startpos, len);
 }
 
+// Return the number of newlines in a given span of text in a subject.  If
+// the number is greater than zero, also return the number of characters
+// between the last newline and the end of the span in `since_newline`.
+static int count_newlines(subject *subj, bufsize_t from, bufsize_t len, int *since_newline) {
+  int nls = 0;
+  int since_nl = 0;
+
+  while (len--) {
+    if (subj->input.data[from++] == '\n') {
+      ++nls;
+      since_nl = 0;
+    } else {
+      ++since_nl;
+    }
+  }
+
+  if (!nls)
+    return 0;
+
+  *since_newline = since_nl;
+  return nls;
+}
+
+// Adjust `node`'s `end_line`, `end_column`, and `subj`'s `line` and
+// `column_offset` according to the number of newlines in a just-matched span
+// of text in `subj`.
+static void adjust_subj_node_newlines(subject *subj, cmark_node *node, int matchlen, int extra, int options) {
+  if (!(options & CMARK_OPT_SOURCEPOS)) {
+    return;
+  }
+
+  int since_newline;
+  int newlines = count_newlines(subj, subj->pos - matchlen - extra, matchlen, &since_newline);
+  if (newlines) {
+    subj->line += newlines;
+    node->end_line += newlines;
+    node->end_column = since_newline;
+    subj->column_offset = -subj->pos + since_newline + extra;
+  }
+}
+
 // Try to process a backtick code span that began with a
 // span of ticks of length openticklength length (already
 // parsed).  Return 0 if you don't find matching closing
@@ -277,7 +318,7 @@ static bufsize_t scan_to_closing_backticks(subject *subj,
 
 // Parse backtick code section or raw backticks, return an inline.
 // Assumes that the subject has a backtick at the current position.
-static cmark_node *handle_backticks(subject *subj) {
+static cmark_node *handle_backticks(subject *subj, int options) {
   cmark_chunk openticks = take_while(subj, isbacktick);
   bufsize_t startpos = subj->pos;
   bufsize_t endpos = scan_to_closing_backticks(subj, openticks.len);
@@ -293,7 +334,9 @@ static cmark_node *handle_backticks(subject *subj) {
     cmark_strbuf_trim(&buf);
     cmark_strbuf_normalize_whitespace(&buf);
 
-    return make_code(subj, startpos, endpos - openticks.len - 1, cmark_chunk_buf_detach(&buf));
+    cmark_node *node = make_code(subj, startpos, endpos - openticks.len - 1, cmark_chunk_buf_detach(&buf));
+    adjust_subj_node_newlines(subj, node, endpos - startpos, openticks.len, options);
+    return node;
   }
 }
 
@@ -775,7 +818,7 @@ cmark_chunk cmark_clean_title(cmark_mem *mem, cmark_chunk *title) {
 
 // Parse an autolink or HTML tag.
 // Assumes the subject has a '<' character at the current position.
-static cmark_node *handle_pointy_brace(subject *subj, bool liberal_html_tag) {
+static cmark_node *handle_pointy_brace(subject *subj, int options) {
   bufsize_t matchlen = 0;
   cmark_chunk contents;
 
@@ -804,15 +847,19 @@ static cmark_node *handle_pointy_brace(subject *subj, bool liberal_html_tag) {
   if (matchlen > 0) {
     contents = cmark_chunk_dup(&subj->input, subj->pos - 1, matchlen + 1);
     subj->pos += matchlen;
-    return make_raw_html(subj, subj->pos - 1 - matchlen, subj->pos - 1, contents);
+    cmark_node *node = make_raw_html(subj, subj->pos - matchlen - 1, subj->pos - 1, contents);
+    adjust_subj_node_newlines(subj, node, matchlen, 1, options);
+    return node;
   }
 
-  if (liberal_html_tag) {
+  if (options & CMARK_OPT_LIBERAL_HTML_TAG) {
     matchlen = scan_liberal_html_tag(&subj->input, subj->pos);
     if (matchlen > 0) {
       contents = cmark_chunk_dup(&subj->input, subj->pos - 1, matchlen + 1);
       subj->pos += matchlen;
-      return make_raw_html(subj, subj->pos - 1 - matchlen, subj->pos - 1, contents);
+      cmark_node *node = make_raw_html(subj, subj->pos - matchlen - 1, subj->pos - 1, contents);
+      adjust_subj_node_newlines(subj, node, matchlen, 1, options);
+      return node;
     }
   }
 
@@ -1169,7 +1216,7 @@ static int parse_inline(cmark_parser *parser, subject *subj, cmark_node *parent,
     new_inl = handle_newline(subj);
     break;
   case '`':
-    new_inl = handle_backticks(subj);
+    new_inl = handle_backticks(subj, options);
     break;
   case '\\':
     new_inl = handle_backslash(parser, subj);
@@ -1178,7 +1225,7 @@ static int parse_inline(cmark_parser *parser, subject *subj, cmark_node *parent,
     new_inl = handle_entity(subj);
     break;
   case '<':
-    new_inl = handle_pointy_brace(subj, (options & CMARK_OPT_LIBERAL_HTML_TAG) != 0);
+    new_inl = handle_pointy_brace(subj, options);
     break;
   case '*':
   case '_':
@@ -1240,7 +1287,7 @@ void cmark_parse_inlines(cmark_parser *parser,
                          cmark_reference_map *refmap,
                          int options) {
   subject subj;
-  subject_from_buf(parser->mem, parent->start_line, parent->start_column - 1, &subj, &parent->content, refmap);
+  subject_from_buf(parser->mem, parent->start_line, parent->start_column - 1 + parent->internal_offset, &subj, &parent->content, refmap);
   cmark_chunk_rtrim(&subj.input);
 
   while (!is_eof(&subj) && parse_inline(parser, &subj, parent, options))

src/node.h

@@ -68,6 +68,7 @@ struct cmark_node {
   int start_column;
   int end_line;
   int end_column;
+  int internal_offset;
   uint16_t type;
   uint16_t flags;