Find Table Cell Boundaries Before Blanking Code Spans and Quotations

.github/actions/prose-gate/prose_lint.py

@@ -997,6 +997,7 @@ class Context(NamedTuple):
 LABELED_ITEM = re.compile(rf"^\s*[*_]*{LABEL_WORD}(?:\s+{LABEL_WORD}){{0,2}}[*_]*:[*_]*(?:\s|$)")
 LIST_MARKER = re.compile(rf"^\s*{LIST_MARKER_TOKEN}\s")
 CELL_BREAK = re.compile(r"(?<!\\)\|")
+TABLE_ROW = re.compile(r"^\s*(?:>\s*)*\|")
 # A word-joining character before the repeat keeps "either/or or" from reading as a doubled word.
 DUPWORD = re.compile(r"(?<![\w/-])(\w+)\s+\1\b", re.IGNORECASE)
 # A step marker is a label on the sentence that follows, so its dot terminates nothing.
@@ -1039,12 +1040,29 @@ def strip_list_marker(text: str) -> str:
     return LIST_MARKER.sub("", text, count=1).strip()
 
 
+def row_cells(text: str) -> list[str]:
+    """A table row opening with a pipe split into its cells, and any other line left whole.
+
+    A cell ends at a pipe no backslash escapes, inside a code span too, which is how markdown-it-py
+    splits a row. So no code span or quotation read from a cell reaches into the next one.
+    """
+    return CELL_BREAK.split(text) if TABLE_ROW.match(text) else [text]
+
+
+def by_cell(text: str, blank: Callable[[str], str]) -> str:
+    """Blank a line a cell at a time, keeping every pipe that ends a cell where it was.
+
+    An unpaired quote mark in one cell would otherwise hide every cell up to the next quote.
+    """
+    return "|".join(blank(cell) for cell in row_cells(text))
+
+
 def prose_lines(unit: Unit) -> Iterable[tuple[int, str, str]]:
     """Each line's number, its text without code spans, and that text without quotations either."""
     for line in unit.lines:
-        txt = strip_inline_code(line.text)
-        prose = QUOTE_PREFIX.sub(lambda m: " " * len(m.group()), strip_quoted(txt), count=1)
-        yield line.number, txt, prose
+        txt = by_cell(line.text, strip_inline_code)
+        prose = by_cell(txt, strip_quoted)
+        yield line.number, txt, QUOTE_PREFIX.sub(lambda m: " " * len(m.group()), prose, count=1)
 
 
 def in_numeric_context(line: str, pos: int) -> bool:
@@ -1100,14 +1118,14 @@ def rule_charset(unit: Unit, ctx: Context) -> list[Finding]:
 def rule_semicolon(unit: Unit, ctx: Context) -> list[Finding]:
     """A Markdown semicolon, unless its sentence is a list carrying a comma that needs one.
 
-    A table row is read a cell at a time. A cell ends at a pipe no backslash escapes, which is how
-    markdown-it-py reads an escaped pipe.
+    A table row opening with a pipe is read a cell at a time, split where `row_cells` splits it.
+    `prose_lines` blanks each cell on its own and leaves every pipe ending one where it was.
     """
     out: list[Finding] = []
     for n, _txt, line in prose_lines(unit):
-        row = line.lstrip().startswith("|")
-        cells = CELL_BREAK.split(line.strip().strip("|")) if row else [line]
-        pieces = (s for cell in cells for s in SENTENCE_BREAK.split(LABEL_COLON.sub("", cell)))
+        pieces = (
+            s for cell in row_cells(line) for s in SENTENCE_BREAK.split(LABEL_COLON.sub("", cell))
+        )
         for sentence in (s for s in pieces if s.strip()):
             head, _, tail = sentence.partition(";")
             listish = sentence.count(";") > 1 or (
@@ -1134,8 +1152,11 @@ def unit_texts(unit: Unit, ctx: Context) -> Iterable[tuple[int, str]]:
     for line in unit.lines:
         if line.number in ctx.fenced:
             continue
-        text = strip_quoted(strip_inline_code(line.text)) if unit.kind == "prose" else line.text
-        yield line.number, strip_inline_code(text)
+        if unit.kind != "prose":
+            yield line.number, strip_inline_code(line.text)
+            continue
+        text = by_cell(by_cell(line.text, strip_inline_code), strip_quoted)
+        yield line.number, by_cell(text, strip_inline_code)
 
 
 def rule_dupword(unit: Unit, ctx: Context) -> list[Finding]:
@@ -1252,11 +1273,12 @@ def rule_dead_path(unit: Unit, ctx: Context) -> list[Finding]:
         tokens = (
             [(m.group(1), False)]
             if m
-            else [(s.group(1), True) for s in INLINE_SPAN.finditer(line.text)]
+            else [(s.group(1), True) for c in row_cells(line.text) for s in INLINE_SPAN.finditer(c)]
         )
         if not m:
             tokens += [
-                (t.group(1), False) for t in LINK_TARGET.finditer(strip_inline_code(line.text))
+                (t.group(1), False)
+                for t in LINK_TARGET.finditer(by_cell(line.text, strip_inline_code))
             ]
         for token, in_span in tokens:
             rel_path = path_candidate(token, in_span)

scripts/README.md

@@ -85,11 +85,11 @@ The `spelling` rule covers the US English convention where cspell does not reach
 
 **Scope** is every text file git tracks, binaries skipped by a NUL-byte check, with no extension allowlist. An allowlist covers what its author thought of and silently stops covering whatever is added next. That is the same reason the line-endings rule already requires `git ls-files` over a raw `find`. `--list-files` prints the discovered set for auditing.
 
-A double-quoted span in Markdown is treated as a quotation and not scanned for prose rules. A rule that states its own counter-example therefore does not report the document that documents it. Outside Markdown a double quote is structural, so the prose inside it still counts.
+A double-quoted span in Markdown is treated as a quotation and not scanned for prose rules. A rule that states its own counter-example therefore does not report the document that documents it. Outside Markdown a double quote is structural, so the prose inside it still counts. A table row opening with a pipe has its code spans and quotations blanked one cell at a time. markdown-it-py ends a cell at an unescaped pipe inside a code span as well. A quotation blanked across cells would also hide every cell between its two marks.
 
 The `semicolon` and `dash` rules ban a construction rather than a detectable subset of it. Each therefore flags by default, and the exceptions are the ones the rule names. The exceptions for the dash are a compound word, a leading list marker, a range, and the `- **Label** - explanation` separator that opens a governed bullet. The exception for a semicolon is one inside a list that already carries commas. Where such a list is joined by a single semicolon, each of its two items carries a label.
 
-**The semicolon rule reads the list where it lives.** The comma qualifies the list as a whole rather than one separator's position. An enumeration whose commas fall in a later item therefore keeps every semicolon it carries. Reading it positionally split one series in two. That reading flagged the openers of the same list it then exempted the tail of. That would have restructured the enumerated guarantees the exemption exists to protect. A Markdown table row is judged one cell at a time. This is because a row is a record of fields, and a comma in one column cannot excuse a semicolon in another. A cell ends at a pipe no backslash escapes, which is how markdown-it-py reads an escaped pipe, so the escaped pipe stays inside its cell. A bullet's `**Label**:` is dropped before the line is read, because it opens the bullet rather than announcing a list. The `**Label**:` is the same construct the label dash is exempted for. The colon is written inside the emphasis as often as outside it, so `**Label:**` is dropped on the same grounds. Matching only one spelling left the other announcing a list it never announced.
+**The semicolon rule reads the list where it lives.** The comma qualifies the list as a whole rather than one separator's position. An enumeration whose commas fall in a later item therefore keeps every semicolon it carries. Reading it positionally split one series in two. That reading flagged the openers of the same list it then exempted the tail of. That would have restructured the enumerated guarantees the exemption exists to protect. A Markdown table row opening with a pipe is judged one cell at a time. This is because a row is a record of fields, and a comma in one column cannot excuse a semicolon in another. A cell ends at a pipe no backslash escapes, which is how markdown-it-py reads an escaped pipe, so the escaped pipe stays inside its cell. A bullet's `**Label**:` is dropped before the line is read, because it opens the bullet rather than announcing a list. The `**Label**:` is the same construct the label dash is exempted for. The colon is written inside the emphasis as often as outside it, so `**Label:**` is dropped on the same grounds. Matching only one spelling left the other announcing a list it never announced.
 
 **The sentence is the unit the exemption is judged on, because that is where a list lives.** The whole bullet decided it once. A colon anywhere before the first semicolon therefore marked the bullet a list and exempted every semicolon after it. That held however plainly one joined two independent clauses. The colon and the semicolon did not have to be near each other or related at all. Measured over this repo when it was fixed, the exemption was covering 62 spans holding 120 semicolons across 9 files. The rule reported none of them, so the gate read as clean over the docs it exists to check. Scoping it to the sentence reported 43 further semicolons and silenced none, with a 44th from dropping the other spelling of the label colon. The sentence boundary is the run-on rule's. So an initial or an abbreviation ends nothing, and a terminator closing inside emphasis or a bracket (`.**`, `.)`) still ends a sentence. A lone semicolon separates a list only where the items on both sides of it open with a label, as in `Inputs: a, b; outputs: c`. No structure tells an explanatory colon from a list colon. A colon and a comma alone once exempted the one semicolon joining two clauses after them. A sentence with two or more semicolons and a comma stays a series whatever colon it holds.
 

tests/test_prose_lint.py

@@ -654,6 +654,20 @@ def test_an_escaped_pipe_in_a_blockquoted_row_stays_inside_its_cell(self) -> Non
             [], self.kinds("> | x | red, green \\| blue; cyan; teal |\n", {"semicolon"})
         )
 
+    def test_a_pipe_inside_a_code_span_still_ends_its_cell(self) -> None:
+        """The parser ends a cell at a pipe inside a code span, so the comma before it excuses nothing."""
+        table = "| A | B | C |\n| --- | --- | --- |\n| x | a, b `c|d` e; f; g | z |\n"
+        self.assertEqual(["semicolon"] * 2, self.kinds(table, {"semicolon"}))
+
+    def test_an_unpaired_quote_does_not_pair_with_a_later_cell(self) -> None:
+        """An inch mark in one cell would blank every cell up to the next quote mark."""
+        table = '| A | B | C |\n| --- | --- | --- |\n| 2" pipe | a; b; c | "x" |\n'
+        self.assertEqual(["semicolon"] * 2, self.kinds(table, {"semicolon"}))
+
+    def test_a_quotation_inside_one_cell_is_still_blanked(self) -> None:
+        table = '| A | B |\n| --- | --- |\n| x | say "a; b" now |\n'
+        self.assertEqual([], self.kinds(table, {"semicolon"}))
+
     def test_a_bullet_label_colon_does_not_announce_a_list(self) -> None:
         """`- **Label**:` opens the bullet, the same construct the label dash is exempted for."""
         self.assertEqual(
@@ -3546,6 +3560,17 @@ def test_a_deleted_path_in_a_span_is_flagged(self) -> None:
         root = self.repo()
         self.assertIn("dead-path", self.kinds(root, "Run `scripts/gone.py` to apply.\n"))
 
+    def test_a_link_between_two_cells_backticks_is_still_read(self) -> None:
+        """markdown-it-py ends a cell inside a code span, so the backticks pair across no cell."""
+        root = self.repo()
+        table = "| A | B | C |\n| --- | --- | --- |\n| x `a | [d](scripts/gone.py) | b` |\n"
+        self.assertIn("dead-path", self.kinds(root, table))
+
+    def test_a_lone_backtick_in_an_earlier_cell_hides_no_later_span(self) -> None:
+        root = self.repo()
+        table = "| A | B |\n| --- | --- |\n| 2` x | `scripts/gone.py` |\n"
+        self.assertIn("dead-path", self.kinds(root, table))
+
     def test_a_living_path_is_not_a_finding(self) -> None:
         root = self.repo()
         self.assertEqual([], self.kinds(root, "Run `scripts/kept.py` to apply.\n"))
@@ -4810,6 +4835,17 @@ def test_both_markers_are_read(self) -> None:
 class TestMarkdown(BaitCase):
     """Markdown extraction through markdown-it-py: code, blocks, HTML, and front matter."""
 
+    def test_an_unpaired_quote_hides_no_later_cell_from_any_rule(self) -> None:
+        """Each rule blanks quotations a cell at a time, so an inch mark pairs with nothing later."""
+        for rule, cell in (("dash", "a - b"), ("dupword", "the the"), ("spelling", "colour")):
+            table = f'| A | B | C |\n| --- | --- | --- |\n| 2" pipe | {cell} | "x" |\n'
+            with self.subTest(rule=rule):
+                self.assertEqual([rule], self.kinds(table, {rule}))
+
+    def test_a_code_span_reaches_across_no_cell(self) -> None:
+        table = "| A | B | C |\n| --- | --- | --- |\n| x `a | p - q | b` |\n"
+        self.assertEqual(["dash"], self.kinds(table, {"dash"}))
+
     def test_structure_is_not_a_sentence(self) -> None:
         """A heading, a link definition, and a blockquote hold no prose sentence.