Read Digit-Labelled and Second Heredocs, Skip Non-Shell Run Splits, and Pair Backticks as CommonMark Does

.github/actions/prose-gate/prose_lint.py

@@ -38,6 +38,7 @@
 MISSING = ""
 try:
     from markdown_it import MarkdownIt
+    from markdown_it.common.html_re import HTML_TAG_RE
     from markdown_it.token import Token
     from pygments.lexer import Lexer, RegexLexer, bygroups
     from pygments.lexers import get_lexer_by_name, get_lexer_for_filename
@@ -527,7 +528,12 @@ def lexer_name(path: Path, first: str) -> str | None:
 
 # Pygments ends a heredoc at any later `STUB` in a `$STUB_DIR`, and reads no double-quoted label.
 # This rule ends one only on its terminator line, tab-indented for `<<-`.
-HEREDOC = r"(?s)<<(-)?[ \t]*(['\"]?)\\?([A-Za-z_][\w-]*)\2[^\n]*\n(?:(?:[^\n]*\n)*?(?(1)\t*)\3(?=\n|\Z)|.*)"
+# A label opening on a digit is more often a shift, `(( x << 2 ))`, so it needs its terminator.
+HEREDOC = (
+    r"(?s)<<(-)?[ \t]*(['\"]?)\\?(?:(?=[A-Za-z_])())?(\w[\w-]*+)\2"
+    r"(?:[^\n'\"#]*?(?<!<)<<(-)?[ \t]*(['\"]?)\\?(\w[\w-]*+)\6)?[^\n]*\n(?:(?:[^\n]*\n)*?(?(1)\t*)\4"
+    r"(?=\n|\Z)(?(7)(?:\n(?:[^\n]*\n)*?(?(5)\t*)\7(?=\n|\Z))?)|(?(3).*|(?!)))"
+)
 # Pygments reads a C# collection expression opening on its own line as an attribute ending at `]`.
 # The comments inside it are lost, so an attribute here is read within its own line.
 ATTRIBUTE = r"^([ \t]*)(\[[^\n]*?\])"
@@ -641,21 +647,21 @@ def comment_lines(
     return out
 
 
-def nested_spans(text: str, rules: list[dict[str, str]]) -> list[tuple[int, int, str]]:
-    """The YAML scalars under a key or a tag the data names, as (start, end, lexer) offsets."""
+def nested_spans(text: str, rules: list[dict[str, str]]) -> list[tuple[int, int, str, int]]:
+    """The YAML scalars under a key or a tag the data names, as (start, end, lexer, key) offsets."""
     keys = {r["key"]: r["lexer"] for r in rules if "key" in r}
     tags = {r["tag"]: r["lexer"] for r in rules if "tag" in r}
     tokens = list(lexer_for("yaml").get_tokens_unprocessed(text))
-    out: list[tuple[int, int, str]] = []
+    out: list[tuple[int, int, str, int]] = []
     pending: str | None = None
-    i = 0
+    i = key = 0
     while i < len(tokens):
         offset, kind, value = tokens[i]
         i += 1
         if kind in Name.Tag and value in keys:
-            pending = keys[value]
+            pending, key = keys[value], offset
         elif kind in Keyword.Type and value in tags:
-            pending = tags[value]
+            pending, key = tags[value], offset
         elif pending and kind in Punctuation.Indicator and value in "|>":
             # The rest of the header line, a comment after the indicator included, opens nothing.
             while i < len(tokens) and "\n" not in tokens[i][2]:
@@ -668,20 +674,36 @@ def nested_spans(text: str, rules: list[dict[str, str]]) -> list[tuple[int, int,
             if j > i:
                 # The span opens after the header's own line break, so its lines map one to one.
                 start = text.index("\n", offset) + 1
-                out.append((start, max(start, tokens[j - 1][0] + len(tokens[j - 1][2])), pending))
+                out.append(
+                    (start, max(start, tokens[j - 1][0] + len(tokens[j - 1][2])), pending, key)
+                )
             i, pending = j, None
         elif pending and kind in String and value in "'\"":
             j = i
             while j < len(tokens) and not (tokens[j][1] in String and tokens[j][2] == value):
                 j += 1
             if j < len(tokens):
-                out.append((offset + 1, tokens[j][0], pending))
+                out.append((offset + 1, tokens[j][0], pending, key))
             i, pending = j + 1, None
         elif not (kind in Text.Whitespace or kind in Punctuation or kind in Keyword.Type):
             pending = None
     return out
 
 
+def step_shell(starts: Starts, key: int) -> str:
+    """The program `shell:` names beside the YAML key at offset `key`, or "" where none is named."""
+    (row, col), last = starts.at(key), len(starts.starts)
+    inside = lambda n: not starts.line(n)[:col].strip() or starts.line(n).lstrip().startswith("#")
+    top, end = row, row + 1
+    while top > 1 and inside(top):
+        top -= 1
+    top += not inside(top) and starts.line(top)[:col].strip() != "-"
+    while end <= last and inside(end):
+        end += 1
+    sibling = re.compile(rf"[ \t-]{{{col}}}shell:[ \t]*['\"]?(?:\S*/)?([\w-]+)")
+    return next((m[1] for n in range(top, end) if (m := sibling.match(starts.line(n)))), "")
+
+
 def split_heredocs(text: str) -> list[tuple[int, str]]:
     """A script as its code with heredoc bodies blanked, then each body, as (line offset, text).
 
@@ -708,12 +730,12 @@ def nested_comments(text: str, spec: dict[str, Any]) -> list[tuple[Line, bool]]:
     """Comments in the scalars a YAML file hands to another lexer, numbered by the file's lines."""
     starts = Starts(text)
     out: list[tuple[Line, bool]] = []
-    for start, end, lexer in nested_spans(text, spec.get("nested", [])):
+    for start, end, lexer, key in nested_spans(text, spec.get("nested", [])):
         number, col = starts.at(start)
         inner = language_of(lexer) or {}
         # The column pad keeps a one-line scalar's comment trailing, and C++ ends a comment on a break.
         body = text[start:end]
-        if lexer == "bash-heredoc-as-code":
+        if lexer == "bash-heredoc-as-code" and step_shell(starts, key) in ("", "bash", "sh"):
             # Each body starts on a line of its own, so only the code piece takes the column pad.
             for skip, piece in split_heredocs(body):
                 pad = "x" * col if skip == 0 else ""
@@ -869,8 +891,8 @@ def inline_comments(tok: Token, lines: list[str]) -> list[Line]:
     first, last = (tok.map or [0, 1])[0], (tok.map or [0, 1])[1]
     number, col = first + 1, 0
     # A code span can hold a comment's text, so it is blanked, lengths kept, before the search.
-    blank = lambda m: re.sub(r"[^\n]", " ", m.group())
-    seen = re.sub(CODE_SPAN, blank, "\n".join(lines[first:last])).split("\n")
+    blank = lambda m: re.sub(r"[^\n]", " ", m.group()) if m[1] else m.group()
+    seen = re.sub(MARKDOWN_SPAN, blank, "\n".join(lines[first:last])).split("\n")
     for child in tok.children or ():
         if child.type == "html_inline" and child.content.startswith("<!--"):
             # The source line is found by the comment's own text, searched on from the last one.
@@ -1405,6 +1427,7 @@ def rule_comment_added(unit: Unit, ctx: Context) -> list[Finding]:
 
 
 CODE_SPAN = re.compile(r"(?<!`)(`+)(?!`).+?(?<!`)\1(?!`)", re.DOTALL)
+MARKDOWN_SPAN = re.compile(rf"\\.|{HTML_TAG_RE.pattern[1:]}|{CODE_SPAN.pattern}", re.DOTALL)
 
 
 def mask_code_and_quotes(text: str, starts: list[int], lengths: list[int]) -> str:

tests/test_prose_lint.py

@@ -1230,10 +1230,89 @@ def test_a_hyphenated_delimiter_ends_a_shell_file_heredoc(self) -> None:
         text = "#!/usr/bin/env bash\ncat <<END-OF-TEXT\nhi\nEND-OF-TEXT\n# retry the upload.\n"
         self.assertEqual([5], [c[0] for c in self.comments("h.sh", text)])
 
+    def test_a_delimiter_opening_on_a_digit_ends_on_its_terminator(self) -> None:
+        """The apostrophe in a `<<123` body opened a string hiding the comment after it."""
+        rows = ("cat <<123", "The build didn't publish.", "123", "# retry the upload.", "jq '.x'")
+        for name, text in (
+            ("x.yml", "- run: |\n" + "".join(f"    {r}\n" for r in rows)),
+            ("h.sh", "".join(f"{r}\n" for r in rows)),
+        ):
+            got = [c[:2] for c in self.comments(name, text)]
+            self.assertEqual([(text.count("\n") - 1, "retry the upload.")], got, name)
+
+    def test_a_shift_by_a_number_opens_no_heredoc(self) -> None:
+        """Bash reads `<<2` as a shift in arithmetic, so a label opening on a digit needs its line."""
+        for code in ("(( x << 2 ))", "a[1<<2]=v", "echo $[1<<2]", "let x=1<<2", "echo $((1<<2))"):
+            with self.subTest(code=code):
+                text = f"{code}\nThe build didn't publish.\n# After.\n"
+                self.assertEqual([3], [c[0] for c in self.comments("a.sh", text)])
+
+    def test_a_second_heredoc_on_one_opener_line_ends_on_its_own_terminator(self) -> None:
+        rows = ("diff - /dev/fd/3 <<A 3<<'B'", "one", "A", "The build didn't publish.", "B")
+        tail = ("# retry the upload.", "jq -r '.x' o.json")
+        for name, text in (
+            ("x.yml", "- run: |\n" + "".join(f"    {r}\n" for r in rows + tail)),
+            ("h.sh", "".join(f"{r}\n" for r in rows + tail)),
+        ):
+            got = [c[:2] for c in self.comments(name, text)]
+            self.assertEqual([(text.count("\n") - 1, "retry the upload.")], got, name)
+
+    def test_a_second_opener_with_no_terminator_leaves_the_first_body_alone(self) -> None:
+        text = "cat <<A <<<x y<<z\nbody\nA\n# After.\n"
+        self.assertEqual([(4, "After.")], [c[:2] for c in self.comments("a.sh", text)])
+
+    def test_a_quoted_or_commented_opener_queues_no_second_body(self) -> None:
+        """A `<<EOF` after a quote or a `#` ran the body on to a later heredoc's `EOF`."""
+        for opener in ("cat <<EOF | grep -c '<<EOF'", "cat <<EOF  # read up to the <<EOF line"):
+            with self.subTest(opener=opener):
+                text = f"{opener}\nx\nEOF\n# After.\ncat <<EOF\ny\nEOF\n"
+                self.assertEqual([4], [c[0] for c in self.comments("a.sh", text)])
+
+    def test_a_dash_second_opener_ends_on_its_tab_indented_terminator(self) -> None:
+        text = "diff - /dev/fd/3 <<A 3<<-B\none\nA\nThe build didn't.\n\tB\n# After.\njq '.x'\n"
+        self.assertEqual([6], [c[0] for c in self.comments("a.sh", text)])
+
+    def test_a_non_shell_run_step_takes_no_heredoc_split(self) -> None:
+        """A Python shift by a name read as an opener cut a triple-quoted string in two."""
+        script = '    m = 1 << n; s = """\n    # not a comment\n    """\n    # real\n'
+        for step in (
+            "- shell: python\n  run: |\n" + script,
+            "- name: x\n  run: |\n" + script + "  shell: python\n",
+            "- shell: pwsh {0}\n  run: |\n" + script,
+            "- shell: 'python'\n  run: |\n" + script,
+            "- shell: /usr/bin/python3 {0}\n# name: old\n  run: |\n" + script,
+        ):
+            with self.subTest(step=step):
+                got = [c[1] for c in self.comments("x.yml", step) if c[1] != "name: old"]
+                self.assertEqual(["real"], got)
+        for step in ("- shell: bash -e {0}\n  run: |\n", "- shell: python\n- run: |\n"):
+            with self.subTest(step=step):
+                got = [c[1] for c in self.comments("x.yml", step + script)]
+                self.assertEqual(["not a comment"], got)
+
     def test_a_code_span_sharing_words_with_earlier_prose_does_not_move_the_comment(self) -> None:
         text = "I see this, `see <!-- check this. --> this`\n    <!-- check this. -->\n"
         self.assertEqual([2], [c[0] for c in self.comments("a.md", text)])
 
+    def test_a_stray_backtick_does_not_hide_the_first_of_two_equal_comments(self) -> None:
+        """An escaped, commented, or attribute backtick paired with a later one and blanked both."""
+        for stray in ("\\`", "<!-- a ` -->", '<a title="`">'):
+            with self.subTest(stray=stray):
+                text = f"Start {stray} and <!-- x --> one\nthen \\` and\nlater <!-- x --> two.\n"
+                got = [c[0] for c in self.comments("a.md", text) if c[1] == "x"]
+                self.assertEqual([1, 3], got)
+
+    def test_a_less_than_sign_opens_no_tag_around_a_code_span(self) -> None:
+        """A tag is what the Markdown parser reads as one, so the code span after `a<b` is blanked."""
+        for text in (
+            "Use a<b then `<!-- x -->` > c,\nthen <!-- x --> two.\n",
+            'Use a<b "see `<!-- x -->`\nand" > c, then <!-- x --> two.\n',
+            "Start <!--> a `<!-- x -->` b\nthen <!-- x --> two.\n",
+        ):
+            with self.subTest(text=text):
+                got = [c[0] for c in self.comments("a.md", text) if c[1] == "x"]
+                self.assertEqual([2], got)
+
     def test_an_inline_code_span_holding_a_comment_text_is_not_its_line(self) -> None:
         """The search for the comment stopped on the code span's line, one line too early."""
         text = "Write `<!-- check this later. -->` here,\nthen <!-- check this later. -->\n"
@@ -7377,7 +7456,7 @@ def test_this_module_collects_a_plausible_number_of_cases(self) -> None:
     def test_the_gate_stays_under_its_size_ceiling(self) -> None:
         """Vocabulary and exemptions live in the data file, so the script holding them is a defect."""
         lines = PROSE_LINT_SCRIPT.read_text(encoding="utf-8").count("\n")
-        self.assertLessEqual(lines, 1840)
+        self.assertLessEqual(lines, 1862)
 
 
 if __name__ == "__main__":