diff --git a/.gitignore b/.gitignore index c7cf56216af..e46280d731b 100644 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,8 @@ tags .mypy_cache/ reports/ -test_report.* +test_report*.html +test_report*.db test_short_report.html # Test failures will dump the cluster state in here diff --git a/continuous_integration/scripts/test_report.py b/continuous_integration/scripts/test_report.py index 109bc84fb60..2c39fb50018 100644 --- a/continuous_integration/scripts/test_report.py +++ b/continuous_integration/scripts/test_report.py @@ -9,7 +9,7 @@ import sys import zipfile from collections.abc import Iterator -from typing import Any +from typing import Any, Iterable, cast import altair import altair_saver @@ -102,6 +102,39 @@ def maybe_get_next_page_path(response: requests.Response) -> str | None: return next_page_path +def get_jobs(workflow): + with shelve.open("test_report_jobs") as cache: + url = workflow["jobs_url"] + try: + jobs = cache[url] + except KeyError: + params = {"per_page": 100} + r = get_from_github(workflow["jobs_url"], params) + jobs = r.json()["jobs"] + while next_page := maybe_get_next_page_path(r): + r = get_from_github(next_page, params=params) + jobs.extend(r.json()["jobs"]) + cache[url] = jobs + + df_jobs = pandas.DataFrame.from_records(jobs) + extracted = df_jobs.name.str.extract( + r"\(([\w\-]+), (\d\.\d+),\s([\w|\s]+)\)" + ).dropna() + df_jobs["OS"] = extracted[0] + df_jobs["python_version"] = extracted[1] + df_jobs["partition"] = extracted[2] + # We later need to join on this. Somehow the job ID is not part of the workflow schema and we have no other way to join + df_jobs["suite_name"] = ( + df_jobs["OS"] + + "-" + + df_jobs["python_version"] + + "-" + + df_jobs["partition"].str.replace(" ", "") + ) + + return df_jobs + + def get_workflow_listing(repo: str, branch: str, event: str, days: int) -> list[dict]: """ Get a list of workflow runs from GitHub actions. @@ -153,17 +186,21 @@ def download_and_parse_artifact(url: str) -> junitparser.JUnitXml | None: """ Download the artifact at the url parse it. """ + with shelve.open("test_report") as cache: + try: + xml_raw = cache[url] + except KeyError: + r = get_from_github(url, params={}) + f = zipfile.ZipFile(io.BytesIO(r.content)) + cache[url] = xml_raw = f.read(f.filelist[0].filename) try: - r = get_from_github(url, params={}) - f = zipfile.ZipFile(io.BytesIO(r.content)) - run = junitparser.JUnitXml.fromstring(f.read(f.filelist[0].filename)) - return run + return junitparser.JUnitXml.fromstring(xml_raw) except Exception: - print(f"Failed to download/parse {url}") + # XMLs also include things like schedule which is a simple json return None -def dataframe_from_jxml(run: list) -> pandas.DataFrame: +def dataframe_from_jxml(run: Iterable) -> pandas.DataFrame: """ Turn a parsed JXML into a pandas dataframe """ @@ -171,8 +208,10 @@ def dataframe_from_jxml(run: list) -> pandas.DataFrame: tname = [] status = [] message = [] + sname = [] for suite in run: for test in suite: + sname.append(suite.name) fname.append(test.classname) tname.append(test.name) s = "✓" @@ -195,7 +234,13 @@ def dataframe_from_jxml(run: list) -> pandas.DataFrame: status.append(s) message.append(html.escape(m)) df = pandas.DataFrame( - {"file": fname, "test": tname, "status": status, "message": message} + { + "file": fname, + "test": tname, + "status": status, + "message": message, + "suite_name": sname, + } ) # There are sometimes duplicate tests in the report for some unknown reason. @@ -212,7 +257,7 @@ def dedup(group): else: return group - return df.groupby(["file", "test"]).agg(dedup) + return df.groupby(["file", "test"], as_index=False).agg(dedup) def download_and_parse_artifacts( @@ -258,40 +303,39 @@ def download_and_parse_artifacts( ndownloaded = 0 print(f"Downloading and parsing {nartifacts} artifacts...") - cache: shelve.Shelf[pandas.DataFrame | None] - # FIXME https://github.com/python/typeshed/pull/8190 - with shelve.open("test_report") as cache: # type: ignore[assignment] - for w in workflows: - w["dfs"] = [] - for a in w["artifacts"]: - url = a["archive_download_url"] - df: pandas.DataFrame | None - try: - df = cache[url] - except KeyError: - xml = download_and_parse_artifact(url) - if xml: - df = dataframe_from_jxml(xml) - # Note: we assign a column with the workflow timestamp rather - # than the artifact timestamp so that artifacts triggered under - # the same workflow can be aligned according to the same trigger - # time. - df = df.assign( - name=a["name"], - suite=suite_from_name(a["name"]), - date=w["created_at"], - url=w["html_url"], - ) - else: - df = None - cache[url] = df - - if df is not None: - yield df - - ndownloaded += 1 - if ndownloaded and not ndownloaded % 20: - print(f"{ndownloaded}... ", end="") + for w in workflows: + jobs_df = get_jobs(w) + w["dfs"] = [] + for a in w["artifacts"]: + url = a["archive_download_url"] + df: pandas.DataFrame | None + xml = download_and_parse_artifact(url) + if xml is None: + continue + df = dataframe_from_jxml(cast(Iterable, xml)) + # Note: we assign a column with the workflow timestamp rather + # than the artifact timestamp so that artifacts triggered under + # the same workflow can be aligned according to the same trigger + # time. + html_url = jobs_df[jobs_df["suite_name"] == a["name"]].html_url.unique() + assert ( + len(html_url) == 1 + ), f"Artifact suit name {a['name']} did not match any jobs dataframe {jobs_df['suite_name'].unique()}" + html_url = html_url[0] + assert html_url is not None + df2 = df.assign( + name=a["name"], + suite=suite_from_name(a["name"]), + date=w["created_at"], + html_url=html_url, + ) + + if df2 is not None: + yield df2 + + ndownloaded += 1 + if ndownloaded and not ndownloaded % 20: + print(f"{ndownloaded}... ", end="") def main(argv: list[str] | None = None) -> None: @@ -313,10 +357,20 @@ def main(argv: list[str] | None = None) -> None: max_workflows=args.max_workflows, ) ) - total = pandas.concat(dfs, axis=0) + # Reduce the size of the DF since the entire thing is encoded in the vega spec + required_columns = [ + "test", + "date", + "suite", + "file", + "html_url", + "status", + "message", + ] + total = total[required_columns] grouped = ( - total.groupby(total.index) + total.groupby([total.file, total.test]) .filter(lambda g: (g.status == "x").sum() >= args.nfails) .reset_index() .assign(test=lambda df: df.file + "." + df.test) @@ -356,7 +410,7 @@ def main(argv: list[str] | None = None) -> None: .encode( x=altair.X("date:O", scale=altair.Scale(domain=sorted(list(times)))), y=altair.Y("suite:N", title=None), - href=altair.Href("url:N"), + href=altair.Href("html_url:N"), color=altair.Color( "status:N", scale=altair.Scale( @@ -364,7 +418,7 @@ def main(argv: list[str] | None = None) -> None: range=list(COLORS.values()), ), ), - tooltip=["suite:N", "date:O", "status:N", "message:N", "url:N"], + tooltip=["suite:N", "date:O", "status:N", "message:N", "html_url:N"], ) .properties(title=name) | altair.Chart(df_agg.assign(_="_"))