Skip to content

Commit 370c994

Browse files
authored
Merge pull request #27 from kernelci/add-dtc-parser
kbuild: parse device tree compiler errors
2 parents de1b1c8 + 808e014 commit 370c994

8 files changed

Lines changed: 300 additions & 3 deletions

File tree

logspec/errors/kbuild.py

Lines changed: 140 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,130 @@ def _parse(self, text):
255255
return parse_end_pos
256256

257257

258+
class KbuildDtcError(Error):
259+
"""Models an error emitted by the Device Tree Compiler (DTC)."""
260+
261+
_source_diagnostic_re = re.compile(
262+
rf'^{TIMESTAMP}(?P<kind>Lexical error|Error): '
263+
r'(?P<src_file>.*?):'
264+
r'(?P<location>\d+\.\d+(?:-\d+(?:\.\d+)?)?) '
265+
r'(?P<message>.*)$',
266+
flags=re.MULTILINE,
267+
)
268+
_check_diagnostic_re = re.compile(
269+
rf'^{TIMESTAMP}(?P<src_file>.*?)'
270+
r'(?::(?P<location>\d+\.\d+(?:-\d+(?:\.\d+)?)?))?'
271+
r': ERROR \((?P<check>[^)]+)\): (?P<message>.*)$',
272+
flags=re.MULTILINE,
273+
)
274+
_fatal_re = re.compile(
275+
rf'^{TIMESTAMP}FATAL ERROR: (?P<message>.*)$',
276+
flags=re.MULTILINE,
277+
)
278+
_check_terminal_re = re.compile(
279+
rf'^{TIMESTAMP}ERROR: Input tree has errors,.*$',
280+
flags=re.MULTILINE,
281+
)
282+
283+
def __init__(self, script=None, target=None):
284+
super().__init__()
285+
self.script = script
286+
self.target = target
287+
self.src_file = ""
288+
self.location = ""
289+
self.check = ""
290+
291+
@classmethod
292+
def has_diagnostic(cls, text):
293+
"""Return whether *text* contains a structured DTC error."""
294+
return any(regex.search(text) for regex in (
295+
cls._source_diagnostic_re,
296+
cls._check_diagnostic_re,
297+
cls._fatal_re,
298+
))
299+
300+
def _set_report_end(self, text, match, diagnostic_type):
301+
"""Find the terminal line belonging to a DTC diagnostic."""
302+
report_end = match.end()
303+
following_text = text[report_end:]
304+
305+
if diagnostic_type == "source":
306+
terminal_match = re.match(
307+
rf'\n{TIMESTAMP}FATAL ERROR: .*',
308+
following_text,
309+
)
310+
elif diagnostic_type == "check":
311+
terminal_match = self._check_terminal_re.search(
312+
text, match.end()
313+
)
314+
else:
315+
terminal_match = None
316+
317+
if terminal_match:
318+
report_end = (
319+
report_end + terminal_match.end()
320+
if diagnostic_type == "source"
321+
else terminal_match.end()
322+
)
323+
return report_end
324+
325+
def _parse(self, text):
326+
"""Extract the last DTC diagnostic preceding the Make failure."""
327+
candidates = []
328+
for diagnostic_type, regex in (
329+
("source", self._source_diagnostic_re),
330+
("check", self._check_diagnostic_re),
331+
):
332+
matches = list(regex.finditer(text))
333+
if matches:
334+
candidates.append((diagnostic_type, matches[-1]))
335+
336+
if candidates:
337+
diagnostic_type, match = max(
338+
candidates, key=lambda candidate: candidate[1].start()
339+
)
340+
else:
341+
fatal_matches = list(self._fatal_re.finditer(text))
342+
if not fatal_matches:
343+
return 0
344+
diagnostic_type = "fatal"
345+
match = fatal_matches[-1]
346+
347+
if diagnostic_type == "source":
348+
kind = match.group('kind')
349+
self.error_type = (
350+
"kbuild.dtc.lexical_error"
351+
if kind == "Lexical error"
352+
else "kbuild.dtc.error"
353+
)
354+
self.src_file = match.group('src_file')
355+
self.location = match.group('location')
356+
self.error_summary = f"{kind}: {match.group('message')}"
357+
elif diagnostic_type == "check":
358+
self.error_type = "kbuild.dtc.check_error"
359+
self.src_file = match.group('src_file')
360+
self.location = match.group('location') or ""
361+
self.check = match.group('check')
362+
self.error_summary = match.group('message')
363+
else:
364+
self.error_type = "kbuild.dtc.fatal_error"
365+
self.error_summary = match.group('message')
366+
file_match = re.search(
367+
r'(?:Couldn.t open|Error closing) "(?P<src_file>[^"]+)"',
368+
self.error_summary,
369+
)
370+
if file_match:
371+
self.src_file = file_match.group('src_file')
372+
373+
for field in ('src_file', 'location', 'check'):
374+
if getattr(self, field):
375+
self._signature_fields.append(field)
376+
377+
report_end = self._set_report_end(text, match, diagnostic_type)
378+
self._report = text[match.start():report_end] + "\n"
379+
return report_end
380+
381+
258382
class KbuildProcessError(Error):
259383
"""Models the information extracted from a kbuild error caused by a
260384
script, configuration or other runtime error.
@@ -458,6 +582,14 @@ def _is_kbuild_target(target):
458582
return False
459583

460584

585+
def _is_dtc_target(script, target):
586+
"""Return whether a Make failure belongs to a DTC build target."""
587+
return (
588+
target.endswith(('.dtb', '.dtbo'))
589+
or os.path.basename(script.split(':', maxsplit=1)[0]) == 'Makefile.dtbs'
590+
)
591+
592+
461593
def _find_script_target(error_str, text):
462594
match = re.search(r'\[(?P<script>.*?): (?P<target>.*?)\] Error', error_str)
463595
if not match:
@@ -510,7 +642,13 @@ def find_kbuild_error(text):
510642
logging.debug(f"[find_kbuild_error] script: {script}, target: {target}")
511643
error = None
512644
# Kbuild error classification
513-
if _is_object_file(target) or _is_other_compiler_target(target, text[:start]):
645+
error_text = text[:start]
646+
if (
647+
_is_dtc_target(script, target)
648+
and KbuildDtcError.has_diagnostic(error_text)
649+
):
650+
error = KbuildDtcError(script=script, target=target)
651+
elif _is_object_file(target) or _is_other_compiler_target(target, error_text):
514652
error = KbuildCompilerError(script=script, target=target)
515653
elif 'modpost' in script:
516654
error = KbuildModpostError(script=script, target=target)
@@ -519,8 +657,7 @@ def find_kbuild_error(text):
519657
else:
520658
# Catch-all condition for non-specific errors
521659
error = KbuildGenericError(script=script, target=target)
522-
text = text[:start]
523-
error.parse(text)
660+
error.parse(error_text)
524661
else:
525662
# Unrecognized error, these are marked as unknown and not parsed
526663
error = KbuildUnknownError(error_str)

tests/logs/index.txt

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,23 @@
116116
- kbuild_017.log: Modpost error: Section mismatches detected.
117117
FATAL: modpost: Section mismatches detected.
118118

119+
- kbuild_020.log: Device Tree Compiler lexical error
120+
Lexical error: /tmp/kci/linux/arch/arm64/boot/dts/qcom/purwa.dtsi:169.14-35 Unexpected 'VIDEO_CC_MVS0_BSE_CLK'
121+
FATAL ERROR: Syntax error parsing input tree
122+
123+
- kbuild_021.log: Device Tree Compiler syntax error
124+
Error: ../arch/arm64/boot/dts/intel/socfpga_agilex.dtsi:313.15-16 syntax error
125+
FATAL ERROR: Unable to parse input tree
126+
127+
- kbuild_022.log: Device Tree Compiler phandle check error
128+
arch/arm64/boot/dts/qcom/qcs8300.dtsi:714.22-851.5: ERROR (phandle_references): Reference to non-existent node or label
129+
130+
- kbuild_023.log: Device Tree Compiler overlay duplicate label error
131+
arch/arm64/boot/dts/overlays/imx477_378.dtsi:26.20-31.3: ERROR (duplicate_label): Duplicate label
132+
133+
- kbuild_024.log: Device Tree Compiler fatal file error
134+
FATAL ERROR: Couldn't open "arch/arm64/boot/dts/qcom/missing.dtsi": No such file or directory
135+
119136

120137
./linux_boot
121138

tests/logs/kbuild/kbuild_020.log

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
make --silent --keep-going --jobs=16 O=/tmp/kci/artifacts/build ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu-
2+
Lexical error: /tmp/kci/linux/arch/arm64/boot/dts/qcom/purwa.dtsi:169.14-35 Unexpected 'VIDEO_CC_MVS0_BSE_CLK'
3+
FATAL ERROR: Syntax error parsing input tree
4+
make[4]: *** [/tmp/kci/linux/scripts/Makefile.dtbs:140: arch/arm64/boot/dts/qcom/purwa-iot-evk.dtb] Error 1
5+
Lexical error: /tmp/kci/linux/arch/arm64/boot/dts/qcom/purwa.dtsi:169.14-35 Unexpected 'VIDEO_CC_MVS0_BSE_CLK'
6+
FATAL ERROR: Syntax error parsing input tree
7+
make[4]: *** [/tmp/kci/linux/scripts/Makefile.dtbs:140: arch/arm64/boot/dts/qcom/x1p42100-crd.dtb] Error 1

tests/logs/kbuild/kbuild_021.log

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Error: ../arch/arm64/boot/dts/intel/socfpga_agilex.dtsi:313.15-16 syntax error
2+
FATAL ERROR: Unable to parse input tree
3+
make[3]: *** [scripts/Makefile.lib:314: arch/arm64/boot/dts/intel/socfpga_agilex_socdk.dtb] Error 1
4+
make[2]: *** [scripts/Makefile.build:497: arch/arm64/boot/dts/intel] Error 2

tests/logs/kbuild/kbuild_022.log

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
arch/arm64/boot/dts/qcom/qcs8300.dtsi:714.22-851.5: ERROR (phandle_references): /soc@0/pci@1c00000: Reference to non-existent node or label "pcie_smmu"
2+
also defined at arch/arm64/boot/dts/qcom/qcs8300-ride.dts:288.8-296.3
3+
ERROR: Input tree has errors, aborting (use -f to force output)
4+
make[3]: *** [scripts/Makefile.dtbs:131: arch/arm64/boot/dts/qcom/qcs8300-ride.dtb] Error 2
5+
make[2]: *** [scripts/Makefile.build:461: arch/arm64/boot/dts/qcom] Error 2

tests/logs/kbuild/kbuild_023.log

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
arch/arm64/boot/dts/overlays/imx477_378.dtsi:26.20-31.3: ERROR (duplicate_label): /fragment@200/__overlay__/pca@70/i2c@1/cef168@d: Duplicate label 'vcm_node'
2+
ERROR: Input tree has errors, aborting (use -f to force output)
3+
make[3]: *** [scripts/Makefile.dtbs:142: arch/arm64/boot/dts/overlays/camera-mux-2port.dtbo] Error 2
4+
make[2]: *** [scripts/Makefile.build:544: arch/arm64/boot/dts/overlays] Error 2

tests/logs/kbuild/kbuild_024.log

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
FATAL ERROR: Couldn't open "arch/arm64/boot/dts/qcom/missing.dtsi": No such file or directory
2+
make[3]: *** [scripts/Makefile.dtbs:142: arch/arm64/boot/dts/qcom/example.dtb] Error 1
3+
make[2]: *** [scripts/Makefile.build:544: arch/arm64/boot/dts/qcom] Error 2

tests/test_kbuild.py

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -476,6 +476,97 @@
476476
"target": "vmlinux.unstripped"
477477
}
478478
]
479+
}),
480+
481+
# Device Tree Compiler lexical error. The source file named by DTC may
482+
# differ from the .dtb target reported by Make.
483+
#
484+
# Example:
485+
#
486+
# Lexical error: /tmp/kci/linux/arch/arm64/boot/dts/qcom/purwa.dtsi:169.14-35 Unexpected 'VIDEO_CC_MVS0_BSE_CLK'
487+
# FATAL ERROR: Syntax error parsing input tree
488+
# make[4]: *** [/tmp/kci/linux/scripts/Makefile.dtbs:140: arch/arm64/boot/dts/qcom/purwa-iot-evk.dtb] Error 1
489+
('kbuild_020.log',
490+
'kbuild',
491+
{
492+
"errors": [
493+
{
494+
"error_summary": "Lexical error: Unexpected 'VIDEO_CC_MVS0_BSE_CLK'",
495+
"error_type": "kbuild.dtc.lexical_error",
496+
"location": "169.14-35",
497+
"script": "/tmp/kci/linux/scripts/Makefile.dtbs:140",
498+
"src_file": "/tmp/kci/linux/arch/arm64/boot/dts/qcom/purwa.dtsi",
499+
"target": "arch/arm64/boot/dts/qcom/purwa-iot-evk.dtb"
500+
}
501+
]
502+
}),
503+
504+
# Standard DTC parser error. Older Makefile.lib-based DTB builds use the
505+
# same source diagnostic format as current Makefile.dtbs builds.
506+
('kbuild_021.log',
507+
'kbuild',
508+
{
509+
"errors": [
510+
{
511+
"error_summary": "Error: syntax error",
512+
"error_type": "kbuild.dtc.error",
513+
"location": "313.15-16",
514+
"script": "scripts/Makefile.lib:314",
515+
"src_file": "../arch/arm64/boot/dts/intel/socfpga_agilex.dtsi",
516+
"target": "arch/arm64/boot/dts/intel/socfpga_agilex_socdk.dtb"
517+
}
518+
]
519+
}),
520+
521+
# A DTC semantic check error has a different prefix and terminates with
522+
# "ERROR: Input tree has errors" rather than "FATAL ERROR".
523+
('kbuild_022.log',
524+
'kbuild',
525+
{
526+
"errors": [
527+
{
528+
"check": "phandle_references",
529+
"error_summary": "/soc@0/pci@1c00000: Reference to non-existent node or label \"pcie_smmu\"",
530+
"error_type": "kbuild.dtc.check_error",
531+
"location": "714.22-851.5",
532+
"script": "scripts/Makefile.dtbs:131",
533+
"src_file": "arch/arm64/boot/dts/qcom/qcs8300.dtsi",
534+
"target": "arch/arm64/boot/dts/qcom/qcs8300-ride.dtb"
535+
}
536+
]
537+
}),
538+
539+
# DTC check errors also occur while compiling overlays.
540+
('kbuild_023.log',
541+
'kbuild',
542+
{
543+
"errors": [
544+
{
545+
"check": "duplicate_label",
546+
"error_summary": "/fragment@200/__overlay__/pca@70/i2c@1/cef168@d: Duplicate label 'vcm_node'",
547+
"error_type": "kbuild.dtc.check_error",
548+
"location": "26.20-31.3",
549+
"script": "scripts/Makefile.dtbs:142",
550+
"src_file": "arch/arm64/boot/dts/overlays/imx477_378.dtsi",
551+
"target": "arch/arm64/boot/dts/overlays/camera-mux-2port.dtbo"
552+
}
553+
]
554+
}),
555+
556+
# Some DTC failures only emit a fatal diagnostic without a source
557+
# location, for example when an input file cannot be opened.
558+
('kbuild_024.log',
559+
'kbuild',
560+
{
561+
"errors": [
562+
{
563+
"error_summary": "Couldn't open \"arch/arm64/boot/dts/qcom/missing.dtsi\": No such file or directory",
564+
"error_type": "kbuild.dtc.fatal_error",
565+
"script": "scripts/Makefile.dtbs:142",
566+
"src_file": "arch/arm64/boot/dts/qcom/missing.dtsi",
567+
"target": "arch/arm64/boot/dts/qcom/example.dtb"
568+
}
569+
]
479570
})
480571
])
481572
def test_kbuild(log_file, parser_id, expected):
@@ -484,3 +575,32 @@ def test_kbuild(log_file, parser_id, expected):
484575
expected_as_str = json.dumps(expected, indent=4, sort_keys=True, ensure_ascii=False)
485576
parsed_data_as_str = format_data_output(parsed_data)
486577
assert expected_as_str == parsed_data_as_str
578+
579+
580+
def test_kbuild_dtc_report():
581+
log_file = os.path.join(LOG_DIR, 'kbuild_020.log')
582+
parsed_data = load_parser_and_parse_log(
583+
log_file, 'kbuild', tests.setup.PARSER_DEFS_FILE
584+
)
585+
586+
assert parsed_data['errors'][0]._report == (
587+
"Lexical error: /tmp/kci/linux/arch/arm64/boot/dts/qcom/purwa.dtsi:"
588+
"169.14-35 Unexpected 'VIDEO_CC_MVS0_BSE_CLK'\n"
589+
"FATAL ERROR: Syntax error parsing input tree\n"
590+
)
591+
592+
593+
def test_kbuild_dtc_check_report():
594+
log_file = os.path.join(LOG_DIR, 'kbuild_022.log')
595+
parsed_data = load_parser_and_parse_log(
596+
log_file, 'kbuild', tests.setup.PARSER_DEFS_FILE
597+
)
598+
599+
assert parsed_data['errors'][0]._report == (
600+
"arch/arm64/boot/dts/qcom/qcs8300.dtsi:714.22-851.5: "
601+
"ERROR (phandle_references): /soc@0/pci@1c00000: Reference to "
602+
"non-existent node or label \"pcie_smmu\"\n"
603+
" also defined at arch/arm64/boot/dts/qcom/qcs8300-ride.dts:"
604+
"288.8-296.3\n"
605+
"ERROR: Input tree has errors, aborting (use -f to force output)\n"
606+
)

0 commit comments

Comments
 (0)