diff --git a/hotkdump/core/hotkdump.py b/hotkdump/core/hotkdump.py index a5d14a9..2769bc5 100644 --- a/hotkdump/core/hotkdump.py +++ b/hotkdump/core/hotkdump.py @@ -17,6 +17,13 @@ import textwrap from datetime import datetime from dataclasses import dataclass, field +import warnings + +try: + from importlib.resources import read_text +except ModuleNotFoundError: + from importlib_resources import read_text + try: from ubuntutools.pullpkg import PullPkg @@ -27,6 +34,8 @@ raise ModuleNotFoundError("\n\n`hotkdump` needs ubuntu.pullpkg to function.\n" "Install it via `sudo apt install ubuntu-dev-tools`") from exc +from jinja2 import Template + from hotkdump.core.exceptions import ExceptionWithLog from hotkdump.core.kdumpfile import KdumpFile from hotkdump.core.utils import pretty_size @@ -107,7 +116,6 @@ def __init__(self, parameters: HotkdumpParameters): self.temp_working_dir = tempfile.TemporaryDirectory() logging.debug( "created %s temporary directory for the intermediary files", self.temp_working_dir.name) - self.commands_file_path = self.write_crash_commands_file() # Create the ddeb path if not exists os.makedirs(self.params.ddebs_folder_path, exist_ok=True) @@ -181,90 +189,23 @@ def touch_file(fname): pass def write_crash_commands_file(self): - """The crash_commands file we generate should look like - - !echo "Output of sys\n" >> hotkdump.out - sys >> hotkdump.out - !echo "\nOutput of bt\n" >> hotkdump.out - bt >> hotkdump.out - !echo "\nOutput of log with audit messages filtered out\n" >> hotkdump.out - log | grep -vi audit >> hotkdump.out - !echo "\nOutput of kmem -i\n" >> hotkdump.out - kmem -i >> hotkdump.out - !echo "\nOutput of dev -d\n" >> hotkdump.out - dev -d >> hotkdump.out - !echo "\nLongest running blocked processes\n" >> hotkdump.out - ps -m | grep UN | tail >> hotkdump.out - quit >> hotkdump.out - """ + """Render and write the crash_commands file.""" commands_file = f"{self.temp_working_dir.name}/crash_commands" of_path = self.params.output_file_path - # pylint + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + # Read & render the template + jinja_template_content = read_text( + "hotkdump.templates", "crash_commands.jinja") + + template = Template(jinja_template_content) + rendered_content = template.render( + output_file_path=of_path, commands_file_name=commands_file + ) + with open(commands_file, "w", encoding="utf-8") as ccfile: - # (mkg): the file uses self-append to evaluate commands depend on - # the information extracted from a prior command invocation. This - # is possible because POSIX guarantees that: - # "If a read() of file data can be proven (by any means) to occur - # after a write() of the data, it must reflect that write(), even - # if the calls are made by different processes." - # pylint: disable=line-too-long - commands_file_content = fr""" - !echo "---------------------------------------" >> {of_path} - !echo "Output of 'sys'" >> {of_path} - !echo "---------------------------------------" >> {of_path} - sys >> {of_path} - !echo "---------------------------------------" >> {of_path} - !echo "Output of 'bt'" >> {of_path} - !echo "---------------------------------------" >> {of_path} - bt >> {of_path} - !echo "---------------------------------------" >> {of_path} - !echo "Output of 'log' with audit messages filtered out" >> {of_path} - !echo "---------------------------------------" >> {of_path} - log | grep -vi audit >> {of_path} - !echo "---------------------------------------" >> {of_path} - !echo "Output of 'kmem -i'" >> {of_path} - !echo "---------------------------------------" >> {of_path} - kmem -i >> {of_path} - !echo "---------------------------------------" >> {of_path} - !echo "Output of 'dev -d'" >> {of_path} - !echo "---------------------------------------" >> {of_path} - dev -d >> {of_path} - !echo "---------------------------------------" >> {of_path} - !echo "Output of 'mount'" >> {of_path} - !echo "---------------------------------------" >> {of_path} - mount >> {of_path} - !echo "---------------------------------------" >> {of_path} - !echo "Output of 'files'" >> {of_path} - !echo "---------------------------------------" >> {of_path} - files >> {of_path} - !echo "---------------------------------------" >> {of_path} - !echo "Output of 'vm'" >> {of_path} - !echo "---------------------------------------" >> {of_path} - vm >> {of_path} - !echo "---------------------------------------" >> {of_path} - !echo "Output of 'net'" >> {of_path} - !echo "---------------------------------------" >> {of_path} - net >> {of_path} - !echo "---------------------------------------" >> {of_path} - !echo "Longest running blocked processes" >> {of_path} - !echo "---------------------------------------" >> {of_path} - ps -m | grep UN | tail >> {of_path} - !echo "---------------------------------------" >> {of_path} - !echo "Top 20 memory consumers" >> {of_path} - !echo "---------------------------------------" >> {of_path} - ps -G | sed 's/>//g' | sort -k 8,8 -n | awk '$8 ~ /[0-9]/{{ $8 = $8/1024" MB"; print }}' | tail -20 | sort -r -k8,8 -g >> {of_path} - !echo "\n!echo '---------------------------------------' >> {of_path}" >> {commands_file} - !echo "\n!echo 'BT of the longest running blocked process' >> {of_path}" >> {commands_file} - !echo "\n!echo '---------------------------------------' >> {of_path}" >> {commands_file} - ps -m | grep UN | tail -n1 | grep -oE "PID: [0-9]+" | grep -oE "[0-9]+" | awk '{{print "bt " $1 " >> {of_path}"}}' >> {commands_file} - !echo "\nquit >> {of_path}" >> {commands_file} - !echo "" >> {of_path}""" - # (mkg): The last empty echo is important to allow - # crash to pick up the commands appended to the command - # file at the runtime. - final_cmdfile_contents = textwrap.dedent( - commands_file_content).strip() + final_cmdfile_contents = textwrap.dedent(rendered_content).strip() ccfile.write(final_cmdfile_contents) logging.debug( "command file %s rendered with contents: %s", commands_file, final_cmdfile_contents) @@ -352,7 +293,6 @@ def _digest_debuginfod_find_output(self, line): pct = int((current / maximum) * 100) self.debuginfod_find_progress.update(pct, 100) - def maybe_download_vmlinux_via_debuginfod(self): """Try downloading vmlinux image with debug information using debuginfod-find.""" @@ -471,9 +411,9 @@ def summarize_vmcore_file(self, vmlinux_path:str): """Print a summary of the vmcore file to the output file """ logging.info("Loading `vmcore` file %s into `crash`, please wait..", self.params.dump_file_path) - + commands_file_path = self.write_crash_commands_file() self.exec(self.crash_executable, - f"-x -i {self.commands_file_path} -s {self.params.dump_file_path} {vmlinux_path}") + f"-x -i {commands_file_path} -s {self.params.dump_file_path} {vmlinux_path}") logging.info("See %s for logs, %s for outputs", self.params.log_file_path, self.params.output_file_path) def launch_crash(self, vmlinux_path:str): diff --git a/hotkdump/templates/__init__.py b/hotkdump/templates/__init__.py new file mode 100644 index 0000000..a917722 --- /dev/null +++ b/hotkdump/templates/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 + +# Copyright 2023 Canonical Limited. +# SPDX-License-Identifier: GPL-3.0 diff --git a/hotkdump/templates/crash_commands.jinja b/hotkdump/templates/crash_commands.jinja new file mode 100644 index 0000000..8f0efc3 --- /dev/null +++ b/hotkdump/templates/crash_commands.jinja @@ -0,0 +1,64 @@ +{# +# (mkg): the file uses self-append to evaluate commands depend on +# the information extracted from a prior command invocation. This +# is possible because POSIX guarantees that: +# "If a read() of file data can be proven (by any means) to occur +# after a write() of the data, it must reflect that write(), even +# if the calls are made by different processes." +#} + +!echo "---------------------------------------" >> {{ output_file_path }} +!echo "Output of 'sys'" >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +sys >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +!echo "Output of 'bt'" >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +bt >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +!echo "Output of 'log' with audit messages filtered out" >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +log | grep -vi audit >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +!echo "Output of 'kmem -i'" >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +kmem -i >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +!echo "Output of 'dev -d'" >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +dev -d >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +!echo "Output of 'mount'" >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +mount >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +!echo "Output of 'files'" >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +files >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +!echo "Output of 'vm'" >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +vm >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +!echo "Output of 'net'" >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +net >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +!echo "Longest running blocked processes" >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +ps -m | grep UN | tail >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +!echo "Top 20 memory consumers" >> {{ output_file_path }} +!echo "---------------------------------------" >> {{ output_file_path }} +ps -G | sed 's/>//g' | sort -k 8,8 -n | awk '$8 ~ /[0-9]/{ $8 = $8/1024" MB"; print }' | tail -20 | sort -r -k8,8 -g >> {{ output_file_path }} +!echo "\n!echo '---------------------------------------' >> {{ output_file_path }}" >> {{ commands_file_name }} +!echo "\n!echo 'BT of the longest running blocked process' >> {{ output_file_path }}" >> {{ commands_file_name }} +!echo "\n!echo '---------------------------------------' >> {{ output_file_path }}" >> {{ commands_file_name }} +ps -m | grep UN | tail -n1 | grep -oE "PID: [0-9]+" | grep -oE "[0-9]+" | awk '{print "bt " $1 " >> {{ output_file_path }}"}' >> {{ commands_file_name }} +!echo "\nquit >> {{ output_file_path }}" >> {{ commands_file_name }} +!echo "" >> {{ output_file_path }} +{# +# (mkg): The last empty echo is important to allow +# crash to pick up the commands appended to the command +# file at the runtime. +#} diff --git a/pyproject.toml b/pyproject.toml index 1fc3e2f..638806e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ readme = "README.md" requires-python = ">=3.6" license = { file = "LICENSE" } keywords = ["crash", "debugging", "kdump"] -dependencies = [ "dataclasses;python_version<'3.7'"] +dependencies = [ "dataclasses;python_version<'3.7'", "jinja2==3.0.3", "importlib_resources;python_version<'3.7'"] classifiers = [ # How mature is this project? Common values are # 3 - Alpha diff --git a/tests/test_hotkdump.py b/tests/test_hotkdump.py index 590bf3d..0022eec 100644 --- a/tests/test_hotkdump.py +++ b/tests/test_hotkdump.py @@ -45,7 +45,7 @@ "os", remove=lambda x: True, listdir=lambda x: [], - stat=lambda x: "a", + stat=lambda x, *args, **kwargs: "a", makedirs=lambda *a, **kw: None, ) @mock.patch.multiple( @@ -223,7 +223,7 @@ def test_write_crash_commands_file(self): !echo "---------------------------------------" >> hkd.test !echo "Top 20 memory consumers" >> hkd.test !echo "---------------------------------------" >> hkd.test - ps -G | sed 's/>//g' | sort -k 8,8 -n | awk '$8 ~ /[0-9]/{ $8 = $8/1024" MB"; print }' | tail -20 | sort -r -k8,8 -g >> hkd.test + ps -G | sed 's/>//g' | sort -k 8,8 -n | awk '$8 ~ /[0-9]/{ $8 = $8/1024" MB"; print }' | tail -20 | sort -r -k8,8 -g >> hkd.test !echo "\n!echo '---------------------------------------' >> hkd.test" >> /tmpdir/crash_commands !echo "\n!echo 'BT of the longest running blocked process' >> hkd.test" >> /tmpdir/crash_commands !echo "\n!echo '---------------------------------------' >> hkd.test" >> /tmpdir/crash_commands diff --git a/tox.ini b/tox.ini index 9d65427..f0b2a53 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,5 @@ [tox] env_list = py{36,37,38,39,310,311,312},pylint -isolated_build=true skipsdist = true [testenv] @@ -20,12 +19,11 @@ deps = # Note that this is awkwardly installs the package # itself and not only [optional-dependencies.testing]. # see: https://github.com/pypa/pip/issues/11440 - py37,py38,py39,py310,py311,py312: .[testing] # Install & test dependencies + py37,py38,py39,py310,py311,py312,pylint: .[testing] # Install & test dependencies commands = py36: pip3 install toml py36: bash -c "pip3 install $(python extras/py36-all-requirements.py)" pytest {posargs} [testenv:pylint] -deps = pylint commands = pylint --recursive=y -v {toxinidir}/hotkdump