crash_commands: move it to a jinja template

Added jinja2 3.0.3 dependency as it's the last version that
supports the python36

Closes: SET-940

Signed-off-by: Mustafa Kemal Gilor <mustafa.gilor@canonical.com>

hotkdump/core/hotkdump.py

@@ -17,6 +17,13 @@
 import textwrap
 from datetime import datetime
 from dataclasses import dataclass, field
+import warnings
+
+try:
+    from importlib.resources import read_text
+except ModuleNotFoundError:
+    from importlib_resources import read_text
+
 
 try:
     from ubuntutools.pullpkg import PullPkg
@@ -27,6 +34,8 @@
     raise ModuleNotFoundError("\n\n`hotkdump` needs ubuntu.pullpkg to function.\n"
                               "Install it via `sudo apt install ubuntu-dev-tools`") from exc
 
+from jinja2 import Template
+
 from hotkdump.core.exceptions import ExceptionWithLog
 from hotkdump.core.kdumpfile import KdumpFile
 from hotkdump.core.utils import pretty_size
@@ -107,7 +116,6 @@ def __init__(self, parameters: HotkdumpParameters):
         self.temp_working_dir = tempfile.TemporaryDirectory()
         logging.debug(
             "created %s temporary directory for the intermediary files", self.temp_working_dir.name)
-        self.commands_file_path = self.write_crash_commands_file()
 
         # Create the ddeb path if not exists
         os.makedirs(self.params.ddebs_folder_path, exist_ok=True)
@@ -181,90 +189,23 @@ def touch_file(fname):
             pass
 
     def write_crash_commands_file(self):
-        """The crash_commands file we generate should look like
-
-            !echo "Output of sys\n" >> hotkdump.out
-            sys >> hotkdump.out
-            !echo "\nOutput of bt\n" >> hotkdump.out
-            bt >> hotkdump.out
-            !echo "\nOutput of log with audit messages filtered out\n" >> hotkdump.out
-            log | grep -vi audit >> hotkdump.out
-            !echo "\nOutput of kmem -i\n" >> hotkdump.out
-            kmem -i >> hotkdump.out
-            !echo "\nOutput of dev -d\n" >> hotkdump.out
-            dev -d >> hotkdump.out
-            !echo "\nLongest running blocked processes\n" >> hotkdump.out
-            ps -m | grep UN | tail >> hotkdump.out
-            quit >> hotkdump.out
-        """
+        """Render and write the crash_commands file."""
         commands_file = f"{self.temp_working_dir.name}/crash_commands"
         of_path = self.params.output_file_path
 
-        # pylint
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", category=DeprecationWarning)
+            # Read & render the template
+            jinja_template_content = read_text(
+                "hotkdump.templates", "crash_commands.jinja")
+
+        template = Template(jinja_template_content)
+        rendered_content = template.render(
+            output_file_path=of_path, commands_file_name=commands_file
+        )
+
         with open(commands_file, "w", encoding="utf-8") as ccfile:
-            # (mkg): the file uses self-append to evaluate commands depend on
-            # the information extracted from a prior command invocation. This
-            # is possible because POSIX guarantees that:
-            #   "If a read() of file data can be proven (by any means) to occur
-            #   after a write() of the data, it must reflect that write(), even
-            #   if the calls are made by different processes."
-            # pylint: disable=line-too-long
-            commands_file_content = fr"""
-            !echo "---------------------------------------" >> {of_path}
-            !echo "Output of 'sys'" >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            sys >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            !echo "Output of 'bt'" >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            bt >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            !echo "Output of 'log' with audit messages filtered out" >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            log | grep -vi audit >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            !echo "Output of 'kmem -i'" >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            kmem -i >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            !echo "Output of 'dev -d'" >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            dev -d >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            !echo "Output of 'mount'" >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            mount >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            !echo "Output of 'files'" >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            files >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            !echo "Output of 'vm'" >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            vm >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            !echo "Output of 'net'" >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            net >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            !echo "Longest running blocked processes" >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            ps -m | grep UN | tail >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            !echo "Top 20 memory consumers" >> {of_path}
-            !echo "---------------------------------------" >> {of_path}
-            ps -G | sed 's/>//g' | sort -k 8,8 -n |  awk '$8 ~ /[0-9]/{{ $8 = $8/1024" MB"; print }}' | tail -20 | sort -r -k8,8 -g >> {of_path}
-            !echo "\n!echo '---------------------------------------' >> {of_path}" >> {commands_file}
-            !echo "\n!echo 'BT of the longest running blocked process' >> {of_path}" >> {commands_file}
-            !echo "\n!echo '---------------------------------------' >> {of_path}" >> {commands_file}
-            ps -m | grep UN | tail -n1 | grep -oE "PID: [0-9]+" | grep -oE "[0-9]+" | awk '{{print "bt " $1 " >> {of_path}"}}' >> {commands_file}
-            !echo "\nquit >> {of_path}" >> {commands_file}
-            !echo "" >> {of_path}"""
-            # (mkg): The last empty echo is important to allow
-            # crash to pick up the commands appended to the command
-            # file at the runtime.
-            final_cmdfile_contents = textwrap.dedent(
-                commands_file_content).strip()
+            final_cmdfile_contents = textwrap.dedent(rendered_content).strip()
             ccfile.write(final_cmdfile_contents)
             logging.debug(
                 "command file %s rendered with contents: %s", commands_file, final_cmdfile_contents)
@@ -352,7 +293,6 @@ def _digest_debuginfod_find_output(self, line):
                 pct = int((current / maximum) * 100)
                 self.debuginfod_find_progress.update(pct, 100)
 
-
     def maybe_download_vmlinux_via_debuginfod(self):
         """Try downloading vmlinux image with debug information
         using debuginfod-find."""
@@ -471,9 +411,9 @@ def summarize_vmcore_file(self, vmlinux_path:str):
         """Print a summary of the vmcore file to the output file
         """
         logging.info("Loading `vmcore` file %s into `crash`, please wait..", self.params.dump_file_path)
-
+        commands_file_path = self.write_crash_commands_file()
         self.exec(self.crash_executable,
-                  f"-x -i {self.commands_file_path} -s {self.params.dump_file_path} {vmlinux_path}")
+                  f"-x -i {commands_file_path} -s {self.params.dump_file_path} {vmlinux_path}")
         logging.info("See %s for logs, %s for outputs", self.params.log_file_path, self.params.output_file_path)
 
     def launch_crash(self, vmlinux_path:str):

hotkdump/templates/__init__.py

@@ -0,0 +1,4 @@
+#!/usr/bin/env python3
+
+# Copyright 2023 Canonical Limited.
+# SPDX-License-Identifier: GPL-3.0

hotkdump/templates/crash_commands.jinja

@@ -0,0 +1,64 @@
+{#
+# (mkg): the file uses self-append to evaluate commands depend on
+# the information extracted from a prior command invocation. This
+# is possible because POSIX guarantees that:
+#   "If a read() of file data can be proven (by any means) to occur
+#   after a write() of the data, it must reflect that write(), even
+#   if the calls are made by different processes."
+#}
+
+!echo "---------------------------------------" >> {{ output_file_path }}
+!echo "Output of 'sys'" >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+sys >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+!echo "Output of 'bt'" >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+bt >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+!echo "Output of 'log' with audit messages filtered out" >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+log | grep -vi audit >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+!echo "Output of 'kmem -i'" >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+kmem -i >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+!echo "Output of 'dev -d'" >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+dev -d >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+!echo "Output of 'mount'" >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+mount >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+!echo "Output of 'files'" >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+files >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+!echo "Output of 'vm'" >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+vm >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+!echo "Output of 'net'" >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+net >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+!echo "Longest running blocked processes" >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+ps -m | grep UN | tail >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+!echo "Top 20 memory consumers" >> {{ output_file_path }}
+!echo "---------------------------------------" >> {{ output_file_path }}
+ps -G | sed 's/>//g' | sort -k 8,8 -n | awk '$8 ~ /[0-9]/{ $8 = $8/1024" MB"; print }' | tail -20 | sort -r -k8,8 -g >> {{ output_file_path }}
+!echo "\n!echo '---------------------------------------' >> {{ output_file_path }}" >> {{ commands_file_name }}
+!echo "\n!echo 'BT of the longest running blocked process' >> {{ output_file_path }}" >> {{ commands_file_name }}
+!echo "\n!echo '---------------------------------------' >> {{ output_file_path }}" >> {{ commands_file_name }}
+ps -m | grep UN | tail -n1 | grep -oE "PID: [0-9]+" | grep -oE "[0-9]+" | awk '{print "bt " $1 " >> {{ output_file_path }}"}' >> {{ commands_file_name }}
+!echo "\nquit >> {{ output_file_path }}" >> {{ commands_file_name }}
+!echo "" >> {{ output_file_path }}
+{#
+# (mkg): The last empty echo is important to allow
+# crash to pick up the commands appended to the command
+# file at the runtime.
+#}

pyproject.toml

@@ -8,7 +8,7 @@ readme = "README.md"
 requires-python = ">=3.6"
 license = { file = "LICENSE" }
 keywords = ["crash", "debugging", "kdump"]
-dependencies = [ "dataclasses;python_version<'3.7'"]
+dependencies = [ "dataclasses;python_version<'3.7'", "jinja2==3.0.3", "importlib_resources;python_version<'3.7'"]
 classifiers = [
   # How mature is this project? Common values are
   #   3 - Alpha

tests/test_hotkdump.py

@@ -45,7 +45,7 @@
     "os",
     remove=lambda x: True,
     listdir=lambda x: [],
-    stat=lambda x: "a",
+    stat=lambda x, *args, **kwargs: "a",
     makedirs=lambda *a, **kw: None,
 )
 @mock.patch.multiple(
@@ -223,7 +223,7 @@ def test_write_crash_commands_file(self):
         !echo "---------------------------------------" >> hkd.test
         !echo "Top 20 memory consumers" >> hkd.test
         !echo "---------------------------------------" >> hkd.test
-        ps -G | sed 's/>//g' | sort -k 8,8 -n |  awk '$8 ~ /[0-9]/{ $8 = $8/1024" MB"; print }' | tail -20 | sort -r -k8,8 -g >> hkd.test
+        ps -G | sed 's/>//g' | sort -k 8,8 -n | awk '$8 ~ /[0-9]/{ $8 = $8/1024" MB"; print }' | tail -20 | sort -r -k8,8 -g >> hkd.test
         !echo "\n!echo '---------------------------------------' >> hkd.test" >> /tmpdir/crash_commands
         !echo "\n!echo 'BT of the longest running blocked process' >> hkd.test" >> /tmpdir/crash_commands
         !echo "\n!echo '---------------------------------------' >> hkd.test" >> /tmpdir/crash_commands

tox.ini

@@ -1,6 +1,5 @@
 [tox]
 env_list = py{36,37,38,39,310,311,312},pylint
-isolated_build=true
 skipsdist = true
 
 [testenv]
@@ -20,12 +19,11 @@ deps =
     # Note that this is awkwardly installs the package
     # itself and not only [optional-dependencies.testing].
     # see: https://github.com/pypa/pip/issues/11440
-    py37,py38,py39,py310,py311,py312: .[testing] # Install & test dependencies
+    py37,py38,py39,py310,py311,py312,pylint: .[testing] # Install & test dependencies
 commands =
     py36: pip3 install toml
     py36: bash -c "pip3 install $(python extras/py36-all-requirements.py)"
     pytest {posargs}
 
 [testenv:pylint]
-deps = pylint
 commands = pylint --recursive=y -v {toxinidir}/hotkdump