kdump-hdr: implement kdump flattened file format support

the code now properly identifies the flattened format and parses
the structure correctly.

the code now uses struct.unpack() to read integers from binary
file.

added unit tests to cover the both file formats.

Signed-off-by: Mustafa Kemal Gilor <mustafa.gilor@canonical.com>

hotkdump/core/kdumpfile.py

@@ -13,6 +13,7 @@
 
 import os
 import logging
+import struct
 from dataclasses import dataclass, field
 from hotkdump.core.exceptions import NotAKernelCrashDumpException
 
@@ -45,18 +46,18 @@ def seek_to_first_non_nul(f):
         f.seek(pos)
 
     @staticmethod
-    def read_int32(f, off=None):
-        """Read a 4-byte integer from given file."""
-        if off:
-            f.seek(off, os.SEEK_SET)
-        return int.from_bytes(f.read(4), byteorder="little")
+    def read_int(f, fmt, off=None):
+        """Read an integer value from binary file stream."""
 
-    @staticmethod
-    def read_int64(f, off=None):
-        """Read a 8-byte integer from given file."""
         if off:
             f.seek(off, os.SEEK_SET)
-        return int.from_bytes(f.read(8), byteorder="little")
+        byte_cnt = struct.calcsize(fmt)
+        raw_bytes = f.read(byte_cnt)
+
+        if not raw_bytes or len(raw_bytes) != byte_cnt:
+            return None
+
+        return struct.unpack(fmt, raw_bytes)[0]
 
     @staticmethod
     def read_str(f, ln):
@@ -102,6 +103,9 @@ def __post_init__(self):
         self.normalized_version = self.version.split("-", maxsplit=1)[0].lstrip("#")
 
 
+# TO-DO: Switch to struct.pack / struct.unpack?
+
+
 @dataclass()
 # pylint: disable-next=too-many-instance-attributes
 class DiskDumpHeader:
@@ -143,10 +147,10 @@ def from_fd(fd):
                 machine=BinaryFileReader.read_cstr(fd),
                 domain=BinaryFileReader.read_cstr(fd),
             ),
-            timestamp_sec=BinaryFileReader.read_int64(fd, timestamp_offset),
-            timestamp_usec=BinaryFileReader.read_int64(fd),
-            status=BinaryFileReader.read_int32(fd),
-            block_size=BinaryFileReader.read_int32(fd),
+            timestamp_sec=BinaryFileReader.read_int(fd, "<q", timestamp_offset),
+            timestamp_usec=BinaryFileReader.read_int(fd, "<q"),
+            status=BinaryFileReader.read_int(fd, "<i"),
+            block_size=BinaryFileReader.read_int(fd, "<i"),
         )
 
 
@@ -171,32 +175,29 @@ class KdumpSubHeader:
     max_mapnr_64: int
 
     @staticmethod
-    def from_fd(fd, block_size):
+    def from_fd(fd):
         """Read a KDumpSubHeader from given file."""
-
-        disk_dump_header_blocks = 1
-        offset = disk_dump_header_blocks * block_size
-        fd.seek(offset, os.SEEK_SET)
         return KdumpSubHeader(
-            phys_base=BinaryFileReader.read_int64(fd),
-            dump_level=BinaryFileReader.read_int32(fd),
-            split=BinaryFileReader.read_int32(fd),
-            start_pfn=BinaryFileReader.read_int64(fd),
-            end_pfn=BinaryFileReader.read_int64(fd),
-            offset_vmcoreinfo=BinaryFileReader.read_int64(fd),
-            size_vmcoreinfo=BinaryFileReader.read_int64(fd),
-            offset_note=BinaryFileReader.read_int64(fd),
-            size_note=BinaryFileReader.read_int64(fd),
-            offset_eraseinfo=BinaryFileReader.read_int64(fd),
-            size_eraseinfo=BinaryFileReader.read_int64(fd),
-            start_pfn_64=BinaryFileReader.read_int64(fd),
-            end_pfn_64=BinaryFileReader.read_int64(fd),
-            max_mapnr_64=BinaryFileReader.read_int64(fd),
+            phys_base=BinaryFileReader.read_int(fd, "<q"),
+            dump_level=BinaryFileReader.read_int(fd, "<i"),
+            split=BinaryFileReader.read_int(fd, "<i"),
+            start_pfn=BinaryFileReader.read_int(fd, "<q"),
+            end_pfn=BinaryFileReader.read_int(fd, "<q"),
+            offset_vmcoreinfo=BinaryFileReader.read_int(fd, "<q"),
+            size_vmcoreinfo=BinaryFileReader.read_int(fd, "<q"),
+            offset_note=BinaryFileReader.read_int(fd, "<q"),
+            size_note=BinaryFileReader.read_int(fd, "<q"),
+            offset_eraseinfo=BinaryFileReader.read_int(fd, "<q"),
+            size_eraseinfo=BinaryFileReader.read_int(fd, "<q"),
+            start_pfn_64=BinaryFileReader.read_int(fd, "<q"),
+            end_pfn_64=BinaryFileReader.read_int(fd, "<q"),
+            max_mapnr_64=BinaryFileReader.read_int(fd, "<q"),
         )
 
 
 class VMCoreInfo:
     """Class for parsing VMCoreInfo section text into a dict."""
+
     def __init__(self, raw):
         self.data = {}
         for line in raw.split("\n"):
@@ -212,20 +213,134 @@ def from_fd(fd, vmcoreinfo_offset, vmcoreinfo_size):
         fd.seek(vmcoreinfo_offset, os.SEEK_SET)
         return VMCoreInfo(BinaryFileReader.read_str(fd, vmcoreinfo_size))
 
-    def get(self, key):
+    def get(self, key, default=None):
         """Retrieve a VMCoreInfo key's value."""
-        return self.data[key]
+        if key in self.data:
+            return self.data[key]
+        return default
 
     def __repr__(self):
         return str(self.data)
 
 
+@dataclass
+class MakeDumpFileHeader:
+    """makedumpfile_header struct."""
+
+    signature: str
+    vtype: int
+    version: int
+
+
+@dataclass
+class MakeDumpFileDataHeader:
+    """makedumpfile_data_header struct."""
+
+    self_offset: int
+    offset: int
+    buf_size: int
+    sizeof: int = 16
+
+    @staticmethod
+    def from_fd(fd):
+        """Read a MakeDumpFileDataHeader from given file."""
+        return MakeDumpFileDataHeader(
+            self_offset=fd.tell(),
+            offset=BinaryFileReader.read_int(fd, ">q"),
+            buf_size=BinaryFileReader.read_int(fd, ">q"),
+        )
+
+    def next(self, fd):
+        """Seek the given file's offset to the next makedumpfile_header"""
+
+        fd.seek(self.self_offset + self.sizeof + self.buf_size, os.SEEK_SET)
+        return MakeDumpFileDataHeader.from_fd(fd)
+
+    def in_range(self, offset):
+        """Check whether given offset is in range of this block or not."""
+        return self.offset <= offset < (self.offset + self.buf_size)
+
+    @property
+    def data_offset(self):
+        """Offset to the beginning of the data."""
+        return self.self_offset + self.sizeof
+
+    def __bool__(self):
+        """Check whether this header is valid."""
+        return (self.offset is not None and self.offset >= 0) and (
+            self.buf_size is not None and self.buf_size >= 0
+        )
+
+
 class KdumpFile:
     """Helper class for parsing headers from kernel crash
     dumps generated with kdump.
     """
 
-    # pylint: disable=too-many-instance-attributes
+    def parse_flattened(self, fd):
+        """Parse the diskdumpfile, ksubhdr and vmcoreinfo from a
+        flattened makedumpfile."""
+
+        # The first header is guaranteed to be disk_dump_header.
+        # Read it first.
+
+        mdhdr = MakeDumpFileDataHeader.from_fd(fd)
+        assert mdhdr.buf_size == DiskDumpHeader.sizeof
+        self._ddhdr = DiskDumpHeader.from_fd(fd)
+
+        # The next header is kdump_sub_header, which is located at the
+        # page offset.
+        disk_dump_header_blocks = 1
+        kdump_sub_header_off = disk_dump_header_blocks * self.ddhdr.block_size
+
+        flat_blocks = []
+        mdhdr = mdhdr.next(fd)
+        while mdhdr:
+            if mdhdr.offset == kdump_sub_header_off:
+                logging.debug("found makedumpfile_data block for ksubhdr: %s", mdhdr)
+                self._ksubhdr = KdumpSubHeader.from_fd(fd)
+
+            flat_blocks.append(mdhdr)
+
+            if self.ksubhdr:
+                for flat_block in flat_blocks:
+                    if flat_block.in_range(self.ksubhdr.offset_vmcoreinfo):
+                        logging.debug(
+                            "found makedumpfile_data block for vmcore: %s", flat_block
+                        )
+                        fd.seek(flat_block.data_offset)
+                        translated_offset = (
+                            self.ksubhdr.offset_vmcoreinfo - flat_block.offset
+                        )
+                        self._vmcoreinfo = VMCoreInfo.from_fd(
+                            fd,
+                            flat_block.data_offset + translated_offset,
+                            self.ksubhdr.size_vmcoreinfo,
+                        )
+                        return
+            mdhdr = mdhdr.next(fd)
+
+    def parse_compressed(self, fd):
+        """Parse the diskdumpfile, ksubhdr and vmcoreinfo from a
+        kdump compressed file."""
+        kdump_hdr_signature = b"KDUMP   "
+        magic = fd.peek(len(kdump_hdr_signature))[0 : len(kdump_hdr_signature)]
+        if magic != kdump_hdr_signature:
+            raise NotAKernelCrashDumpException(
+                f"{self.kdump_file_path} is not a kernel crash dump file"
+            )
+
+        self._ddhdr = DiskDumpHeader.from_fd(fd)
+        # Skip a block
+        disk_dump_header_blocks = 1
+        offset = disk_dump_header_blocks * self.ddhdr.block_size
+        fd.seek(offset, os.SEEK_SET)
+
+        self._ksubhdr = KdumpSubHeader.from_fd(fd)
+        self._vmcoreinfo = VMCoreInfo.from_fd(
+            fd, self.ksubhdr.offset_vmcoreinfo, self.ksubhdr.size_vmcoreinfo
+        )
+
     def __init__(self, kdump_file_path) -> None:
         """Parse kdump file header and expose
         them as member variables
@@ -237,28 +352,29 @@ def __init__(self, kdump_file_path) -> None:
             Exception: If the kdump_file_path is not recognized as a kdump file
         """
 
+        self.kdump_file_path = kdump_file_path
+        self._ddhdr = None
+        self._ksubhdr = None
+        self._vmcoreinfo = None
         with open(kdump_file_path, "rb") as fd:
-
-            # Let's be more forgiving about locating
-            # the KDUMP signature:
-            blob = fd.read(1024 * 8)
-            expected_magic = b"KDUMP   "
-            offset = blob.find(expected_magic)
-            if offset == -1:
-                raise NotAKernelCrashDumpException(
-                    f"{kdump_file_path} is not a kernel crash dump file"
-                )
-
-            # Seek to the KDUMP signature offset
-            fd.seek(offset, os.SEEK_SET)
-            self._ddhdr = DiskDumpHeader.from_fd(fd)
-            self._ksubhdr = KdumpSubHeader.from_fd(fd, self.ddhdr.block_size)
-            self._vmcoreinfo = VMCoreInfo.from_fd(
-                fd, self.ksubhdr.offset_vmcoreinfo, self.ksubhdr.size_vmcoreinfo
-            )
+            # First, check if it's makedumpfile signature
+            makedumpfile_signature = b"makedumpfile\0\0\0\0"
+            signature = fd.peek(len(makedumpfile_signature))[
+                0 : len(makedumpfile_signature)
+            ]
+            # First check if it's flattened format
+            # https://github.com/makedumpfile/makedumpfile/blob/bad2a7c4fa75d37a41578441468584963028bdda/IMPLEMENTATION#L285
+            if signature == makedumpfile_signature:
+                logging.debug("the file is in flattened format")
+                # 1 page makedumpfile_header
+                # 1 makedumpfile_data_header (16 bytes)
+                fd.seek(4096)
+                self.parse_flattened(fd)
+            else:
+                self.parse_compressed(fd)
 
             logging.debug("kdump_hdr: %s", str(self.ddhdr))
-            logging.debug("kdump_subhdr: %s", str(self.ddhdr))
+            logging.debug("kdump_subhdr: %s", str(self.ksubhdr))
             logging.debug("vmcore-info: %s", str(self.vmcoreinfo))
 
     @property