Load Mach-O bundles and kernel extensions

cle/backends/macho/macho.py

@@ -41,6 +41,16 @@
 
 __all__ = ("MachO", "MachOSection", "MachOSegment", "SymbolList")
 
+# Filetypes whose segments are linked relative to 0 instead of to a fixed load address: shared
+# libraries, the loadable plugin bundles that dlopen() maps, and kernel extensions.
+ZERO_BASED_FILETYPES = frozenset(
+    {
+        MachoFiletype.MH_DYLIB,
+        MachoFiletype.MH_BUNDLE,
+        MachoFiletype.MH_KEXT_BUNDLE,
+    }
+)
+
 
 # pylint: disable=abstract-method
 class SymbolList(SortedKeyList):
@@ -107,7 +117,7 @@ def __init__(self, *args, **kwargs):
         self._mapped_base = None  # temporary holder für mapped base derived via loading
         self.cputype = None
         self.cpusubtype = None
-        self.filetype: int = None
+        self.filetype: MachoFiletype = None
         self.flags = None  # binary flags
         self.imported_libraries: list[str] = ["Self"]  # ordinal 0 = SELF_LIBRARY_ORDINAL
         self.sections_by_ordinal = [None]  # ordinal 0 = None == Self
@@ -153,12 +163,20 @@ def __init__(self, *args, **kwargs):
 
             # parse the mach header:
             # (ignore all irrelevant fields)
-            _, self.cputype, self.cpusubtype, self.filetype, self.ncmds, self.sizeofcmds, self.flags = self._unpack(
+            _, self.cputype, self.cpusubtype, filetype, self.ncmds, self.sizeofcmds, self.flags = self._unpack(
                 "7I", binary_file, 0, 28
             )
+            try:
+                self.filetype = MachoFiletype(filetype)
+            except ValueError as e:
+                # A header with a filetype outside the enum is damaged, which is a compatibility
+                # problem for this backend rather than a bare ValueError for the caller to guess at
+                raise CLECompatibilityError(f"Unknown Mach-O file type: {filetype:#x}") from e
 
-            # Libraries are always implicitly PIC
-            self.pic = bool(self.flags & MH_flags.MH_PIE) or bool(self.filetype & MachoFiletype.MH_DYLIB)
+            # MH_PIE only ever appears on executables; everything linked relative to 0 is implicitly
+            # position independent. filetype is an ordinal, so it has to be compared, not masked.
+            # self.pic already carries the force_rebase option, which stays in force either way.
+            self.pic = self.pic or bool(self.flags & MH_flags.MH_PIE) or self.filetype in ZERO_BASED_FILETYPES
 
             if not bool(self.flags & MH_flags.MH_TWOLEVEL):  # ensure MH_TWOLEVEL
                 log.error(
@@ -178,7 +196,7 @@ def __init__(self, *args, **kwargs):
 
             # Determine the base address the binary was linked against
             # and set the values for the Backend and Loader accordingly
-            if self.pic and self.filetype == MachoFiletype.MH_EXECUTE:
+            if self.filetype == MachoFiletype.MH_EXECUTE:
                 assert self.is_main_bin, "An file of type MH_EXECUTE should be the main bin, this should not happen"
                 # a Position Independent Main binary would later be loaded at 0x400000, which isn't legal for Mach-O
                 # Also, its segment vaddrs are relative to 0x100000000, so we set this as the linked base
@@ -188,30 +206,36 @@ def __init__(self, *args, **kwargs):
                     self.linked_base = self.mapped_base = 2**32
                 elif self.arch.bits == 32:
                     self.linked_base = self.mapped_base = 0x4000
-            elif self.filetype == MachoFiletype.MH_DYLIB and self.is_main_bin:
-                # the segments of dylibs are just relative to the load address, i.e. the lowest segment addr is 0
-                # we need to set the load address to something because otherwise the loader will try to map the
-                # file to 0x400000, which is technically illegal for Mach-O because of PAGEZERO
-                #
-                # The problem is that libraries also tend to have relative pointers (e.g. inside ObjC Metadata),
-                # which are rebased by parsing the rebase_blob, which isn't supported yet (but coming soon)
-                # so we set the base addr to 0 to make them work out without having to deal with this
-                # IDA and Ghidra both seem to handle it this way too
-                # AFAIU this isn't a problem with iOS15+ binaries anymore that use the new binding fixups
-                # but for now we just load all libraries, that are loaded as the main object, at address 0
-                #
-                # We can't set the linked base to request this, because the MachO Backend implementation
-                # uses this to recalculate the addresses
-                self._custom_base_addr = 0
-            elif self.filetype == MachoFiletype.MH_DYLIB and not self.is_main_bin:
-                # A Library is loaded as a dependency, this is fine, the loader will map it to somewhere above the main
-                # binary, so we don't need to do anything
-                pass
+            elif self.filetype in ZERO_BASED_FILETYPES:
+                if self.is_main_bin:
+                    # the segments of dylibs are just relative to the load address, i.e. the lowest segment addr is 0
+                    # we need to set the load address to something because otherwise the loader will try to map the
+                    # file to 0x400000, which is technically illegal for Mach-O because of PAGEZERO
+                    #
+                    # The problem is that libraries also tend to have relative pointers (e.g. inside ObjC Metadata),
+                    # which are rebased by parsing the rebase_blob, which isn't supported yet (but coming soon)
+                    # so we set the base addr to 0 to make them work out without having to deal with this
+                    # IDA and Ghidra both seem to handle it this way too
+                    # AFAIU this isn't a problem with iOS15+ binaries anymore that use the new binding fixups
+                    # but for now we just load all libraries, that are loaded as the main object, at address 0
+                    #
+                    # We can't set the linked base to request this, because the MachO Backend implementation
+                    # uses this to recalculate the addresses
+                    self._custom_base_addr = 0
+                # Otherwise it is loaded as a dependency, which is fine: the loader will map it somewhere above
+                # the main binary, so we don't need to do anything
+            elif self.filetype == MachoFiletype.MH_DSYM:
+                # A dSYM companion holds DWARF and a symbol table for a binary that lives elsewhere; its
+                # __TEXT segment is the mach header and nothing else, so there is nothing to map or analyze
+                raise CLECompatibilityError(
+                    "Mach-O dSYM companion files carry debug information only, not a loadable image"
+                )
             else:
                 # This case is not explicitly supported yet.
-                # There are various other MachoFiletypes, which might have different quirks in their loading
+                # MH_OBJECT in particular needs per-section mapping and relocation processing, the way ELF
+                # handles ET_REL, before its addresses would mean anything.
                 raise CLECompatibilityError(
-                    f"Unsupported Mach-O file type: {MachoFiletype(self.filetype)}. "
+                    f"Unsupported Mach-O file type: {self.filetype.name}. "
                     "Please open an issue if you need support for this"
                 )
 

cle/backends/macho/symbol.py

@@ -26,6 +26,7 @@
 LIBRARY_ORDINAL_SELF = 0x0
 LIBRARY_ORDINAL_OLD_MAX = 0xFE
 LIBRARY_ORDINAL_DYN_LOOKUP = 0xFE
+LIBRARY_ORDINAL_EXECUTABLE = 0xFF
 
 BIND_SPECIAL_DYLIB_SELF = 0x0
 BIND_SPECIAL_DYLIB_WEAK_LOOKUP = 0xFD
@@ -146,6 +147,11 @@ def library_name(self) -> str | None:
             if LIBRARY_ORDINAL_DYN_LOOKUP == self.library_ordinal:
                 log.warning("LIBRARY_ORDINAL_DYN_LOOKUP found, cannot handle")
                 return None
+            elif LIBRARY_ORDINAL_EXECUTABLE == self.library_ordinal:
+                # Bundles are dlopen-ed by an executable and bind against it, so the defining image is
+                # whatever loaded them at runtime rather than one of the libraries they import
+                log.warning("LIBRARY_ORDINAL_EXECUTABLE found, cannot handle")
+                return None
             else:
                 return self.owner.imported_libraries[self.library_ordinal]
         return None

tests/test_macho.py

@@ -1,21 +1,39 @@
 #!/usr/bin/env python
 from __future__ import annotations
 
+import ctypes
 import logging
 import os
 import struct
+import tempfile
 from io import BytesIO
 
+import pytest
+
 import cle
 from cle import MachO
-from cle.backends.macho.macho_enums import LoadCommands
+from cle.backends.macho.macho_enums import LoadCommands, MachoFiletype, MH_flags
 from cle.backends.macho.section import MachOSection
+from cle.backends.macho.structs import DyldImportStruct, dyld_chained_fixups_header
+from cle.backends.macho.symbol import (
+    LIBRARY_ORDINAL_DYN_LOOKUP,
+    LIBRARY_ORDINAL_EXECUTABLE,
+    N_EXT,
+    N_STAB,
+    SYMBOL_TYPE_UNDEF,
+)
+from cle.errors import CLECompatibilityError
 
 TEST_BASE = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.path.join("..", "..", "binaries"))
 
 SEGMENT_COMMAND_64 = "<2I16s4Q4I"
 SEGMENT_COMMAND_64_SIZE = struct.calcsize(SEGMENT_COMMAND_64)
 
+# offsets of mach header fields, and the leading fields of an nlist entry
+FILETYPE_OFFSET = 12
+FLAGS_OFFSET = 24
+NLIST_PREFIX = "<IBBH"  # n_strx, n_type, n_sect, n_desc
+
 
 def test_fauxware():
     machofile = os.path.join(TEST_BASE, "tests", "x86_64", "fauxware.macho")
@@ -349,6 +367,178 @@ def test_filesize_larger_than_vmsize():
     assert macho.max_addr == 0x100003FFF
 
 
+def _copy_with_header_field(source, destination, offset, value):
+    """Copy a little-endian Mach-O file, replacing one 32-bit field of its mach header"""
+    with open(source, "rb") as f:
+        data = bytearray(f.read())
+    struct.pack_into("<I", data, offset, value)
+    with open(destination, "wb") as f:
+        f.write(data)
+    return destination
+
+
+def _copy_with_executable_library_ordinal(source, destination):
+    """Copy a Mach-O file, re-encoding every dynamic lookup import as one bound to the main executable
+
+    That is how the linker encodes the undefined symbols of a dlopen-ed bundle, which resolve against
+    whichever executable loaded it. The ordinal is recorded both in the symbol table and in the chained
+    fixup imports, and the two have to agree for a symbol to be matched up again while loading.
+    """
+    ld = cle.Loader(source, auto_load_libs=False)
+    assert isinstance(ld.main_object, MachO)
+    macho: MachO = ld.main_object
+    assert macho.symtab_offset is not None and macho.symtab_nsyms is not None
+    assert macho._dyld_chained_fixups_offset is not None
+
+    with open(source, "rb") as f:
+        data = bytearray(f.read())
+
+    nlist_size = 16 if macho.arch.bits == 64 else 12
+    for i in range(macho.symtab_nsyms):
+        entry = macho.symtab_offset + i * nlist_size
+        n_strx, n_type, n_sect, n_desc = struct.unpack_from(NLIST_PREFIX, data, entry)
+        if n_type & N_STAB or not n_type & N_EXT or n_type & 0x0E != SYMBOL_TYPE_UNDEF:
+            continue
+        if (n_desc >> 8) & 0xFF == LIBRARY_ORDINAL_DYN_LOOKUP:
+            n_desc = (n_desc & 0xFF) | (LIBRARY_ORDINAL_EXECUTABLE << 8)
+            struct.pack_into(NLIST_PREFIX, data, entry, n_strx, n_type, n_sect, n_desc)
+
+    header = dyld_chained_fixups_header.from_buffer(data, macho._dyld_chained_fixups_offset)
+    import_struct = DyldImportStruct.get_struct(header.imports_format)
+    imports_offset = macho._dyld_chained_fixups_offset + header.imports_offset
+    for i in range(header.imports_count):
+        imported = import_struct.from_buffer(data, imports_offset + i * ctypes.sizeof(import_struct))
+        if imported.lib_ordinal == LIBRARY_ORDINAL_DYN_LOOKUP:
+            imported.lib_ordinal = LIBRARY_ORDINAL_EXECUTABLE
+
+    with open(destination, "wb") as f:
+        f.write(data)
+    return destination
+
+
+def test_kext():
+    """A kernel extension is an ordinary linked image whose segments are relative to 0, like a dylib's"""
+    machofile = os.path.join(TEST_BASE, "tests", "aarch64", "IPwnKit.macho.kext")
+    ld = cle.Loader(machofile, auto_load_libs=False)
+    assert isinstance(ld.main_object, MachO)
+    macho: MachO = ld.main_object
+
+    assert macho.filetype == MachoFiletype.MH_KEXT_BUNDLE
+    assert macho.pic
+    assert [seg.segname for seg in macho.segments] == [
+        "__TEXT",
+        "__TEXT_EXEC",
+        "__DATA",
+        "__DATA_CONST",
+        "__LINKEDIT",
+    ]
+    assert (macho.min_addr, macho.max_addr) == (0x0, 0x23FFF)
+    assert len(macho.symbols) == 846
+
+
+def test_bundle():
+    """A dlopen-ed plugin bundle is laid out just like a dylib, and only its filetype field differs"""
+    machofile = os.path.join(
+        TEST_BASE,
+        "tests",
+        "aarch64",
+        "macho_lib_loading",
+        "FrameWorkApp.app_14",
+        "Frameworks",
+        "dynamicLibrary.framework",
+        "dynamicLibrary",
+    )
+    dylib_ld = cle.Loader(machofile, auto_load_libs=False)
+    assert isinstance(dylib_ld.main_object, MachO)
+    dylib: MachO = dylib_ld.main_object
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        bundle_file = _copy_with_header_field(
+            machofile, os.path.join(tmpdir, "dynamicLibrary.bundle"), FILETYPE_OFFSET, MachoFiletype.MH_BUNDLE
+        )
+        bundle_ld = cle.Loader(bundle_file, auto_load_libs=False)
+        assert isinstance(bundle_ld.main_object, MachO)
+        bundle: MachO = bundle_ld.main_object
+
+    assert bundle.filetype == MachoFiletype.MH_BUNDLE
+    assert bundle.pic
+    assert [seg.segname for seg in bundle.segments] == [seg.segname for seg in dylib.segments]
+    assert (bundle.min_addr, bundle.max_addr) == (dylib.min_addr, dylib.max_addr)
+
+
+def test_executable_library_ordinal():
+    """Library ordinal 255 names the executable that loaded the image, not one of its imported libraries"""
+    machofile = os.path.join(TEST_BASE, "tests", "aarch64", "IPwnKit.macho.kext")
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        patched = _copy_with_executable_library_ordinal(machofile, os.path.join(tmpdir, "IPwnKit.macho.kext"))
+        ld = cle.Loader(patched, auto_load_libs=False)
+        assert isinstance(ld.main_object, MachO)
+        macho: MachO = ld.main_object
+
+    imports = [sym for sym in macho.symbols if sym.library_ordinal == LIBRARY_ORDINAL_EXECUTABLE]
+    assert imports
+    assert all(sym.library_name is None for sym in imports)
+
+
+def test_non_pie_executable_is_not_pic():
+    """Position independence comes from the MH_PIE flag, not from a bitwise test against a filetype ordinal"""
+    machofile = os.path.join(TEST_BASE, "tests", "x86_64", "fauxware.macho")
+    stock_ld = cle.Loader(machofile, auto_load_libs=False)
+    assert isinstance(stock_ld.main_object, MachO)
+    stock: MachO = stock_ld.main_object
+    assert stock.flags is not None and stock.flags & MH_flags.MH_PIE
+    assert stock.pic
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        non_pie = _copy_with_header_field(
+            machofile, os.path.join(tmpdir, "fauxware.nonpie"), FLAGS_OFFSET, stock.flags & ~MH_flags.MH_PIE
+        )
+
+        ld = cle.Loader(non_pie, auto_load_libs=False)
+        assert isinstance(ld.main_object, MachO)
+        macho: MachO = ld.main_object
+        assert macho.filetype == MachoFiletype.MH_EXECUTE
+        assert not macho.pic
+        assert macho.linked_base == macho.mapped_base == 0x100000000
+
+        # force_rebase is a loader option, so it still decides position independence on its own
+        forced_ld = cle.Loader(non_pie, auto_load_libs=False, main_opts={"force_rebase": True})
+        assert isinstance(forced_ld.main_object, MachO)
+        assert forced_ld.main_object.pic
+
+
+def test_dsym_is_rejected():
+    """A dSYM companion holds debug information only, so the refusal says that instead of asking for a report"""
+    machofile = os.path.join(TEST_BASE, "tests", "x86_64", "fauxware.macho")
+    with tempfile.TemporaryDirectory() as tmpdir:
+        dsym = _copy_with_header_field(
+            machofile, os.path.join(tmpdir, "fauxware.dSYM"), FILETYPE_OFFSET, MachoFiletype.MH_DSYM
+        )
+        with pytest.raises(CLECompatibilityError, match="debug information"):
+            cle.Loader(dsym, auto_load_libs=False)
+
+
+def test_unsupported_filetype_is_named():
+    """A filetype this backend cannot map yet is refused by name, not as a bare number"""
+    machofile = os.path.join(TEST_BASE, "tests", "x86_64", "fauxware.macho")
+    with tempfile.TemporaryDirectory() as tmpdir:
+        obj = _copy_with_header_field(
+            machofile, os.path.join(tmpdir, "fauxware.o"), FILETYPE_OFFSET, MachoFiletype.MH_OBJECT
+        )
+        with pytest.raises(CLECompatibilityError, match="MH_OBJECT"):
+            cle.Loader(obj, auto_load_libs=False)
+
+
+def test_unknown_filetype():
+    """A filetype outside the enum is a damaged header, which is this backend's problem to report"""
+    machofile = os.path.join(TEST_BASE, "tests", "x86_64", "fauxware.macho")
+    with tempfile.TemporaryDirectory() as tmpdir:
+        damaged = _copy_with_header_field(machofile, os.path.join(tmpdir, "fauxware.damaged"), FILETYPE_OFFSET, 99)
+        with pytest.raises(CLECompatibilityError, match="Unknown Mach-O file type: 0x63"):
+            cle.Loader(damaged, auto_load_libs=False)
+
+
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
     test_dummy()
@@ -360,3 +550,10 @@ def test_filesize_larger_than_vmsize():
     test_find_symbol()
     test_zero_vmsize_segment()
     test_filesize_larger_than_vmsize()
+    test_kext()
+    test_bundle()
+    test_executable_library_ordinal()
+    test_non_pie_executable_is_not_pic()
+    test_dsym_is_rejected()
+    test_unsupported_filetype_is_named()
+    test_unknown_filetype()