Stop container objects from claiming their children's addresses

cle/backends/cartfile.py

@@ -52,6 +52,8 @@ def __init__(self, binary, binary_stream, *args, arc4_key=None, **kwargs):
         if self.loader._main_object is None:
             self.loader._main_object = self
         child = self.loader._load_object_isolated(ostream, obj_ident=self.unpacked_name)
+        # the child was loaded from a stream and would otherwise report the stream's repr as its name
+        child.binary_basename = self.unpacked_name
         self.child_objects.append(child)
         self.has_memory = False
 

cle/loader.py

@@ -1053,8 +1053,11 @@ def _map_object(self, obj: Backend):
                     obj.binary_basename,
                 )
             base_addr = obj.linked_base
-            if not self._is_range_free(obj.linked_base, obj_size):
-                raise CLEError(f"Position-DEPENDENT object {obj.binary} cannot be loaded at {base_addr:#x}")
+            conflict = self._describe_range_conflict(obj.linked_base, obj_size)
+            if conflict is not None:
+                raise CLEError(
+                    f"Position-DEPENDENT object {obj.binary_basename} cannot be loaded at {base_addr:#x}: {conflict}"
+                )
 
         assert obj.mapped_base >= 0
 
@@ -1102,6 +1105,9 @@ def _free_gaps(self, start: int, end: int) -> Iterator[tuple[int, int]]:
         ``Loader.memory``.
         """
         for o in self.all_objects:  # sorted by min_addr
+            if o.is_outer:
+                # outer objects occupy no address space; see _describe_range_conflict
+                continue
             if o.max_addr < start:
                 continue
             if o.min_addr >= end:
@@ -1116,15 +1122,29 @@ def _free_gaps(self, start: int, end: int) -> Iterator[tuple[int, int]]:
             yield start, end
 
     def _is_range_free(self, va, size):
+        return self._describe_range_conflict(va, size) is None
+
+    def _describe_range_conflict(self, va, size) -> str | None:
+        """
+        Describe what keeps an object of ``size`` bytes from being placed at ``va``, or return None if nothing does.
+        The description is a sentence fragment about the object being placed, meant to be appended to an error message.
+        """
         # self.main_object should not be None here
-        if va < 0 or va + size > 2**self.main_object.arch.bits:
-            return False
+        bits = self.main_object.arch.bits
+        if va < 0:
+            return "the address is negative"
+        if va + size > 2**bits:
+            return f"it is {size:#x} bytes long and would run past the end of the {bits}-bit address space"
 
         for o in self.all_objects:
+            # an outer object is only a container for the objects it unpacks and backs no memory of its own, so like
+            # find_object_containing, placement does not count it as part of the address space
+            if o.is_outer:
+                continue
             if o.min_addr <= va <= o.max_addr or va <= o.min_addr < va + size:
-                return False
+                return f"it would overlap {o.binary_basename}, which is mapped at [{o.min_addr:#x}, {o.max_addr:#x}]"
 
-        return True
+        return None
 
     # Functions of the form "use some heuristic to tell me about this spec"
 

tests/test_cart.py

@@ -116,9 +116,46 @@ def test_cart_find_object_containing_excludes_wrapper():
     assert obj_at_0 is None, f"find_object_containing(0) should return None, not {type(obj_at_0).__name__}"
 
 
+def test_cart_child_is_named_after_the_wrapper():
+    """
+    The unpacked object is handed to the loader as a stream and has no path of its own, so it reports the wrapper's
+    unpacked name instead of None. Error messages about the object are the only place that name shows up.
+    """
+    cartfile = os.path.join(
+        TEST_BASE,
+        "tests",
+        "x86_64",
+        "1after909.cart",
+    )
+    ld = cle.Loader(
+        cartfile, auto_load_libs=False, main_opts={"arc4_key": b"\x02\xf53asdf\x00\x00\x00\x00\x00\x00\x00\x00\x00"}
+    )
+    assert ld.main_object.binary_basename == "1after909.cart.unpacked"
+
+
+def test_cart_layout_matches_unwrapped():
+    """
+    The wrapper is mapped first and reports a one-byte span at 0. That byte used to move the address the loader picks
+    for the objects it rebases, so a wrapped binary came out laid out differently from the same binary unwrapped.
+    """
+    plain = cle.Loader(os.path.join(TEST_BASE, "tests", "x86_64", "1after909"), auto_load_libs=False)
+    ld = cle.Loader(
+        os.path.join(TEST_BASE, "tests", "x86_64", "1after909.cart"),
+        auto_load_libs=False,
+        main_opts={"arc4_key": b"\x02\xf53asdf\x00\x00\x00\x00\x00\x00\x00\x00\x00"},
+    )
+
+    def layout(loader):
+        return [(type(o).__name__, o.min_addr, o.max_addr) for o in loader.all_objects if not o.is_outer]
+
+    assert layout(ld) == layout(plain)
+
+
 if __name__ == "__main__":
     test_cart_pe()
     test_cart_elf()
     test_cart_elf_with_load_options()
     test_cart_blob_with_load_options()
     test_cart_find_object_containing_excludes_wrapper()
+    test_cart_child_is_named_after_the_wrapper()
+    test_cart_layout_matches_unwrapped()

tests/test_overlap.py

@@ -2,15 +2,22 @@
 
 import os
 
+import pytest
+
 import cle
 
+TEST_BASE = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.path.join("..", "..", "binaries"))
+
+#: the key tests/x86_64/1after909.cart was packed with
+CART_KEY = b"\x02\xf53asdf\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+
 
 class MockBackend(cle.backends.Backend):  # pylint: disable=missing-class-docstring
-    def __init__(self, linked_base, size, **kwargs):
+    def __init__(self, linked_base, size, pic=True, **kwargs):
         super().__init__("/dev/zero", None, **kwargs)
         self.mapped_base = self.linked_base = linked_base
         self.size = size
-        self.pic = True
+        self.pic = pic
 
     @property
     def max_addr(self):
@@ -33,5 +40,104 @@ def test_overlap():
     assert obj1.mapped_base > 0x8048000
 
 
+def test_outer_object_does_not_occupy_address_space():
+    """
+    An outer object backs no memory and reports a one-byte span at whatever base it was placed at. That byte must not
+    keep the object it unpacks from being mapped there. The CaRT wrapper is mapped first and keeps its linked base of
+    0, so a position-dependent child linked at 0 was refused for overlapping its own container.
+    """
+    cartfile = os.path.join(TEST_BASE, "tests", "x86_64", "1after909.cart")
+    ld = cle.Loader(
+        cartfile,
+        auto_load_libs=False,
+        main_opts={"arc4_key": CART_KEY},
+        lib_opts={
+            cle.backends.CARTFile.get_unpacked_name(cartfile): {
+                "backend": "blob",
+                "arch": "amd64",
+                "base_addr": 0,
+            }
+        },
+    )
+
+    (container,) = (o for o in ld.all_objects if o.is_outer)
+    assert isinstance(container, cle.backends.CARTFile)
+    # the container claims exactly the address its child is linked at
+    assert container.min_addr == container.max_addr == 0
+
+    (child,) = container.child_objects
+    assert child is ld.main_object
+    assert not child.pic
+    assert child.mapped_base == 0
+    # and the child, not the container, is what answers for that address
+    assert ld.find_object_containing(0) is child
+    with open(os.path.join(TEST_BASE, "tests", "x86_64", "1after909"), "rb") as fp:
+        assert ld.memory.load(0, 16) == fp.read(16)
+
+
+def test_outer_object_does_not_move_rebased_objects():
+    """
+    The free space the loader rebases into starts at 0 when the main object reaches into the top half of the address
+    space, which is where an outer object mapped at its linked base of 0 sits. Counting that one byte pushed everything
+    the loader rebases up by a full granule, so a wrapped binary came out laid out differently from the same binary
+    unwrapped.
+    """
+    plain_path = os.path.join(TEST_BASE, "tests", "x86_64", "1after909")
+    cartfile = os.path.join(TEST_BASE, "tests", "x86_64", "1after909.cart")
+    # loading the image high is what puts the free space, and so the container's byte, below the main object
+    blob_opts = {"backend": "blob", "arch": "amd64", "base_addr": 0xFFFFFFFF80000000}
+
+    plain = cle.Loader(plain_path, auto_load_libs=False, main_opts=dict(blob_opts))
+    wrapped = cle.Loader(
+        cartfile,
+        auto_load_libs=False,
+        main_opts={"arc4_key": CART_KEY},
+        lib_opts={cle.backends.CARTFile.get_unpacked_name(cartfile): dict(blob_opts)},
+    )
+
+    (container,) = (o for o in wrapped.all_objects if o.is_outer)
+    assert container.min_addr == container.max_addr == 0
+    assert wrapped.main_object.min_addr == plain.main_object.min_addr
+
+    def rebase(ld):
+        obj = MockBackend(0, 0x1000, arch=ld.main_object.arch)
+        ld.dynamic_load(obj)
+        return obj.mapped_base
+
+    assert rebase(wrapped) == rebase(plain) == 0
+
+
+def test_memoryless_region_still_reserves_address_space():
+    """
+    Backing no memory is not what keeps an object out of the address space: a NamedRegion has no memory on purpose and
+    exists to reserve a range cle has no data for, so it has to keep taking part in the overlap check.
+    """
+    ld = cle.Loader(os.path.join(TEST_BASE, "tests", "i386", "manysum"), auto_load_libs=False)
+
+    region = cle.NamedRegion("mmio", 0x8000000, 0x8001000, arch=ld.main_object.arch)
+    assert not region.has_memory
+    assert not region.is_outer
+    ld.dynamic_load(region)
+
+    with pytest.raises(cle.CLEError, match="would overlap mmio"):
+        ld.dynamic_load(MockBackend(0x8000000, 0x1000, pic=False, arch=ld.main_object.arch))
+
+
+def test_placement_past_the_end_of_the_address_space():
+    """
+    An object that does not fit in the architecture's address space is refused before any overlap check, so the error
+    has to name the address space rather than an object that is in the way.
+    """
+    with pytest.raises(cle.CLEError, match="past the end of the 32-bit address space"):
+        cle.Loader(
+            os.path.join(TEST_BASE, "tests", "i386", "manysum"),
+            main_opts={"backend": "blob", "arch": "i386", "base_addr": 0xFFFFF000},
+        )
+
+
 if __name__ == "__main__":
     test_overlap()
+    test_outer_object_does_not_occupy_address_space()
+    test_outer_object_does_not_move_rebased_objects()
+    test_memoryless_region_still_reserves_address_space()
+    test_placement_past_the_end_of_the_address_space()