feat(aarch64): add Linux boot target for Firecracker

build.rs

@@ -12,6 +12,7 @@ fn set_linker_script() {
 
 	let linker_script = match cfg_target_arch.as_str() {
 		"x86_64" if cfg_feature.contains("multiboot") => "platform/multiboot/link.ld",
+		"aarch64" if cfg_feature.contains("linux") => "linux.ld",
 		_ => return,
 	};
 

src/arch/aarch64/console.rs

@@ -17,11 +17,15 @@ fn stdout() -> SerialPort {
 	const SERIAL_PORT_ADDRESS: u32 = 0x09000000;
 
 	let fdt = unsafe {
-		Fdt::from_ptr(ptr::with_exposed_provenance(super::DEVICE_TREE as usize))
+		Fdt::from_ptr(ptr::with_exposed_provenance(super::device_tree() as usize))
 			.expect(".fdt file has invalid header")
 	};
 
-	let property = fdt.chosen().stdout();
+	// Firecracker does not set `stdout-path`.
+	let property = fdt
+		.chosen()
+		.stdout()
+		.or_else(|| fdt.find_compatible(&["ns16550a"]));
 	property
 		.and_then(|node| get_device(node))
 		.unwrap_or(SerialPort::Qemu(QemuSerial::from_addr(
@@ -60,6 +64,13 @@ impl Console {
 impl Default for Console {
 	fn default() -> Self {
 		let stdout = stdout();
+		#[cfg(feature = "linux")]
+		unsafe {
+			ptr::write_volatile(
+				&raw mut super::entry::EXCEPTION_UART,
+				stdout.get_addr() as usize,
+			);
+		}
 		Self { stdout }
 	}
 }

src/arch/aarch64/drivers/mod.rs

@@ -33,7 +33,9 @@ pub fn get_device<'a>(node: FdtNode<'_, 'a>) -> Option<SerialPort> {
 	let reg = node.reg()?.next()?;
 
 	for id in compat.all() {
-		if id == "arm,pl011" {
+		// Writing a byte to offset 0 transmits it on both, so the PL011
+		// driver also serves the 16550 of Firecracker.
+		if id == "arm,pl011" || id == "ns16550a" {
 			return Some(SerialPort::Qemu(QemuSerial::from_addr(
 				NonZeroU32::new(reg.starting_address as u32).unwrap(),
 			)));

src/arch/aarch64/entry.rs

@@ -19,14 +19,100 @@ global_asm!(
 	start_rust = sym start_rust,
 );
 
+// Header of the arm64 Linux boot protocol, see
+// <https://docs.kernel.org/arch/arm64/booting.html>
+#[cfg(feature = "linux")]
+global_asm!(
+	".section .text.head, \"ax\"",
+	"0:",
+	"b _start",            // code0
+	".long 0",             // code1
+	".quad 0",             // text_offset
+	".quad _end - 0b",     // image_size
+	".quad 0b0010",        // little endian, 4 KiB pages
+	".quad 0",             // res2
+	".quad 0",             // res3
+	".quad 0",             // res4
+	".ascii \"ARM\\x64\"", // magic
+	".long 0",             // res5
+);
+
+/// Address of the UART used by the exception handler below, 0 if unknown.
+#[cfg(feature = "linux")]
+pub static mut EXCEPTION_UART: usize = 0;
+
+// Prints ESR, ELR and FAR of an unexpected exception and powers off.
+#[cfg(feature = "linux")]
+global_asm!(
+	".section .text",
+	".balign 2048",
+	"early_vectors:",
+	".rept 16",
+	".balign 128",
+	"b 5f",
+	".endr",
+	"5:",
+	"adrp x9, {uart}",
+	"ldr x9, [x9, #:lo12:{uart}]",
+	"cbz x9, 6f",
+	"mov w11, #'E'",
+	"strb w11, [x9]",
+	"mrs x10, esr_el1",
+	"bl 2f",
+	"mrs x10, elr_el1",
+	"bl 2f",
+	"mrs x10, far_el1",
+	"bl 2f",
+	"mov w11, #'\\n'",
+	"strb w11, [x9]",
+	// PSCI SYSTEM_OFF
+	"6: movz x0, #0x8400, lsl #16",
+	"movk x0, #0x0008",
+	"hvc #0",
+	"3: wfe",
+	"b 3b",
+	"2: mov x12, #60",
+	"1: lsr x13, x10, x12",
+	"and x13, x13, #0xf",
+	"cmp x13, #10",
+	"b.lt 4f",
+	"add x13, x13, #39",
+	"4: add x13, x13, #48",
+	"strb w13, [x9]",
+	"subs x12, x12, #4",
+	"b.ge 1b",
+	"mov w13, #' '",
+	"strb w13, [x9]",
+	"ret",
+	uart = sym EXCEPTION_UART,
+);
+
+/// Entered with the address of the device tree in `x0`.
 #[inline(never)]
-pub unsafe fn start_rust() -> ! {
-	unsafe { pre_init() }
+pub unsafe extern "C" fn start_rust(device_tree: u64) -> ! {
+	#[cfg(feature = "linux")]
+	unsafe {
+		asm!(
+			"adr {tmp}, early_vectors",
+			"msr vbar_el1, {tmp}",
+			"isb",
+			tmp = out(reg) _,
+			options(nostack),
+		);
+	}
+	unsafe {
+		super::set_device_tree(device_tree);
+		pre_init()
+	}
 }
 
 unsafe fn pre_init() -> ! {
-	crate::log::init();
-	info!("Enter startup code");
+	// Logging uses atomics, which need the MMU under KVM.
+	#[cfg(not(feature = "linux"))]
+	{
+		crate::log::init();
+		info!("Enter startup code");
+	}
 
 	/* disable interrupts */
 	/*
@@ -142,6 +228,15 @@ unsafe fn pre_init() -> ! {
 			+ endian,
 	);
 
+	#[cfg(feature = "linux")]
+	{
+		unsafe {
+			super::page_tables::enable_early();
+		}
+		crate::log::init();
+		info!("Enter startup code");
+	}
+
 	// Enter loader
 	unsafe {
 		crate::os::loader_main();

src/arch/aarch64/entry.s

@@ -5,6 +5,9 @@
 .section .text
 
 _start:
+	// Keep the device tree address the Linux boot protocol passes in x0.
+	mov	x19, x0
+
 	// Only proceed on the boot core. Park it otherwise.
 	mrs	x1, mpidr_el1
 	and	x1, x1, _core_id_mask
@@ -70,6 +73,7 @@ el_2_entry:
 
 	// Jump to Rust code.
 el_1_entry:
+	mov	x0, x19
 	b	{start_rust}
 
 	// Infinitely wait for events (aka "park the core").

src/arch/aarch64/linux.ld

@@ -0,0 +1,30 @@
+/* Flat arm64 Linux image, loaded by Firecracker at DRAM start + 2 MiB */
+ENTRY(_start)
+phys = 0x80200000;
+
+SECTIONS
+{
+  __executable_start = phys;
+  .text phys : {
+    KEEP(*(.text.head))
+    *(.text)
+    *(.text.*)
+  }
+  .rodata ALIGN(4096) : {
+    *(.rodata)
+    *(.rodata.*)
+  }
+  .data ALIGN(4096) : {
+    *(.data)
+    *(.data.*)
+  }
+  .bss ALIGN(4096) : {
+    *(.bss)
+    *(.bss.*)
+  }
+  _end = .;
+  /DISCARD/ : {
+    *(.eh_frame)
+    *(.eh_frame_hdr)
+  }
+}

src/arch/aarch64/mod.rs

@@ -23,20 +23,59 @@ use crate::os::CONSOLE;
 use crate::{BootInfoExt, stack};
 
 /// start address of the RAM at Qemu's virt emulation
+#[cfg(not(feature = "linux"))]
 const RAM_START: u64 = 0x40000000;
+/// start address of the RAM at Firecracker
+#[cfg(feature = "linux")]
+const RAM_START: u64 = 0x80000000;
+
+/// Address of the device tree.
+///
 /// Qemu assumes for ELF kernel that the fdt is located at
-/// start of RAM (0x4000_0000)
-/// see <https://qemu.readthedocs.io/en/latest/system/arm/virt.html>
-const DEVICE_TREE: u64 = RAM_START;
+/// start of RAM (0x4000_0000), see
+/// <https://qemu.readthedocs.io/en/latest/system/arm/virt.html>.
+/// The Linux boot protocol passes it in `x0` instead.
+static mut DEVICE_TREE: u64 = RAM_START;
+
+unsafe fn set_device_tree(addr: u64) {
+	if cfg!(feature = "linux") {
+		unsafe { ptr::write_volatile(&raw mut DEVICE_TREE, addr) }
+	}
+}
+
+fn device_tree() -> u64 {
+	unsafe { ptr::read_volatile(&raw const DEVICE_TREE) }
+}
 
 pub unsafe fn get_memory(_memory_size: u64) -> u64 {
 	let loader_end = elf_symbols::executable_end();
-	(loader_end.expose_provenance() as u64).align_up(LargePageSize::SIZE as u64)
+	let loader_end = (loader_end.expose_provenance() as u64).align_up(LargePageSize::SIZE as u64);
+
+	if cfg!(feature = "linux") {
+		// The device tree lies at the end of RAM, which the page tables do
+		// not cover, so it is moved behind the loader.
+		let fdt = unsafe {
+			Fdt::from_ptr(ptr::with_exposed_provenance(device_tree() as usize))
+				.expect(".fdt file has invalid header")
+		};
+		let size = fdt.total_size();
+		unsafe {
+			ptr::copy_nonoverlapping(
+				ptr::with_exposed_provenance::<u8>(device_tree() as usize),
+				ptr::with_exposed_provenance_mut::<u8>(loader_end as usize),
+				size,
+			);
+			set_device_tree(loader_end);
+		}
+		return (loader_end + size as u64).align_up(LargePageSize::SIZE as u64);
+	}
+
+	loader_end
 }
 
 pub fn find_kernel() -> &'static [u8] {
 	let fdt = unsafe {
-		Fdt::from_ptr(ptr::with_exposed_provenance(DEVICE_TREE as usize))
+		Fdt::from_ptr(ptr::with_exposed_provenance(device_tree() as usize))
 			.expect(".fdt file has invalid header")
 	};
 
@@ -50,8 +89,13 @@ pub unsafe fn boot_kernel(kernel_info: LoadedKernel) -> ! {
 		entry_point,
 	} = kernel_info;
 
+	#[cfg(feature = "linux")]
+	unsafe {
+		sync_icache(load_info.kernel_image_addr_range.clone());
+	}
+
 	let fdt = unsafe {
-		Fdt::from_ptr(ptr::with_exposed_provenance(DEVICE_TREE as usize))
+		Fdt::from_ptr(ptr::with_exposed_provenance(device_tree() as usize))
 			.expect(".fdt file has invalid header")
 	};
 	let cpus = fdt.cpus().count();
@@ -69,9 +113,13 @@ pub unsafe fn boot_kernel(kernel_info: LoadedKernel) -> ! {
 	unsafe {
 		page_tables::enable();
 	}
+	#[cfg(feature = "linux")]
+	unsafe {
+		ptr::write_volatile(&raw mut entry::EXCEPTION_UART, 0x1000);
+	}
 
 	let fdt = unsafe {
-		Fdt::from_ptr(ptr::with_exposed_provenance(DEVICE_TREE as usize))
+		Fdt::from_ptr(ptr::with_exposed_provenance(device_tree() as usize))
 			.expect(".fdt file has invalid header")
 	};
 
@@ -94,7 +142,7 @@ pub unsafe fn boot_kernel(kernel_info: LoadedKernel) -> ! {
 		hardware_info: HardwareInfo {
 			phys_addr_range: ram_start..ram_start + ram_size,
 			serial_port_base: SerialPortBase::new(0x1000),
-			device_tree: core::num::NonZeroU64::new(DEVICE_TREE),
+			device_tree: core::num::NonZeroU64::new(device_tree()),
 		},
 		load_info,
 		platform_info: PlatformInfo::LinuxBoot,
@@ -107,6 +155,28 @@ pub unsafe fn boot_kernel(kernel_info: LoadedKernel) -> ! {
 	unsafe { enter_kernel(stack, entry, raw_boot_info) }
 }
 
+/// Makes code written through the data cache visible to instruction fetches.
+#[cfg(feature = "linux")]
+unsafe fn sync_icache(range: core::ops::Range<u64>) {
+	let ctr: u64;
+	unsafe {
+		asm!("mrs {}, ctr_el0", out(reg) ctr, options(nomem, nostack));
+	}
+	let line = 4u64 << ((ctr >> 16) & 0xf);
+
+	let mut addr = range.start & !(line - 1);
+	while addr < range.end {
+		unsafe {
+			asm!("dc cvau, {}", in(reg) addr, options(nostack));
+		}
+		addr += line;
+	}
+
+	unsafe {
+		asm!("dsb ish", "ic iallu", "dsb ish", "isb", options(nostack));
+	}
+}
+
 unsafe fn enter_kernel(stack: *mut u8, entry: *const (), raw_boot_info: &'static RawBootInfo) -> ! {
 	// Check expected signature of entry function
 	let entry: Entry = {