From 266251fbb3adfa4858c588eef69157a119694e05 Mon Sep 17 00:00:00 2001 From: Stefan Lankes Date: Sat, 31 Oct 2026 23:47:31 +0100 Subject: [PATCH] feat(aarch64): add Linux boot target for Firecracker Add the target aarch64-linux, which builds a flat arm64 Linux image linked at 0x80200000. The device tree is taken from x0 and the kernel from linux,initrd-start. The MMU is enabled early, because KVM faults on unaligned and exclusive device accesses. The console also supports the 16550 UART. These changes are required to boot an aarch64 kernel on firecracker. --- build.rs | 1 + src/arch/aarch64/console.rs | 15 ++++- src/arch/aarch64/drivers/mod.rs | 4 +- src/arch/aarch64/entry.rs | 103 ++++++++++++++++++++++++++++++-- src/arch/aarch64/entry.s | 4 ++ src/arch/aarch64/linux.ld | 30 ++++++++++ src/arch/aarch64/mod.rs | 86 +++++++++++++++++++++++--- src/arch/aarch64/page_tables.rs | 64 +++++++++++++++++++- xtask/src/build.rs | 5 ++ xtask/src/ci/qemu.rs | 2 +- xtask/src/clippy.rs | 1 + xtask/src/object.rs | 8 +++ xtask/src/target.rs | 18 ++++-- 13 files changed, 318 insertions(+), 23 deletions(-) create mode 100644 src/arch/aarch64/linux.ld diff --git a/build.rs b/build.rs index cc5bf99d..36875b49 100644 --- a/build.rs +++ b/build.rs @@ -12,6 +12,7 @@ fn set_linker_script() { let linker_script = match cfg_target_arch.as_str() { "x86_64" if cfg_feature.contains("multiboot") => "platform/multiboot/link.ld", + "aarch64" if cfg_feature.contains("linux") => "linux.ld", _ => return, }; diff --git a/src/arch/aarch64/console.rs b/src/arch/aarch64/console.rs index 3144d2d6..b741a1f1 100644 --- a/src/arch/aarch64/console.rs +++ b/src/arch/aarch64/console.rs @@ -17,11 +17,15 @@ fn stdout() -> SerialPort { const SERIAL_PORT_ADDRESS: u32 = 0x09000000; let fdt = unsafe { - Fdt::from_ptr(ptr::with_exposed_provenance(super::DEVICE_TREE as usize)) + Fdt::from_ptr(ptr::with_exposed_provenance(super::device_tree() as usize)) .expect(".fdt file has invalid header") }; - let property = fdt.chosen().stdout(); + // Firecracker does not set `stdout-path`. + let property = fdt + .chosen() + .stdout() + .or_else(|| fdt.find_compatible(&["ns16550a"])); property .and_then(|node| get_device(node)) .unwrap_or(SerialPort::Qemu(QemuSerial::from_addr( @@ -60,6 +64,13 @@ impl Console { impl Default for Console { fn default() -> Self { let stdout = stdout(); + #[cfg(feature = "linux")] + unsafe { + ptr::write_volatile( + &raw mut super::entry::EXCEPTION_UART, + stdout.get_addr() as usize, + ); + } Self { stdout } } } diff --git a/src/arch/aarch64/drivers/mod.rs b/src/arch/aarch64/drivers/mod.rs index 7dc0b51f..9817d795 100644 --- a/src/arch/aarch64/drivers/mod.rs +++ b/src/arch/aarch64/drivers/mod.rs @@ -33,7 +33,9 @@ pub fn get_device<'a>(node: FdtNode<'_, 'a>) -> Option { let reg = node.reg()?.next()?; for id in compat.all() { - if id == "arm,pl011" { + // Writing a byte to offset 0 transmits it on both, so the PL011 + // driver also serves the 16550 of Firecracker. + if id == "arm,pl011" || id == "ns16550a" { return Some(SerialPort::Qemu(QemuSerial::from_addr( NonZeroU32::new(reg.starting_address as u32).unwrap(), ))); diff --git a/src/arch/aarch64/entry.rs b/src/arch/aarch64/entry.rs index b429c02f..0f30e97d 100644 --- a/src/arch/aarch64/entry.rs +++ b/src/arch/aarch64/entry.rs @@ -19,14 +19,100 @@ global_asm!( start_rust = sym start_rust, ); +// Header of the arm64 Linux boot protocol, see +// +#[cfg(feature = "linux")] +global_asm!( + ".section .text.head, \"ax\"", + "0:", + "b _start", // code0 + ".long 0", // code1 + ".quad 0", // text_offset + ".quad _end - 0b", // image_size + ".quad 0b0010", // little endian, 4 KiB pages + ".quad 0", // res2 + ".quad 0", // res3 + ".quad 0", // res4 + ".ascii \"ARM\\x64\"", // magic + ".long 0", // res5 +); + +/// Address of the UART used by the exception handler below, 0 if unknown. +#[cfg(feature = "linux")] +pub static mut EXCEPTION_UART: usize = 0; + +// Prints ESR, ELR and FAR of an unexpected exception and powers off. +#[cfg(feature = "linux")] +global_asm!( + ".section .text", + ".balign 2048", + "early_vectors:", + ".rept 16", + ".balign 128", + "b 5f", + ".endr", + "5:", + "adrp x9, {uart}", + "ldr x9, [x9, #:lo12:{uart}]", + "cbz x9, 6f", + "mov w11, #'E'", + "strb w11, [x9]", + "mrs x10, esr_el1", + "bl 2f", + "mrs x10, elr_el1", + "bl 2f", + "mrs x10, far_el1", + "bl 2f", + "mov w11, #'\\n'", + "strb w11, [x9]", + // PSCI SYSTEM_OFF + "6: movz x0, #0x8400, lsl #16", + "movk x0, #0x0008", + "hvc #0", + "3: wfe", + "b 3b", + "2: mov x12, #60", + "1: lsr x13, x10, x12", + "and x13, x13, #0xf", + "cmp x13, #10", + "b.lt 4f", + "add x13, x13, #39", + "4: add x13, x13, #48", + "strb w13, [x9]", + "subs x12, x12, #4", + "b.ge 1b", + "mov w13, #' '", + "strb w13, [x9]", + "ret", + uart = sym EXCEPTION_UART, +); + +/// Entered with the address of the device tree in `x0`. #[inline(never)] -pub unsafe fn start_rust() -> ! { - unsafe { pre_init() } +pub unsafe extern "C" fn start_rust(device_tree: u64) -> ! { + #[cfg(feature = "linux")] + unsafe { + asm!( + "adr {tmp}, early_vectors", + "msr vbar_el1, {tmp}", + "isb", + tmp = out(reg) _, + options(nostack), + ); + } + unsafe { + super::set_device_tree(device_tree); + pre_init() + } } unsafe fn pre_init() -> ! { - crate::log::init(); - info!("Enter startup code"); + // Logging uses atomics, which need the MMU under KVM. + #[cfg(not(feature = "linux"))] + { + crate::log::init(); + info!("Enter startup code"); + } /* disable interrupts */ /* @@ -142,6 +228,15 @@ unsafe fn pre_init() -> ! { + endian, ); + #[cfg(feature = "linux")] + { + unsafe { + super::page_tables::enable_early(); + } + crate::log::init(); + info!("Enter startup code"); + } + // Enter loader unsafe { crate::os::loader_main(); diff --git a/src/arch/aarch64/entry.s b/src/arch/aarch64/entry.s index d27db610..ef4062c2 100644 --- a/src/arch/aarch64/entry.s +++ b/src/arch/aarch64/entry.s @@ -5,6 +5,9 @@ .section .text _start: + // Keep the device tree address the Linux boot protocol passes in x0. + mov x19, x0 + // Only proceed on the boot core. Park it otherwise. mrs x1, mpidr_el1 and x1, x1, _core_id_mask @@ -70,6 +73,7 @@ el_2_entry: // Jump to Rust code. el_1_entry: + mov x0, x19 b {start_rust} // Infinitely wait for events (aka "park the core"). diff --git a/src/arch/aarch64/linux.ld b/src/arch/aarch64/linux.ld new file mode 100644 index 00000000..7b705622 --- /dev/null +++ b/src/arch/aarch64/linux.ld @@ -0,0 +1,30 @@ +/* Flat arm64 Linux image, loaded by Firecracker at DRAM start + 2 MiB */ +ENTRY(_start) +phys = 0x80200000; + +SECTIONS +{ + __executable_start = phys; + .text phys : { + KEEP(*(.text.head)) + *(.text) + *(.text.*) + } + .rodata ALIGN(4096) : { + *(.rodata) + *(.rodata.*) + } + .data ALIGN(4096) : { + *(.data) + *(.data.*) + } + .bss ALIGN(4096) : { + *(.bss) + *(.bss.*) + } + _end = .; + /DISCARD/ : { + *(.eh_frame) + *(.eh_frame_hdr) + } +} diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 78a2987e..8ce4be62 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -23,20 +23,59 @@ use crate::os::CONSOLE; use crate::{BootInfoExt, stack}; /// start address of the RAM at Qemu's virt emulation +#[cfg(not(feature = "linux"))] const RAM_START: u64 = 0x40000000; +/// start address of the RAM at Firecracker +#[cfg(feature = "linux")] +const RAM_START: u64 = 0x80000000; + +/// Address of the device tree. +/// /// Qemu assumes for ELF kernel that the fdt is located at -/// start of RAM (0x4000_0000) -/// see -const DEVICE_TREE: u64 = RAM_START; +/// start of RAM (0x4000_0000), see +/// . +/// The Linux boot protocol passes it in `x0` instead. +static mut DEVICE_TREE: u64 = RAM_START; + +unsafe fn set_device_tree(addr: u64) { + if cfg!(feature = "linux") { + unsafe { ptr::write_volatile(&raw mut DEVICE_TREE, addr) } + } +} + +fn device_tree() -> u64 { + unsafe { ptr::read_volatile(&raw const DEVICE_TREE) } +} pub unsafe fn get_memory(_memory_size: u64) -> u64 { let loader_end = elf_symbols::executable_end(); - (loader_end.expose_provenance() as u64).align_up(LargePageSize::SIZE as u64) + let loader_end = (loader_end.expose_provenance() as u64).align_up(LargePageSize::SIZE as u64); + + if cfg!(feature = "linux") { + // The device tree lies at the end of RAM, which the page tables do + // not cover, so it is moved behind the loader. + let fdt = unsafe { + Fdt::from_ptr(ptr::with_exposed_provenance(device_tree() as usize)) + .expect(".fdt file has invalid header") + }; + let size = fdt.total_size(); + unsafe { + ptr::copy_nonoverlapping( + ptr::with_exposed_provenance::(device_tree() as usize), + ptr::with_exposed_provenance_mut::(loader_end as usize), + size, + ); + set_device_tree(loader_end); + } + return (loader_end + size as u64).align_up(LargePageSize::SIZE as u64); + } + + loader_end } pub fn find_kernel() -> &'static [u8] { let fdt = unsafe { - Fdt::from_ptr(ptr::with_exposed_provenance(DEVICE_TREE as usize)) + Fdt::from_ptr(ptr::with_exposed_provenance(device_tree() as usize)) .expect(".fdt file has invalid header") }; @@ -50,8 +89,13 @@ pub unsafe fn boot_kernel(kernel_info: LoadedKernel) -> ! { entry_point, } = kernel_info; + #[cfg(feature = "linux")] + unsafe { + sync_icache(load_info.kernel_image_addr_range.clone()); + } + let fdt = unsafe { - Fdt::from_ptr(ptr::with_exposed_provenance(DEVICE_TREE as usize)) + Fdt::from_ptr(ptr::with_exposed_provenance(device_tree() as usize)) .expect(".fdt file has invalid header") }; let cpus = fdt.cpus().count(); @@ -69,9 +113,13 @@ pub unsafe fn boot_kernel(kernel_info: LoadedKernel) -> ! { unsafe { page_tables::enable(); } + #[cfg(feature = "linux")] + unsafe { + ptr::write_volatile(&raw mut entry::EXCEPTION_UART, 0x1000); + } let fdt = unsafe { - Fdt::from_ptr(ptr::with_exposed_provenance(DEVICE_TREE as usize)) + Fdt::from_ptr(ptr::with_exposed_provenance(device_tree() as usize)) .expect(".fdt file has invalid header") }; @@ -94,7 +142,7 @@ pub unsafe fn boot_kernel(kernel_info: LoadedKernel) -> ! { hardware_info: HardwareInfo { phys_addr_range: ram_start..ram_start + ram_size, serial_port_base: SerialPortBase::new(0x1000), - device_tree: core::num::NonZeroU64::new(DEVICE_TREE), + device_tree: core::num::NonZeroU64::new(device_tree()), }, load_info, platform_info: PlatformInfo::LinuxBoot, @@ -107,6 +155,28 @@ pub unsafe fn boot_kernel(kernel_info: LoadedKernel) -> ! { unsafe { enter_kernel(stack, entry, raw_boot_info) } } +/// Makes code written through the data cache visible to instruction fetches. +#[cfg(feature = "linux")] +unsafe fn sync_icache(range: core::ops::Range) { + let ctr: u64; + unsafe { + asm!("mrs {}, ctr_el0", out(reg) ctr, options(nomem, nostack)); + } + let line = 4u64 << ((ctr >> 16) & 0xf); + + let mut addr = range.start & !(line - 1); + while addr < range.end { + unsafe { + asm!("dc cvau, {}", in(reg) addr, options(nostack)); + } + addr += line; + } + + unsafe { + asm!("dsb ish", "ic iallu", "dsb ish", "isb", options(nostack)); + } +} + unsafe fn enter_kernel(stack: *mut u8, entry: *const (), raw_boot_info: &'static RawBootInfo) -> ! { // Check expected signature of entry function let entry: Entry = { diff --git a/src/arch/aarch64/page_tables.rs b/src/arch/aarch64/page_tables.rs index e18ee115..0fc6c448 100644 --- a/src/arch/aarch64/page_tables.rs +++ b/src/arch/aarch64/page_tables.rs @@ -27,7 +27,7 @@ static mut LEVEL_1_TABLE: PageTable = { table[0] = (&raw mut LEVEL_2_TABLE_SERIAL) .wrapping_byte_add(descr::NORMAL) .cast(); - table[1] = (&raw mut LEVEL_2_TABLE_RAM) + table[(RAM_START >> 30) as usize] = (&raw mut LEVEL_2_TABLE_RAM) .wrapping_byte_add(descr::NORMAL) .cast(); @@ -94,11 +94,61 @@ pub unsafe fn init(uart_address: u32) { } } +/// Identity mapping of the first 4 GiB in 1 GiB blocks, RAM as normal memory. +/// +/// Without the MMU, every access is a device access, which faults on +/// unaligned and exclusive accesses under KVM. +#[cfg(feature = "linux")] +static mut EARLY_LEVEL_0_TABLE: PageTable = { + let mut table = [ptr::null_mut(); _]; + + table[0] = (&raw mut EARLY_LEVEL_1_TABLE) + .wrapping_byte_add(descr::NORMAL) + .cast(); + + PageTable(table) +}; + +#[cfg(feature = "linux")] +static mut EARLY_LEVEL_1_TABLE: PageTable = { + let mut table = [ptr::null_mut(); _]; + + let mut i = 0; + while i < 4 { + let addr = i << 30; + let attr = if addr as u64 >= RAM_START { + descr::BLOCK_NORMAL + } else { + descr::BLOCK_DEVICE + }; + table[i] = ptr::with_exposed_provenance_mut::<()>(addr).wrapping_byte_add(attr); + i += 1; + } + + PageTable(table) +}; + +#[cfg(feature = "linux")] +pub unsafe fn enable_early() { + TTBR1_EL1.set(0); + TTBR0_EL1.set((&raw mut EARLY_LEVEL_0_TABLE).expose_provenance() as u64); + dsb(SY); + isb(SY); + + SCTLR_EL1.modify(SCTLR_EL1::M::Enable); + isb(SY); +} + pub unsafe fn enable() { // Set Translation Table Base Registers (TTBR) TTBR1_EL1.set(0); TTBR0_EL1.set((&raw mut LEVEL_0_TABLE).expose_provenance() as u64); dsb(SY); + // Drop the translations of the early tables, if any. + unsafe { + core::arch::asm!("tlbi vmalle1", options(nostack)); + } + dsb(SY); isb(SY); // Set MMU enable in System Control Register (SCTLR) @@ -118,6 +168,18 @@ struct PageTable([*mut (); 512]); mod descr { pub const NORMAL: usize = AF | SH_INNER | attr_indx(4) | TABLE | VALID; pub const NON_CACHEABLE: usize = AF | SH_INNER | attr_indx(3) | TABLE | VALID; + #[cfg(feature = "linux")] + pub const BLOCK_NORMAL: usize = AF | SH_INNER | attr_indx(4) | VALID; + #[cfg(feature = "linux")] + pub const BLOCK_DEVICE: usize = AF | attr_indx(0) | PXN | UXN | VALID; + + /// Privileged execute-never + #[cfg(feature = "linux")] + const PXN: usize = 1 << 53; + + /// Unprivileged execute-never + #[cfg(feature = "linux")] + const UXN: usize = 1 << 54; /// Valid descriptor const VALID: usize = 1; diff --git a/xtask/src/build.rs b/xtask/src/build.rs index fe7272c9..2260e3f3 100644 --- a/xtask/src/build.rs +++ b/xtask/src/build.rs @@ -47,6 +47,11 @@ impl Build { dist_object.convert_to_elf32_i386()?; } + if self.artifact.target == Target::Aarch64Linux { + eprintln!("Converting object to a flat image"); + dist_object.convert_to_binary()?; + } + eprintln!("Loader available at {}", dist_object.as_ref().display()); Ok(()) } diff --git a/xtask/src/ci/qemu.rs b/xtask/src/ci/qemu.rs index a7322d4e..174a8fad 100644 --- a/xtask/src/ci/qemu.rs +++ b/xtask/src/ci/qemu.rs @@ -265,7 +265,7 @@ impl Qemu { } cpu_args } - Target::X86_64Linux => panic!("unsupported"), + Target::X86_64Linux | Target::Aarch64Linux => panic!("unsupported"), Target::Aarch64Elf | Target::Aarch64BeElf => { let mut cpu_args = if self.accel { todo!() diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs index c7dde09c..ff3d62c9 100644 --- a/xtask/src/clippy.rs +++ b/xtask/src/clippy.rs @@ -17,6 +17,7 @@ impl Clippy { Target::X86_64Multiboot, Target::X86_64Uefi, Target::Aarch64Elf, + Target::Aarch64Linux, Target::Riscv64Sbi, ] { target.install()?; diff --git a/xtask/src/object.rs b/xtask/src/object.rs index 50f57768..f96bd87c 100644 --- a/xtask/src/object.rs +++ b/xtask/src/object.rs @@ -31,4 +31,12 @@ impl Object { cmd!(sh, "{objcopy} --output-target elf32-i386 {object}").run()?; Ok(()) } + + pub fn convert_to_binary(&self) -> Result<()> { + let sh = crate::sh()?; + let objcopy = crate::binutil("objcopy")?; + let object = self.as_ref(); + cmd!(sh, "{objcopy} --output-target binary {object}").run()?; + Ok(()) + } } diff --git a/xtask/src/target.rs b/xtask/src/target.rs index 0cec9abe..7ac9354f 100644 --- a/xtask/src/target.rs +++ b/xtask/src/target.rs @@ -9,6 +9,7 @@ pub enum Target { X86_64Multiboot, X86_64Uefi, Aarch64Elf, + Aarch64Linux, Aarch64BeElf, Riscv64Sbi, } @@ -22,7 +23,7 @@ impl Target { cmd!(sh, "rustup target add {triple}").run()?; } - if self == &Self::X86_64Multiboot { + if matches!(self, Self::X86_64Multiboot | Self::Aarch64Linux) { cmd!(sh, "rustup component add llvm-tools-preview").run()?; } @@ -35,7 +36,7 @@ impl Target { Self::X86_64Linux => "x86_64", Self::X86_64Multiboot => "x86_64", Self::X86_64Uefi => "x86_64", - Self::Aarch64Elf => "aarch64", + Self::Aarch64Elf | Self::Aarch64Linux => "aarch64", Self::Aarch64BeElf => "aarch64_be", Self::Riscv64Sbi => "riscv64", } @@ -46,7 +47,7 @@ impl Target { Self::X86_64Linux => "x86_64-unknown-none", Self::X86_64Multiboot => "x86_64-unknown-none", Self::X86_64Uefi => "x86_64-unknown-uefi", - Self::Aarch64Elf => "aarch64-unknown-none-softfloat", + Self::Aarch64Elf | Self::Aarch64Linux => "aarch64-unknown-none-softfloat", Self::Aarch64BeElf => "aarch64_be-unknown-none-softfloat", Self::Riscv64Sbi => "riscv64imac-unknown-none-elf", } @@ -64,7 +65,7 @@ impl Target { Self::X86_64Linux => &["--target=x86_64-unknown-none"], Self::X86_64Multiboot => &["--target=x86_64-unknown-none"], Self::X86_64Uefi => &["--target=x86_64-unknown-uefi"], - Self::Aarch64Elf => &["--target=aarch64-unknown-none-softfloat"], + Self::Aarch64Elf | Self::Aarch64Linux => &["--target=aarch64-unknown-none-softfloat"], Self::Aarch64BeElf => &[ "--target=aarch64_be-unknown-none-softfloat", "-Zbuild-std=core,alloc,panic_abort", @@ -84,7 +85,9 @@ impl Target { "-Clink-arg=--image-base=0x801ffe00", "-Clink-arg=--section-start=.init=0x80200000", ], - Self::X86_64Linux | Self::X86_64Multiboot => &["-Crelocation-model=static"], + Self::X86_64Linux | Self::X86_64Multiboot | Self::Aarch64Linux => { + &["-Crelocation-model=static"] + } _ => &[], } } @@ -94,6 +97,7 @@ impl Target { Self::X86_64Linux => &["--features=x86_64-linux"], Self::X86_64Multiboot => &["--features=x86_64-multiboot"], Self::Aarch64Elf | Self::Aarch64BeElf => &["--features=elf"], + Self::Aarch64Linux => &["--features=linux"], Self::Riscv64Sbi => &["--features=sbi"], _ => &[], } @@ -112,6 +116,7 @@ impl Target { Self::X86_64Multiboot => "hermit-loader-x86_64-multiboot", Self::X86_64Uefi => "hermit-loader-x86_64.efi", Self::Aarch64Elf => "hermit-loader-aarch64-elf", + Self::Aarch64Linux => "hermit-loader-aarch64-linux", Self::Aarch64BeElf => "hermit-loader-aarch64_be-elf", Self::Riscv64Sbi => "hermit-loader-riscv64-sbi", } @@ -121,7 +126,7 @@ impl Target { pub fn qemu(&self) -> &'static str { match self { Self::X86_64Linux | Self::X86_64Multiboot | Self::X86_64Uefi => "x86_64", - Self::Aarch64Elf | Self::Aarch64BeElf => "aarch64", + Self::Aarch64Elf | Self::Aarch64Linux | Self::Aarch64BeElf => "aarch64", Self::Riscv64Sbi => "riscv64", } } @@ -136,6 +141,7 @@ impl FromStr for Target { "x86_64-multiboot" => Ok(Self::X86_64Multiboot), "x86_64-uefi" => Ok(Self::X86_64Uefi), "aarch64-elf" => Ok(Self::Aarch64Elf), + "aarch64-linux" => Ok(Self::Aarch64Linux), "aarch64_be-elf" => Ok(Self::Aarch64BeElf), "riscv64-sbi" => Ok(Self::Riscv64Sbi), s => Err(anyhow!("Unsupported target: {s}")),