diff --git a/build.rs b/build.rs index cc5bf99d..36875b49 100644 --- a/build.rs +++ b/build.rs @@ -12,6 +12,7 @@ fn set_linker_script() { let linker_script = match cfg_target_arch.as_str() { "x86_64" if cfg_feature.contains("multiboot") => "platform/multiboot/link.ld", + "aarch64" if cfg_feature.contains("linux") => "linux.ld", _ => return, }; diff --git a/src/arch/aarch64/console.rs b/src/arch/aarch64/console.rs index 3144d2d6..b741a1f1 100644 --- a/src/arch/aarch64/console.rs +++ b/src/arch/aarch64/console.rs @@ -17,11 +17,15 @@ fn stdout() -> SerialPort { const SERIAL_PORT_ADDRESS: u32 = 0x09000000; let fdt = unsafe { - Fdt::from_ptr(ptr::with_exposed_provenance(super::DEVICE_TREE as usize)) + Fdt::from_ptr(ptr::with_exposed_provenance(super::device_tree() as usize)) .expect(".fdt file has invalid header") }; - let property = fdt.chosen().stdout(); + // Firecracker does not set `stdout-path`. + let property = fdt + .chosen() + .stdout() + .or_else(|| fdt.find_compatible(&["ns16550a"])); property .and_then(|node| get_device(node)) .unwrap_or(SerialPort::Qemu(QemuSerial::from_addr( @@ -60,6 +64,13 @@ impl Console { impl Default for Console { fn default() -> Self { let stdout = stdout(); + #[cfg(feature = "linux")] + unsafe { + ptr::write_volatile( + &raw mut super::entry::EXCEPTION_UART, + stdout.get_addr() as usize, + ); + } Self { stdout } } } diff --git a/src/arch/aarch64/drivers/mod.rs b/src/arch/aarch64/drivers/mod.rs index 7dc0b51f..9817d795 100644 --- a/src/arch/aarch64/drivers/mod.rs +++ b/src/arch/aarch64/drivers/mod.rs @@ -33,7 +33,9 @@ pub fn get_device<'a>(node: FdtNode<'_, 'a>) -> Option { let reg = node.reg()?.next()?; for id in compat.all() { - if id == "arm,pl011" { + // Writing a byte to offset 0 transmits it on both, so the PL011 + // driver also serves the 16550 of Firecracker. + if id == "arm,pl011" || id == "ns16550a" { return Some(SerialPort::Qemu(QemuSerial::from_addr( NonZeroU32::new(reg.starting_address as u32).unwrap(), ))); diff --git a/src/arch/aarch64/entry.rs b/src/arch/aarch64/entry.rs index b429c02f..0f30e97d 100644 --- a/src/arch/aarch64/entry.rs +++ b/src/arch/aarch64/entry.rs @@ -19,14 +19,100 @@ global_asm!( start_rust = sym start_rust, ); +// Header of the arm64 Linux boot protocol, see +// +#[cfg(feature = "linux")] +global_asm!( + ".section .text.head, \"ax\"", + "0:", + "b _start", // code0 + ".long 0", // code1 + ".quad 0", // text_offset + ".quad _end - 0b", // image_size + ".quad 0b0010", // little endian, 4 KiB pages + ".quad 0", // res2 + ".quad 0", // res3 + ".quad 0", // res4 + ".ascii \"ARM\\x64\"", // magic + ".long 0", // res5 +); + +/// Address of the UART used by the exception handler below, 0 if unknown. +#[cfg(feature = "linux")] +pub static mut EXCEPTION_UART: usize = 0; + +// Prints ESR, ELR and FAR of an unexpected exception and powers off. +#[cfg(feature = "linux")] +global_asm!( + ".section .text", + ".balign 2048", + "early_vectors:", + ".rept 16", + ".balign 128", + "b 5f", + ".endr", + "5:", + "adrp x9, {uart}", + "ldr x9, [x9, #:lo12:{uart}]", + "cbz x9, 6f", + "mov w11, #'E'", + "strb w11, [x9]", + "mrs x10, esr_el1", + "bl 2f", + "mrs x10, elr_el1", + "bl 2f", + "mrs x10, far_el1", + "bl 2f", + "mov w11, #'\\n'", + "strb w11, [x9]", + // PSCI SYSTEM_OFF + "6: movz x0, #0x8400, lsl #16", + "movk x0, #0x0008", + "hvc #0", + "3: wfe", + "b 3b", + "2: mov x12, #60", + "1: lsr x13, x10, x12", + "and x13, x13, #0xf", + "cmp x13, #10", + "b.lt 4f", + "add x13, x13, #39", + "4: add x13, x13, #48", + "strb w13, [x9]", + "subs x12, x12, #4", + "b.ge 1b", + "mov w13, #' '", + "strb w13, [x9]", + "ret", + uart = sym EXCEPTION_UART, +); + +/// Entered with the address of the device tree in `x0`. #[inline(never)] -pub unsafe fn start_rust() -> ! { - unsafe { pre_init() } +pub unsafe extern "C" fn start_rust(device_tree: u64) -> ! { + #[cfg(feature = "linux")] + unsafe { + asm!( + "adr {tmp}, early_vectors", + "msr vbar_el1, {tmp}", + "isb", + tmp = out(reg) _, + options(nostack), + ); + } + unsafe { + super::set_device_tree(device_tree); + pre_init() + } } unsafe fn pre_init() -> ! { - crate::log::init(); - info!("Enter startup code"); + // Logging uses atomics, which need the MMU under KVM. + #[cfg(not(feature = "linux"))] + { + crate::log::init(); + info!("Enter startup code"); + } /* disable interrupts */ /* @@ -142,6 +228,15 @@ unsafe fn pre_init() -> ! { + endian, ); + #[cfg(feature = "linux")] + { + unsafe { + super::page_tables::enable_early(); + } + crate::log::init(); + info!("Enter startup code"); + } + // Enter loader unsafe { crate::os::loader_main(); diff --git a/src/arch/aarch64/entry.s b/src/arch/aarch64/entry.s index d27db610..ef4062c2 100644 --- a/src/arch/aarch64/entry.s +++ b/src/arch/aarch64/entry.s @@ -5,6 +5,9 @@ .section .text _start: + // Keep the device tree address the Linux boot protocol passes in x0. + mov x19, x0 + // Only proceed on the boot core. Park it otherwise. mrs x1, mpidr_el1 and x1, x1, _core_id_mask @@ -70,6 +73,7 @@ el_2_entry: // Jump to Rust code. el_1_entry: + mov x0, x19 b {start_rust} // Infinitely wait for events (aka "park the core"). diff --git a/src/arch/aarch64/linux.ld b/src/arch/aarch64/linux.ld new file mode 100644 index 00000000..7b705622 --- /dev/null +++ b/src/arch/aarch64/linux.ld @@ -0,0 +1,30 @@ +/* Flat arm64 Linux image, loaded by Firecracker at DRAM start + 2 MiB */ +ENTRY(_start) +phys = 0x80200000; + +SECTIONS +{ + __executable_start = phys; + .text phys : { + KEEP(*(.text.head)) + *(.text) + *(.text.*) + } + .rodata ALIGN(4096) : { + *(.rodata) + *(.rodata.*) + } + .data ALIGN(4096) : { + *(.data) + *(.data.*) + } + .bss ALIGN(4096) : { + *(.bss) + *(.bss.*) + } + _end = .; + /DISCARD/ : { + *(.eh_frame) + *(.eh_frame_hdr) + } +} diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 78a2987e..8ce4be62 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -23,20 +23,59 @@ use crate::os::CONSOLE; use crate::{BootInfoExt, stack}; /// start address of the RAM at Qemu's virt emulation +#[cfg(not(feature = "linux"))] const RAM_START: u64 = 0x40000000; +/// start address of the RAM at Firecracker +#[cfg(feature = "linux")] +const RAM_START: u64 = 0x80000000; + +/// Address of the device tree. +/// /// Qemu assumes for ELF kernel that the fdt is located at -/// start of RAM (0x4000_0000) -/// see -const DEVICE_TREE: u64 = RAM_START; +/// start of RAM (0x4000_0000), see +/// . +/// The Linux boot protocol passes it in `x0` instead. +static mut DEVICE_TREE: u64 = RAM_START; + +unsafe fn set_device_tree(addr: u64) { + if cfg!(feature = "linux") { + unsafe { ptr::write_volatile(&raw mut DEVICE_TREE, addr) } + } +} + +fn device_tree() -> u64 { + unsafe { ptr::read_volatile(&raw const DEVICE_TREE) } +} pub unsafe fn get_memory(_memory_size: u64) -> u64 { let loader_end = elf_symbols::executable_end(); - (loader_end.expose_provenance() as u64).align_up(LargePageSize::SIZE as u64) + let loader_end = (loader_end.expose_provenance() as u64).align_up(LargePageSize::SIZE as u64); + + if cfg!(feature = "linux") { + // The device tree lies at the end of RAM, which the page tables do + // not cover, so it is moved behind the loader. + let fdt = unsafe { + Fdt::from_ptr(ptr::with_exposed_provenance(device_tree() as usize)) + .expect(".fdt file has invalid header") + }; + let size = fdt.total_size(); + unsafe { + ptr::copy_nonoverlapping( + ptr::with_exposed_provenance::(device_tree() as usize), + ptr::with_exposed_provenance_mut::(loader_end as usize), + size, + ); + set_device_tree(loader_end); + } + return (loader_end + size as u64).align_up(LargePageSize::SIZE as u64); + } + + loader_end } pub fn find_kernel() -> &'static [u8] { let fdt = unsafe { - Fdt::from_ptr(ptr::with_exposed_provenance(DEVICE_TREE as usize)) + Fdt::from_ptr(ptr::with_exposed_provenance(device_tree() as usize)) .expect(".fdt file has invalid header") }; @@ -50,8 +89,13 @@ pub unsafe fn boot_kernel(kernel_info: LoadedKernel) -> ! { entry_point, } = kernel_info; + #[cfg(feature = "linux")] + unsafe { + sync_icache(load_info.kernel_image_addr_range.clone()); + } + let fdt = unsafe { - Fdt::from_ptr(ptr::with_exposed_provenance(DEVICE_TREE as usize)) + Fdt::from_ptr(ptr::with_exposed_provenance(device_tree() as usize)) .expect(".fdt file has invalid header") }; let cpus = fdt.cpus().count(); @@ -69,9 +113,13 @@ pub unsafe fn boot_kernel(kernel_info: LoadedKernel) -> ! { unsafe { page_tables::enable(); } + #[cfg(feature = "linux")] + unsafe { + ptr::write_volatile(&raw mut entry::EXCEPTION_UART, 0x1000); + } let fdt = unsafe { - Fdt::from_ptr(ptr::with_exposed_provenance(DEVICE_TREE as usize)) + Fdt::from_ptr(ptr::with_exposed_provenance(device_tree() as usize)) .expect(".fdt file has invalid header") }; @@ -94,7 +142,7 @@ pub unsafe fn boot_kernel(kernel_info: LoadedKernel) -> ! { hardware_info: HardwareInfo { phys_addr_range: ram_start..ram_start + ram_size, serial_port_base: SerialPortBase::new(0x1000), - device_tree: core::num::NonZeroU64::new(DEVICE_TREE), + device_tree: core::num::NonZeroU64::new(device_tree()), }, load_info, platform_info: PlatformInfo::LinuxBoot, @@ -107,6 +155,28 @@ pub unsafe fn boot_kernel(kernel_info: LoadedKernel) -> ! { unsafe { enter_kernel(stack, entry, raw_boot_info) } } +/// Makes code written through the data cache visible to instruction fetches. +#[cfg(feature = "linux")] +unsafe fn sync_icache(range: core::ops::Range) { + let ctr: u64; + unsafe { + asm!("mrs {}, ctr_el0", out(reg) ctr, options(nomem, nostack)); + } + let line = 4u64 << ((ctr >> 16) & 0xf); + + let mut addr = range.start & !(line - 1); + while addr < range.end { + unsafe { + asm!("dc cvau, {}", in(reg) addr, options(nostack)); + } + addr += line; + } + + unsafe { + asm!("dsb ish", "ic iallu", "dsb ish", "isb", options(nostack)); + } +} + unsafe fn enter_kernel(stack: *mut u8, entry: *const (), raw_boot_info: &'static RawBootInfo) -> ! { // Check expected signature of entry function let entry: Entry = { diff --git a/src/arch/aarch64/page_tables.rs b/src/arch/aarch64/page_tables.rs index e18ee115..0fc6c448 100644 --- a/src/arch/aarch64/page_tables.rs +++ b/src/arch/aarch64/page_tables.rs @@ -27,7 +27,7 @@ static mut LEVEL_1_TABLE: PageTable = { table[0] = (&raw mut LEVEL_2_TABLE_SERIAL) .wrapping_byte_add(descr::NORMAL) .cast(); - table[1] = (&raw mut LEVEL_2_TABLE_RAM) + table[(RAM_START >> 30) as usize] = (&raw mut LEVEL_2_TABLE_RAM) .wrapping_byte_add(descr::NORMAL) .cast(); @@ -94,11 +94,61 @@ pub unsafe fn init(uart_address: u32) { } } +/// Identity mapping of the first 4 GiB in 1 GiB blocks, RAM as normal memory. +/// +/// Without the MMU, every access is a device access, which faults on +/// unaligned and exclusive accesses under KVM. +#[cfg(feature = "linux")] +static mut EARLY_LEVEL_0_TABLE: PageTable = { + let mut table = [ptr::null_mut(); _]; + + table[0] = (&raw mut EARLY_LEVEL_1_TABLE) + .wrapping_byte_add(descr::NORMAL) + .cast(); + + PageTable(table) +}; + +#[cfg(feature = "linux")] +static mut EARLY_LEVEL_1_TABLE: PageTable = { + let mut table = [ptr::null_mut(); _]; + + let mut i = 0; + while i < 4 { + let addr = i << 30; + let attr = if addr as u64 >= RAM_START { + descr::BLOCK_NORMAL + } else { + descr::BLOCK_DEVICE + }; + table[i] = ptr::with_exposed_provenance_mut::<()>(addr).wrapping_byte_add(attr); + i += 1; + } + + PageTable(table) +}; + +#[cfg(feature = "linux")] +pub unsafe fn enable_early() { + TTBR1_EL1.set(0); + TTBR0_EL1.set((&raw mut EARLY_LEVEL_0_TABLE).expose_provenance() as u64); + dsb(SY); + isb(SY); + + SCTLR_EL1.modify(SCTLR_EL1::M::Enable); + isb(SY); +} + pub unsafe fn enable() { // Set Translation Table Base Registers (TTBR) TTBR1_EL1.set(0); TTBR0_EL1.set((&raw mut LEVEL_0_TABLE).expose_provenance() as u64); dsb(SY); + // Drop the translations of the early tables, if any. + unsafe { + core::arch::asm!("tlbi vmalle1", options(nostack)); + } + dsb(SY); isb(SY); // Set MMU enable in System Control Register (SCTLR) @@ -118,6 +168,18 @@ struct PageTable([*mut (); 512]); mod descr { pub const NORMAL: usize = AF | SH_INNER | attr_indx(4) | TABLE | VALID; pub const NON_CACHEABLE: usize = AF | SH_INNER | attr_indx(3) | TABLE | VALID; + #[cfg(feature = "linux")] + pub const BLOCK_NORMAL: usize = AF | SH_INNER | attr_indx(4) | VALID; + #[cfg(feature = "linux")] + pub const BLOCK_DEVICE: usize = AF | attr_indx(0) | PXN | UXN | VALID; + + /// Privileged execute-never + #[cfg(feature = "linux")] + const PXN: usize = 1 << 53; + + /// Unprivileged execute-never + #[cfg(feature = "linux")] + const UXN: usize = 1 << 54; /// Valid descriptor const VALID: usize = 1; diff --git a/xtask/src/build.rs b/xtask/src/build.rs index fe7272c9..2260e3f3 100644 --- a/xtask/src/build.rs +++ b/xtask/src/build.rs @@ -47,6 +47,11 @@ impl Build { dist_object.convert_to_elf32_i386()?; } + if self.artifact.target == Target::Aarch64Linux { + eprintln!("Converting object to a flat image"); + dist_object.convert_to_binary()?; + } + eprintln!("Loader available at {}", dist_object.as_ref().display()); Ok(()) } diff --git a/xtask/src/ci/qemu.rs b/xtask/src/ci/qemu.rs index a7322d4e..174a8fad 100644 --- a/xtask/src/ci/qemu.rs +++ b/xtask/src/ci/qemu.rs @@ -265,7 +265,7 @@ impl Qemu { } cpu_args } - Target::X86_64Linux => panic!("unsupported"), + Target::X86_64Linux | Target::Aarch64Linux => panic!("unsupported"), Target::Aarch64Elf | Target::Aarch64BeElf => { let mut cpu_args = if self.accel { todo!() diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs index c7dde09c..ff3d62c9 100644 --- a/xtask/src/clippy.rs +++ b/xtask/src/clippy.rs @@ -17,6 +17,7 @@ impl Clippy { Target::X86_64Multiboot, Target::X86_64Uefi, Target::Aarch64Elf, + Target::Aarch64Linux, Target::Riscv64Sbi, ] { target.install()?; diff --git a/xtask/src/object.rs b/xtask/src/object.rs index 50f57768..f96bd87c 100644 --- a/xtask/src/object.rs +++ b/xtask/src/object.rs @@ -31,4 +31,12 @@ impl Object { cmd!(sh, "{objcopy} --output-target elf32-i386 {object}").run()?; Ok(()) } + + pub fn convert_to_binary(&self) -> Result<()> { + let sh = crate::sh()?; + let objcopy = crate::binutil("objcopy")?; + let object = self.as_ref(); + cmd!(sh, "{objcopy} --output-target binary {object}").run()?; + Ok(()) + } } diff --git a/xtask/src/target.rs b/xtask/src/target.rs index 0cec9abe..7ac9354f 100644 --- a/xtask/src/target.rs +++ b/xtask/src/target.rs @@ -9,6 +9,7 @@ pub enum Target { X86_64Multiboot, X86_64Uefi, Aarch64Elf, + Aarch64Linux, Aarch64BeElf, Riscv64Sbi, } @@ -22,7 +23,7 @@ impl Target { cmd!(sh, "rustup target add {triple}").run()?; } - if self == &Self::X86_64Multiboot { + if matches!(self, Self::X86_64Multiboot | Self::Aarch64Linux) { cmd!(sh, "rustup component add llvm-tools-preview").run()?; } @@ -35,7 +36,7 @@ impl Target { Self::X86_64Linux => "x86_64", Self::X86_64Multiboot => "x86_64", Self::X86_64Uefi => "x86_64", - Self::Aarch64Elf => "aarch64", + Self::Aarch64Elf | Self::Aarch64Linux => "aarch64", Self::Aarch64BeElf => "aarch64_be", Self::Riscv64Sbi => "riscv64", } @@ -46,7 +47,7 @@ impl Target { Self::X86_64Linux => "x86_64-unknown-none", Self::X86_64Multiboot => "x86_64-unknown-none", Self::X86_64Uefi => "x86_64-unknown-uefi", - Self::Aarch64Elf => "aarch64-unknown-none-softfloat", + Self::Aarch64Elf | Self::Aarch64Linux => "aarch64-unknown-none-softfloat", Self::Aarch64BeElf => "aarch64_be-unknown-none-softfloat", Self::Riscv64Sbi => "riscv64imac-unknown-none-elf", } @@ -64,7 +65,7 @@ impl Target { Self::X86_64Linux => &["--target=x86_64-unknown-none"], Self::X86_64Multiboot => &["--target=x86_64-unknown-none"], Self::X86_64Uefi => &["--target=x86_64-unknown-uefi"], - Self::Aarch64Elf => &["--target=aarch64-unknown-none-softfloat"], + Self::Aarch64Elf | Self::Aarch64Linux => &["--target=aarch64-unknown-none-softfloat"], Self::Aarch64BeElf => &[ "--target=aarch64_be-unknown-none-softfloat", "-Zbuild-std=core,alloc,panic_abort", @@ -84,7 +85,9 @@ impl Target { "-Clink-arg=--image-base=0x801ffe00", "-Clink-arg=--section-start=.init=0x80200000", ], - Self::X86_64Linux | Self::X86_64Multiboot => &["-Crelocation-model=static"], + Self::X86_64Linux | Self::X86_64Multiboot | Self::Aarch64Linux => { + &["-Crelocation-model=static"] + } _ => &[], } } @@ -94,6 +97,7 @@ impl Target { Self::X86_64Linux => &["--features=x86_64-linux"], Self::X86_64Multiboot => &["--features=x86_64-multiboot"], Self::Aarch64Elf | Self::Aarch64BeElf => &["--features=elf"], + Self::Aarch64Linux => &["--features=linux"], Self::Riscv64Sbi => &["--features=sbi"], _ => &[], } @@ -112,6 +116,7 @@ impl Target { Self::X86_64Multiboot => "hermit-loader-x86_64-multiboot", Self::X86_64Uefi => "hermit-loader-x86_64.efi", Self::Aarch64Elf => "hermit-loader-aarch64-elf", + Self::Aarch64Linux => "hermit-loader-aarch64-linux", Self::Aarch64BeElf => "hermit-loader-aarch64_be-elf", Self::Riscv64Sbi => "hermit-loader-riscv64-sbi", } @@ -121,7 +126,7 @@ impl Target { pub fn qemu(&self) -> &'static str { match self { Self::X86_64Linux | Self::X86_64Multiboot | Self::X86_64Uefi => "x86_64", - Self::Aarch64Elf | Self::Aarch64BeElf => "aarch64", + Self::Aarch64Elf | Self::Aarch64Linux | Self::Aarch64BeElf => "aarch64", Self::Riscv64Sbi => "riscv64", } } @@ -136,6 +141,7 @@ impl FromStr for Target { "x86_64-multiboot" => Ok(Self::X86_64Multiboot), "x86_64-uefi" => Ok(Self::X86_64Uefi), "aarch64-elf" => Ok(Self::Aarch64Elf), + "aarch64-linux" => Ok(Self::Aarch64Linux), "aarch64_be-elf" => Ok(Self::Aarch64BeElf), "riscv64-sbi" => Ok(Self::Riscv64Sbi), s => Err(anyhow!("Unsupported target: {s}")),