diff --git a/Makefile b/Makefile index d9e1f3b3af22..fd7aad0eff03 100644 --- a/Makefile +++ b/Makefile @@ -941,6 +941,10 @@ ifeq ($(CONFIG_STRIP_ASM_SYMS),y) LDFLAGS_vmlinux += $(call ld-option, -X,) endif +ifeq ($(CONFIG_RELR),y) +LDFLAGS_vmlinux += --pack-dyn-relocs=relr +endif + # Default kernel image to build when no specific target is given. # KBUILD_IMAGE may be overruled on the command line or # set in the environment diff --git a/arch/Kconfig b/arch/Kconfig index 0f2363604193..784c1fe35583 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1042,4 +1042,18 @@ config HAVE_ARCH_COMPILER_H linux/compiler-*.h in order to override macro definitions that those headers generally provide. +# Select if the architecture has support for applying RELR relocations. +config ARCH_HAS_RELR + bool + +config RELR + bool "Use RELR relocation packing" + depends on ARCH_HAS_RELR && TOOLS_SUPPORT_RELR + default y + help + Store the kernel's dynamic relocations in the RELR relocation packing + format. Requires a compatible linker (LLD supports this feature), as + well as compatible NM and OBJCOPY utilities (llvm-nm and llvm-objcopy + are compatible). + source "kernel/gcov/Kconfig" diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index d469fccc7f31..f90229df23aa 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1041,6 +1041,7 @@ config ARM64_MODULE_PLTS config RELOCATABLE bool + select ARCH_HAS_RELR help This builds the kernel as a Position Independent Executable (PIE), which retains all relocation metadata required to relocate the diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 1371542de0d3..2fde78082d45 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -112,6 +112,8 @@ pe_header: * x23 stext() .. start_kernel() physical misalignment/KASLR offset * x28 __create_page_tables() callee preserved temp register * x19/x20 __primary_switch() callee preserved temp registers + * x24 __primary_switch() .. relocate_kernel() + * current RELR displacement */ ENTRY(stext) bl preserve_boot_args @@ -700,14 +702,93 @@ __relocate_kernel: 0: cmp x9, x10 b.hs 1f - ldp x11, x12, [x9], #24 - ldr x13, [x9, #-8] - cmp w12, #R_AARCH64_RELATIVE + ldp x12, x13, [x9], #24 + ldr x14, [x9, #-8] + cmp w13, #R_AARCH64_RELATIVE b.ne 0b - add x13, x13, x23 // relocate - str x13, [x11, x23] + add x14, x14, x23 // relocate + str x14, [x12, x23] b 0b -1: ret + +1: +#ifdef CONFIG_RELR + /* + * Apply RELR relocations. + * + * RELR is a compressed format for storing relative relocations. The + * encoded sequence of entries looks like: + * [ AAAAAAAA BBBBBBB1 BBBBBBB1 ... AAAAAAAA BBBBBB1 ... ] + * + * i.e. start with an address, followed by any number of bitmaps. The + * address entry encodes 1 relocation. The subsequent bitmap entries + * encode up to 63 relocations each, at subsequent offsets following + * the last address entry. + * + * The bitmap entries must have 1 in the least significant bit. The + * assumption here is that an address cannot have 1 in lsb. Odd + * addresses are not supported. Any odd addresses are stored in the RELA + * section, which is handled above. + * + * Excluding the least significant bit in the bitmap, each non-zero + * bit in the bitmap represents a relocation to be applied to + * a corresponding machine word that follows the base address + * word. The second least significant bit represents the machine + * word immediately following the initial address, and each bit + * that follows represents the next word, in linear order. As such, + * a single bitmap can encode up to 63 relocations in a 64-bit object. + * + * In this implementation we store the address of the next RELR table + * entry in x9, the address being relocated by the current address or + * bitmap entry in x13 and the address being relocated by the current + * bit in x14. + * + * Because addends are stored in place in the binary, RELR relocations + * cannot be applied idempotently. We use x24 to keep track of the + * currently applied displacement so that we can correctly relocate if + * __relocate_kernel is called twice with non-zero displacements (i.e. + * if there is both a physical misalignment and a KASLR displacement). + */ + ldr w9, =__relr_offset // offset to reloc table + ldr w10, =__relr_size // size of reloc table + add x9, x9, x11 // __va(.relr) + add x10, x9, x10 // __va(.relr) + sizeof(.relr) + + sub x15, x23, x24 // delta from previous offset + cbz x15, 7f // nothing to do if unchanged + mov x24, x23 // save new offset + +2: cmp x9, x10 + b.hs 7f + ldr x11, [x9], #8 + tbnz x11, #0, 3f // branch to handle bitmaps + add x13, x11, x23 + ldr x12, [x13] // relocate address entry + add x12, x12, x15 + str x12, [x13], #8 // adjust to start of bitmap + b 2b + +3: mov x14, x13 +4: lsr x11, x11, #1 + cbz x11, 6f + tbz x11, #0, 5f // skip bit if not set + ldr x12, [x14] // relocate bit + add x12, x12, x15 + str x12, [x14] + +5: add x14, x14, #8 // move to next bit's address + b 4b + +6: /* + * Move to the next bitmap's address. 8 is the word size, and 63 is the + * number of significant bits in a bitmap entry. + */ + add x13, x13, #(8 * 63) + b 2b + +7: +#endif + ret + ENDPROC(__relocate_kernel) #endif @@ -719,6 +800,9 @@ __primary_switch: bl __enable_mmu #ifdef CONFIG_RELOCATABLE +#ifdef CONFIG_RELR + mov x24, #0 // no RELR displacement yet +#endif bl __relocate_kernel #ifdef CONFIG_RANDOMIZE_BASE ldr x8, =__primary_switched diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index a2e6d19cbd16..6a632c6fdd7d 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -184,6 +184,15 @@ SECTIONS __rela_offset = ABSOLUTE(ADDR(.rela.dyn) - KIMAGE_VADDR); __rela_size = SIZEOF(.rela.dyn); +#ifdef CONFIG_RELR + .relr.dyn : ALIGN(8) { + *(.relr.dyn) + } + + __relr_offset = ABSOLUTE(ADDR(.relr.dyn) - KIMAGE_VADDR); + __relr_size = SIZEOF(.relr.dyn); +#endif + . = ALIGN(SEGMENT_ALIGN); __initdata_end = .; __init_end = .; diff --git a/init/Kconfig b/init/Kconfig index fc56cab80dc7..b1cabfa234b2 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -16,6 +16,9 @@ config DEFCONFIG_LIST default "$ARCH_DEFCONFIG" default "arch/$ARCH/defconfig" +config TOOLS_SUPPORT_RELR + bool "Declare tool support for RELR" + config CONSTRUCTORS bool depends on !UML