diff --git a/include/linux/init.h b/include/linux/init.h index d501f9704bc4..45f1b7723d86 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -168,10 +168,33 @@ extern bool initcall_debug; * and remove that completely, so the initcall sections have to be marked * as KEEP() in the linker script. */ - -#define __define_initcall(fn, id) \ +#ifdef CONFIG_LTO_CLANG + /* + * With LTO, the compiler doesn't necessarily obey link order for + * initcalls, and the initcall variable needs to be globally unique + * to avoid naming collisions. In order to preserve the correct + * order, we add each variable into its own section and generate a + * linker script (in scripts/link-vmlinux.sh) to ensure the order + * remains correct. We also add a __COUNTER__ prefix to the name, + * so we can retain the order of initcalls within each compilation + * unit, and __LINE__ to make the names more unique. + */ + #define ___lto_initcall(c, l, fn, id, __sec) \ + static initcall_t __initcall_##c##_##l##_##fn##id __used \ + __attribute__((__section__( #__sec \ + __stringify(.init..##c##_##l##_##fn)))) = fn; + #define __lto_initcall(c, l, fn, id, __sec) \ + ___lto_initcall(c, l, fn, id, __sec) + + #define ___define_initcall(fn, id, __sec) \ + __lto_initcall(__COUNTER__, __LINE__, fn, id, __sec) +#else + #define ___define_initcall(fn, id, __sec) \ static initcall_t __initcall_##fn##id __used \ - __attribute__((__section__(".initcall" #id ".init"))) = fn; + __attribute__((__section__(#__sec ".init"))) = fn; +#endif + +#define __define_initcall(fn, id) ___define_initcall(fn, id, .initcall##id) /* * Early initcalls run before initializing SMP. @@ -210,13 +233,8 @@ extern bool initcall_debug; #define __exitcall(fn) \ static exitcall_t __exitcall_##fn __exit_call = fn -#define console_initcall(fn) \ - static initcall_t __initcall_##fn \ - __used __section(.con_initcall.init) = fn - -#define security_initcall(fn) \ - static initcall_t __initcall_##fn \ - __used __section(.security_initcall.init) = fn +#define console_initcall(fn) ___define_initcall(fn, con, .con_initcall) +#define security_initcall(fn) ___define_initcall(fn, security, .security_initcall) struct obs_kernel_param { const char *str; diff --git a/scripts/generate_initcall_order.pl b/scripts/generate_initcall_order.pl new file mode 100755 index 000000000000..f772b4a01caa --- /dev/null +++ b/scripts/generate_initcall_order.pl @@ -0,0 +1,250 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 +# +# Generates a linker script that specifies the correct initcall order. +# +# Copyright (C) 2019 Google LLC + +use strict; +use warnings; +use IO::Handle; + +my $nm = $ENV{'LLVM_NM'} || "llvm-nm"; +my $ar = $ENV{'AR'} || "llvm-ar"; +my $objtree = $ENV{'objtree'} || "."; + +## list of all object files to process, in link order +my @objects; +## currently active child processes +my $jobs = {}; # child process pid -> file handle +## results from child processes +my $results = {}; # object index -> { level, function } + +## reads _NPROCESSORS_ONLN to determine the number of processes to start +sub get_online_processors { + open(my $fh, "getconf _NPROCESSORS_ONLN 2>/dev/null |") + or die "$0: failed to execute getconf: $!"; + my $procs = <$fh>; + close($fh); + + if (!($procs =~ /^\d+$/)) { + return 1; + } + + return int($procs); +} + +## finds initcalls defined in an object file, parses level and function name, +## and prints it out to the parent process +sub find_initcalls { + my ($object) = @_; + + die "$0: object file $object doesn't exist?" if (! -f $object); + + open(my $fh, "\"$nm\" -just-symbol-name -defined-only \"$object\" 2>/dev/null |") + or die "$0: failed to execute \"$nm\": $!"; + + my $initcalls = {}; + + while (<$fh>) { + chomp; + + my ($counter, $line, $symbol) = $_ =~ /^__initcall_(\d+)_(\d+)_(.*)$/; + + if (!defined($counter) || !defined($line) || !defined($symbol)) { + next; + } + + my ($function, $level) = $symbol =~ + /^(.*)((early|rootfs|con|security|[0-9])s?)$/; + + die "$0: duplicate initcall counter value in object $object: $_" + if exists($initcalls->{$counter}); + + $initcalls->{$counter} = { + 'level' => $level, + 'line' => $line, + 'function' => $function + }; + } + + close($fh); + + # sort initcalls in each object file numerically by the counter value + # to ensure they are in the order they were defined + foreach my $counter (sort { $a <=> $b } keys(%{$initcalls})) { + print $initcalls->{$counter}->{"level"} . " " . + $counter . " " . + $initcalls->{$counter}->{"line"} . " " . + $initcalls->{$counter}->{"function"} . "\n"; + } +} + +## waits for any child process to complete, reads the results, and adds them to +## the $results array for later processing +sub wait_for_results { + my $pid = wait(); + if ($pid > 0) { + my $fh = $jobs->{$pid}; + + # the child process prints out results in the following format: + # line 1: + # line 2..n: + + my $index = <$fh>; + chomp($index); + + if (!($index =~ /^\d+$/)) { + die "$0: child $pid returned an invalid index: $index"; + } + $index = int($index); + + while (<$fh>) { + chomp; + my ($level, $counter, $line, $function) = $_ =~ + /^([^\ ]+)\ (\d+)\ (\d+)\ (.*)$/; + + if (!defined($level) || + !defined($counter) || + !defined($line) || + !defined($function)) { + die "$0: child $pid returned invalid data"; + } + + if (!exists($results->{$index})) { + $results->{$index} = []; + } + + push (@{$results->{$index}}, { + 'level' => $level, + 'counter' => $counter, + 'line' => $line, + 'function' => $function + }); + } + + close($fh); + delete($jobs->{$pid}); + } +} + +## launches child processes to find initcalls from the object files, waits for +## each process to complete and collects the results +sub process_objects { + my $index = 0; # link order index of the object file + my $njobs = get_online_processors(); + + while (scalar(@objects) > 0) { + my $object = shift(@objects); + + # fork a child process and read it's stdout + my $pid = open(my $fh, '-|'); + + if (!defined($pid)) { + die "$0: failed to fork: $!"; + } elsif ($pid) { + # save the child process pid and the file handle + $jobs->{$pid} = $fh; + } else { + STDOUT->autoflush(1); + print "$index\n"; + find_initcalls("$objtree/$object"); + exit; + } + + $index++; + + # if we reached the maximum number of processes, wait for one + # to complete before launching new ones + if (scalar(keys(%{$jobs})) >= $njobs && scalar(@objects) > 0) { + wait_for_results(); + } + } + + # wait for the remaining children to complete + while (scalar(keys(%{$jobs})) > 0) { + wait_for_results(); + } +} + +## gets a list of actual object files from thin archives, and adds them to +## @objects in link order +sub find_objects { + while (my $file = shift(@ARGV)) { + my $pid = open (my $fh, "\"$ar\" t \"$file\" 2>/dev/null |") + or die "$0: failed to execute $ar: $!"; + + my @output; + + while (<$fh>) { + chomp; + push(@output, $_); + } + + close($fh); + + # if $ar failed, assume we have an object file + if ($? != 0) { + push(@objects, $file); + next; + } + + # if $ar succeeded, read the list of object files + foreach (@output) { + push(@objects, $_); + } + } +} + +## START +find_objects(); +process_objects(); + +## process results and add them to $sections in the correct order +my $sections = {}; + +foreach my $index (sort { $a <=> $b } keys(%{$results})) { + foreach my $result (@{$results->{$index}}) { + my $level = $result->{'level'}; + + if (!exists($sections->{$level})) { + $sections->{$level} = []; + } + + my $fsname = $result->{'counter'} . '_' . + $result->{'line'} . '_' . + $result->{'function'}; + + push(@{$sections->{$level}}, $fsname); + } +} + +if (!keys(%{$sections})) { + exit(0); # no initcalls...? +} + +## print out a linker script that defines the order of initcalls for each +## level +print "SECTIONS {\n"; + +foreach my $level (sort(keys(%{$sections}))) { + my $section; + + if ($level eq 'con') { + $section = '.con_initcall.init'; + } elsif ($level eq 'security') { + $section = '.security_initcall.init'; + } else { + $section = ".initcall${level}.init"; + } + + print "\t${section} : {\n"; + + foreach my $fsname (@{$sections->{$level}}) { + print "\t\t*(${section}..${fsname}) ;\n" + } + + print "\t}\n"; +} + +print "}\n"; diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh index cfa44718cef7..ecba415423ec 100755 --- a/scripts/link-vmlinux.sh +++ b/scripts/link-vmlinux.sh @@ -70,29 +70,30 @@ archive_builtin() fi } -# If CONFIG_LTO_CLANG is selected, collect generated symbol versions into -# .tmp_symversions -modversions() +# If CONFIG_LTO_CLANG is selected, generate a linker script to ensure correct +# ordering of initcalls, and with CONFIG_MODVERSIONS also enabled, collect the +# previously generated symbol versions into the same script. +lto_lds() { if [ -z "${CONFIG_LTO_CLANG}" ]; then return fi - if [ -z "${CONFIG_MODVERSIONS}" ]; then - return - fi - - rm -f .tmp_symversions - - for a in built-in.o ${KBUILD_VMLINUX_LIBS}; do - for o in $(${AR} t $a); do - if [ -f ${o}.symversions ]; then - cat ${o}.symversions >> .tmp_symversions - fi + ${srctree}/scripts/generate_initcall_order.pl \ + built-in.o ${KBUILD_VMLINUX_LIBS} \ + > .tmp_lto.lds + + if [ -n "${CONFIG_MODVERSIONS}" ]; then + for a in built-in.o ${KBUILD_VMLINUX_LIBS}; do + for o in $(${AR} t $a); do + if [ -f ${o}.symversions ]; then + cat ${o}.symversions >> .tmp_lto.lds + fi + done done - done + fi - echo "-T .tmp_symversions" + echo "-T .tmp_lto.lds" } # Link of vmlinux.o used for section mismatch analysis @@ -124,7 +125,7 @@ modpost_link() info LD vmlinux.o fi - ${LD} ${LDFLAGS} -r -o ${1} $(modversions) ${objects} + ${LD} ${LDFLAGS} -r -o ${1} $(lto_lds) ${objects} } # If CONFIG_LTO_CLANG is selected, we postpone running recordmcount until @@ -251,7 +252,7 @@ cleanup() rm -f .tmp_System.map rm -f .tmp_kallsyms* rm -f .tmp_version - rm -f .tmp_symversions + rm -f .tmp_lto.lds rm -f .tmp_vmlinux* rm -f built-in.o rm -f System.map