summaryrefslogtreecommitdiff
path: root/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'bolt/lib/Rewrite/LinuxKernelRewriter.cpp')
-rw-r--r--bolt/lib/Rewrite/LinuxKernelRewriter.cpp251
1 files changed, 156 insertions, 95 deletions
diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
index 03b414b71cac..5a5e044184d0 100644
--- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
+++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
@@ -21,6 +21,8 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Errc.h"
+#include "llvm/Support/ErrorOr.h"
+#include <regex>
#define DEBUG_TYPE "bolt-linux"
@@ -89,6 +91,34 @@ static cl::opt<bool>
} // namespace opts
+/// Linux kernel version
+struct LKVersion {
+ LKVersion() {}
+ LKVersion(unsigned Major, unsigned Minor, unsigned Rev)
+ : Major(Major), Minor(Minor), Rev(Rev) {}
+
+ bool operator<(const LKVersion &Other) const {
+ return std::make_tuple(Major, Minor, Rev) <
+ std::make_tuple(Other.Major, Other.Minor, Other.Rev);
+ }
+
+ bool operator>(const LKVersion &Other) const { return Other < *this; }
+
+ bool operator<=(const LKVersion &Other) const { return !(*this > Other); }
+
+ bool operator>=(const LKVersion &Other) const { return !(*this < Other); }
+
+ bool operator==(const LKVersion &Other) const {
+ return Major == Other.Major && Minor == Other.Minor && Rev == Other.Rev;
+ }
+
+ bool operator!=(const LKVersion &Other) const { return !(*this == Other); }
+
+ unsigned Major{0};
+ unsigned Minor{0};
+ unsigned Rev{0};
+};
+
/// Linux Kernel supports stack unwinding using ORC (oops rewind capability).
/// ORC state at every IP can be described by the following data structure.
struct ORCState {
@@ -123,7 +153,33 @@ inline raw_ostream &operator<<(raw_ostream &OS, const ORCState &E) {
namespace {
+/// Extension to DataExtractor that supports reading addresses stored in
+/// PC-relative format.
+class AddressExtractor : public DataExtractor {
+ uint64_t DataAddress;
+
+public:
+ AddressExtractor(StringRef Data, uint64_t DataAddress, bool IsLittleEndian,
+ uint8_t AddressSize)
+ : DataExtractor(Data, IsLittleEndian, AddressSize),
+ DataAddress(DataAddress) {}
+
+ /// Extract 32-bit PC-relative address/pointer.
+ uint64_t getPCRelAddress32(Cursor &C) {
+ const uint64_t Base = DataAddress + C.tell();
+ return Base + (int32_t)getU32(C);
+ }
+
+ /// Extract 64-bit PC-relative address/pointer.
+ uint64_t getPCRelAddress64(Cursor &C) {
+ const uint64_t Base = DataAddress + C.tell();
+ return Base + (int64_t)getU64(C);
+ }
+};
+
class LinuxKernelRewriter final : public MetadataRewriter {
+ LKVersion LinuxKernelVersion;
+
/// Information required for updating metadata referencing an instruction.
struct InstructionFixup {
BinarySection &Section; // Section referencing the instruction.
@@ -225,6 +281,8 @@ class LinuxKernelRewriter final : public MetadataRewriter {
ErrorOr<BinarySection &> PCIFixupSection = std::errc::bad_address;
static constexpr size_t PCI_FIXUP_ENTRY_SIZE = 16;
+ Error detectLinuxKernelVersion();
+
/// Process linux kernel special sections and their relocations.
void processLKSections();
@@ -290,6 +348,9 @@ public:
: MetadataRewriter("linux-kernel-rewriter", BC) {}
Error preCFGInitializer() override {
+ if (Error E = detectLinuxKernelVersion())
+ return E;
+
processLKSections();
if (Error E = processSMPLocks())
@@ -370,6 +431,28 @@ public:
}
};
+Error LinuxKernelRewriter::detectLinuxKernelVersion() {
+ if (BinaryData *BD = BC.getBinaryDataByName("linux_banner")) {
+ const BinarySection &Section = BD->getSection();
+ const std::string S =
+ Section.getContents().substr(BD->getOffset(), BD->getSize()).str();
+
+ const std::regex Re(R"---(Linux version ((\d+)\.(\d+)(\.(\d+))?))---");
+ std::smatch Match;
+ if (std::regex_search(S, Match, Re)) {
+ const unsigned Major = std::stoi(Match[2].str());
+ const unsigned Minor = std::stoi(Match[3].str());
+ const unsigned Rev = Match[5].matched ? std::stoi(Match[5].str()) : 0;
+ LinuxKernelVersion = LKVersion(Major, Minor, Rev);
+ BC.outs() << "BOLT-INFO: Linux kernel version is " << Match[1].str()
+ << "\n";
+ return Error::success();
+ }
+ }
+ return createStringError(errc::executable_format_error,
+ "Linux kernel version is unknown");
+}
+
void LinuxKernelRewriter::processLKSections() {
processLKKSymtab();
processLKKSymtab(true);
@@ -423,13 +506,13 @@ Error LinuxKernelRewriter::processSMPLocks() {
return createStringError(errc::executable_format_error,
"bad size of .smp_locks section");
- DataExtractor DE = DataExtractor(SMPLocksSection->getContents(),
- BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
- DataExtractor::Cursor Cursor(0);
+ AddressExtractor AE(SMPLocksSection->getContents(), SectionAddress,
+ BC.AsmInfo->isLittleEndian(),
+ BC.AsmInfo->getCodePointerSize());
+ AddressExtractor::Cursor Cursor(0);
while (Cursor && Cursor.tell() < SectionSize) {
const uint64_t Offset = Cursor.tell();
- const uint64_t IP = SectionAddress + Offset + (int32_t)DE.getU32(Cursor);
+ const uint64_t IP = AE.getPCRelAddress32(Cursor);
// Consume the status of the cursor.
if (!Cursor)
@@ -499,20 +582,17 @@ Error LinuxKernelRewriter::readORCTables() {
return createStringError(errc::executable_format_error,
"ORC entries number mismatch detected");
- const uint64_t IPSectionAddress = ORCUnwindIPSection->getAddress();
- DataExtractor OrcDE = DataExtractor(ORCUnwindSection->getContents(),
- BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
- DataExtractor IPDE = DataExtractor(ORCUnwindIPSection->getContents(),
- BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
+ DataExtractor OrcDE(ORCUnwindSection->getContents(),
+ BC.AsmInfo->isLittleEndian(),
+ BC.AsmInfo->getCodePointerSize());
+ AddressExtractor IPAE(
+ ORCUnwindIPSection->getContents(), ORCUnwindIPSection->getAddress(),
+ BC.AsmInfo->isLittleEndian(), BC.AsmInfo->getCodePointerSize());
DataExtractor::Cursor ORCCursor(0);
DataExtractor::Cursor IPCursor(0);
uint64_t PrevIP = 0;
for (uint32_t Index = 0; Index < NumORCEntries; ++Index) {
- const uint64_t IP =
- IPSectionAddress + IPCursor.tell() + (int32_t)IPDE.getU32(IPCursor);
-
+ const uint64_t IP = IPAE.getPCRelAddress32(IPCursor);
// Consume the status of the cursor.
if (!IPCursor)
return createStringError(errc::executable_format_error,
@@ -856,15 +936,13 @@ Error LinuxKernelRewriter::validateORCTables() {
if (!ORCUnwindIPSection)
return Error::success();
- const uint64_t IPSectionAddress = ORCUnwindIPSection->getAddress();
- DataExtractor IPDE = DataExtractor(ORCUnwindIPSection->getOutputContents(),
- BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
- DataExtractor::Cursor IPCursor(0);
+ AddressExtractor IPAE(
+ ORCUnwindIPSection->getOutputContents(), ORCUnwindIPSection->getAddress(),
+ BC.AsmInfo->isLittleEndian(), BC.AsmInfo->getCodePointerSize());
+ AddressExtractor::Cursor IPCursor(0);
uint64_t PrevIP = 0;
for (uint32_t Index = 0; Index < NumORCEntries; ++Index) {
- const uint64_t IP =
- IPSectionAddress + IPCursor.tell() + (int32_t)IPDE.getU32(IPCursor);
+ const uint64_t IP = IPAE.getPCRelAddress32(IPCursor);
if (!IPCursor)
return createStringError(errc::executable_format_error,
"out of bounds while reading ORC IP table: %s",
@@ -916,16 +994,14 @@ Error LinuxKernelRewriter::readStaticCalls() {
"static call table size error");
const uint64_t SectionAddress = StaticCallSection->getAddress();
- DataExtractor DE(StaticCallSection->getContents(),
- BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
- DataExtractor::Cursor Cursor(StaticCallTableAddress - SectionAddress);
+ AddressExtractor AE(StaticCallSection->getContents(), SectionAddress,
+ BC.AsmInfo->isLittleEndian(),
+ BC.AsmInfo->getCodePointerSize());
+ AddressExtractor::Cursor Cursor(StaticCallTableAddress - SectionAddress);
uint32_t EntryID = 0;
while (Cursor && Cursor.tell() < Stop->getAddress() - SectionAddress) {
- const uint64_t CallAddress =
- SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor);
- const uint64_t KeyAddress =
- SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor);
+ const uint64_t CallAddress = AE.getPCRelAddress32(Cursor);
+ const uint64_t KeyAddress = AE.getPCRelAddress32(Cursor);
// Consume the status of the cursor.
if (!Cursor)
@@ -1027,18 +1103,15 @@ Error LinuxKernelRewriter::readExceptionTable() {
return createStringError(errc::executable_format_error,
"exception table size error");
- const uint64_t SectionAddress = ExceptionsSection->getAddress();
- DataExtractor DE(ExceptionsSection->getContents(),
- BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
- DataExtractor::Cursor Cursor(0);
+ AddressExtractor AE(
+ ExceptionsSection->getContents(), ExceptionsSection->getAddress(),
+ BC.AsmInfo->isLittleEndian(), BC.AsmInfo->getCodePointerSize());
+ AddressExtractor::Cursor Cursor(0);
uint32_t EntryID = 0;
while (Cursor && Cursor.tell() < ExceptionsSection->getSize()) {
- const uint64_t InstAddress =
- SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor);
- const uint64_t FixupAddress =
- SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor);
- const uint64_t Data = DE.getU32(Cursor);
+ const uint64_t InstAddress = AE.getPCRelAddress32(Cursor);
+ const uint64_t FixupAddress = AE.getPCRelAddress32(Cursor);
+ const uint64_t Data = AE.getU32(Cursor);
// Consume the status of the cursor.
if (!Cursor)
@@ -1134,9 +1207,9 @@ Error LinuxKernelRewriter::readParaInstructions() {
if (!ParavirtualPatchSection)
return Error::success();
- DataExtractor DE = DataExtractor(ParavirtualPatchSection->getContents(),
- BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
+ DataExtractor DE(ParavirtualPatchSection->getContents(),
+ BC.AsmInfo->isLittleEndian(),
+ BC.AsmInfo->getCodePointerSize());
uint32_t EntryID = 0;
DataExtractor::Cursor Cursor(0);
while (Cursor && !DE.eof(Cursor)) {
@@ -1235,15 +1308,14 @@ Error LinuxKernelRewriter::readBugTable() {
return createStringError(errc::executable_format_error,
"bug table size error");
- const uint64_t SectionAddress = BugTableSection->getAddress();
- DataExtractor DE(BugTableSection->getContents(), BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
- DataExtractor::Cursor Cursor(0);
+ AddressExtractor AE(
+ BugTableSection->getContents(), BugTableSection->getAddress(),
+ BC.AsmInfo->isLittleEndian(), BC.AsmInfo->getCodePointerSize());
+ AddressExtractor::Cursor Cursor(0);
uint32_t EntryID = 0;
while (Cursor && Cursor.tell() < BugTableSection->getSize()) {
const uint64_t Pos = Cursor.tell();
- const uint64_t InstAddress =
- SectionAddress + Pos + (int32_t)DE.getU32(Cursor);
+ const uint64_t InstAddress = AE.getPCRelAddress32(Cursor);
Cursor.seek(Pos + BUG_TABLE_ENTRY_SIZE);
if (!Cursor)
@@ -1402,23 +1474,20 @@ Error LinuxKernelRewriter::readAltInstructions() {
Error LinuxKernelRewriter::tryReadAltInstructions(uint32_t AltInstFeatureSize,
bool AltInstHasPadLen,
bool ParseOnly) {
- const uint64_t Address = AltInstrSection->getAddress();
- DataExtractor DE = DataExtractor(AltInstrSection->getContents(),
- BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
+ AddressExtractor AE(
+ AltInstrSection->getContents(), AltInstrSection->getAddress(),
+ BC.AsmInfo->isLittleEndian(), BC.AsmInfo->getCodePointerSize());
+ AddressExtractor::Cursor Cursor(0);
uint64_t EntryID = 0;
- DataExtractor::Cursor Cursor(0);
- while (Cursor && !DE.eof(Cursor)) {
- const uint64_t OrgInstAddress =
- Address + Cursor.tell() + (int32_t)DE.getU32(Cursor);
- const uint64_t AltInstAddress =
- Address + Cursor.tell() + (int32_t)DE.getU32(Cursor);
- const uint64_t Feature = DE.getUnsigned(Cursor, AltInstFeatureSize);
- const uint8_t OrgSize = DE.getU8(Cursor);
- const uint8_t AltSize = DE.getU8(Cursor);
+ while (Cursor && !AE.eof(Cursor)) {
+ const uint64_t OrgInstAddress = AE.getPCRelAddress32(Cursor);
+ const uint64_t AltInstAddress = AE.getPCRelAddress32(Cursor);
+ const uint64_t Feature = AE.getUnsigned(Cursor, AltInstFeatureSize);
+ const uint8_t OrgSize = AE.getU8(Cursor);
+ const uint8_t AltSize = AE.getU8(Cursor);
// Older kernels may have the padlen field.
- const uint8_t PadLen = AltInstHasPadLen ? DE.getU8(Cursor) : 0;
+ const uint8_t PadLen = AltInstHasPadLen ? AE.getU8(Cursor) : 0;
if (!Cursor)
return createStringError(
@@ -1537,19 +1606,17 @@ Error LinuxKernelRewriter::readPCIFixupTable() {
return createStringError(errc::executable_format_error,
"PCI fixup table size error");
- const uint64_t Address = PCIFixupSection->getAddress();
- DataExtractor DE = DataExtractor(PCIFixupSection->getContents(),
- BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
+ AddressExtractor AE(
+ PCIFixupSection->getContents(), PCIFixupSection->getAddress(),
+ BC.AsmInfo->isLittleEndian(), BC.AsmInfo->getCodePointerSize());
+ AddressExtractor::Cursor Cursor(0);
uint64_t EntryID = 0;
- DataExtractor::Cursor Cursor(0);
- while (Cursor && !DE.eof(Cursor)) {
- const uint16_t Vendor = DE.getU16(Cursor);
- const uint16_t Device = DE.getU16(Cursor);
- const uint32_t Class = DE.getU32(Cursor);
- const uint32_t ClassShift = DE.getU32(Cursor);
- const uint64_t HookAddress =
- Address + Cursor.tell() + (int32_t)DE.getU32(Cursor);
+ while (Cursor && !AE.eof(Cursor)) {
+ const uint16_t Vendor = AE.getU16(Cursor);
+ const uint16_t Device = AE.getU16(Cursor);
+ const uint32_t Class = AE.getU32(Cursor);
+ const uint32_t ClassShift = AE.getU32(Cursor);
+ const uint64_t HookAddress = AE.getPCRelAddress32(Cursor);
if (!Cursor)
return createStringError(errc::executable_format_error,
@@ -1654,18 +1721,15 @@ Error LinuxKernelRewriter::readStaticKeysJumpTable() {
"static keys jump table size error");
const uint64_t SectionAddress = StaticKeysJumpSection->getAddress();
- DataExtractor DE(StaticKeysJumpSection->getContents(),
- BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
- DataExtractor::Cursor Cursor(StaticKeysJumpTableAddress - SectionAddress);
+ AddressExtractor AE(StaticKeysJumpSection->getContents(), SectionAddress,
+ BC.AsmInfo->isLittleEndian(),
+ BC.AsmInfo->getCodePointerSize());
+ AddressExtractor::Cursor Cursor(StaticKeysJumpTableAddress - SectionAddress);
uint32_t EntryID = 0;
while (Cursor && Cursor.tell() < Stop->getAddress() - SectionAddress) {
- const uint64_t JumpAddress =
- SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor);
- const uint64_t TargetAddress =
- SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor);
- const uint64_t KeyAddress =
- SectionAddress + Cursor.tell() + (int64_t)DE.getU64(Cursor);
+ const uint64_t JumpAddress = AE.getPCRelAddress32(Cursor);
+ const uint64_t TargetAddress = AE.getPCRelAddress32(Cursor);
+ const uint64_t KeyAddress = AE.getPCRelAddress64(Cursor);
// Consume the status of the cursor.
if (!Cursor)
@@ -1859,21 +1923,18 @@ Error LinuxKernelRewriter::updateStaticKeysJumpTablePostEmit() {
return Error::success();
const uint64_t SectionAddress = StaticKeysJumpSection->getAddress();
- DataExtractor DE(StaticKeysJumpSection->getOutputContents(),
- BC.AsmInfo->isLittleEndian(),
- BC.AsmInfo->getCodePointerSize());
- DataExtractor::Cursor Cursor(StaticKeysJumpTableAddress - SectionAddress);
+ AddressExtractor AE(StaticKeysJumpSection->getOutputContents(),
+ SectionAddress, BC.AsmInfo->isLittleEndian(),
+ BC.AsmInfo->getCodePointerSize());
+ AddressExtractor::Cursor Cursor(StaticKeysJumpTableAddress - SectionAddress);
const BinaryData *Stop = BC.getBinaryDataByName("__stop___jump_table");
uint32_t EntryID = 0;
uint64_t NumShort = 0;
uint64_t NumLong = 0;
while (Cursor && Cursor.tell() < Stop->getAddress() - SectionAddress) {
- const uint64_t JumpAddress =
- SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor);
- const uint64_t TargetAddress =
- SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor);
- const uint64_t KeyAddress =
- SectionAddress + Cursor.tell() + (int64_t)DE.getU64(Cursor);
+ const uint64_t JumpAddress = AE.getPCRelAddress32(Cursor);
+ const uint64_t TargetAddress = AE.getPCRelAddress32(Cursor);
+ const uint64_t KeyAddress = AE.getPCRelAddress64(Cursor);
// Consume the status of the cursor.
if (!Cursor)
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-03 16:11:11 +0000