summaryrefslogtreecommitdiff
path: root/llvm/lib/ProfileData
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/ProfileData')
-rw-r--r--llvm/lib/ProfileData/InstrProfReader.cpp2
-rw-r--r--llvm/lib/ProfileData/InstrProfWriter.cpp217
-rw-r--r--llvm/lib/ProfileData/MemProf.cpp170
3 files changed, 304 insertions, 85 deletions
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 7ac5c561dc08..884334ed070e 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -1303,7 +1303,7 @@ Error IndexedInstrProfReader::readHeader() {
MemProfRecordTable.reset(MemProfRecordHashTable::Create(
/*Buckets=*/Start + RecordTableOffset,
/*Payload=*/Ptr,
- /*Base=*/Start, memprof::RecordLookupTrait(Schema)));
+ /*Base=*/Start, memprof::RecordLookupTrait(memprof::Version1, Schema)));
// Initialize the frame table reader with the payload and bucket offsets.
MemProfFrameTable.reset(MemProfFrameHashTable::Create(
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index c2c94ba30c65..72d77d5580a5 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -414,6 +414,144 @@ static void setSummary(IndexedInstrProf::Summary *TheSummary,
TheSummary->setEntry(I, Res[I]);
}
+// Serialize Schema.
+static void writeMemProfSchema(ProfOStream &OS,
+ const memprof::MemProfSchema &Schema) {
+ OS.write(static_cast<uint64_t>(Schema.size()));
+ for (const auto Id : Schema)
+ OS.write(static_cast<uint64_t>(Id));
+}
+
+// Serialize MemProfRecordData. Return RecordTableOffset.
+static uint64_t writeMemProfRecords(
+ ProfOStream &OS,
+ llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
+ &MemProfRecordData,
+ memprof::MemProfSchema *Schema) {
+ auto RecordWriter =
+ std::make_unique<memprof::RecordWriterTrait>(memprof::Version1);
+ RecordWriter->Schema = Schema;
+ OnDiskChainedHashTableGenerator<memprof::RecordWriterTrait>
+ RecordTableGenerator;
+ for (auto &I : MemProfRecordData) {
+ // Insert the key (func hash) and value (memprof record).
+ RecordTableGenerator.insert(I.first, I.second, *RecordWriter.get());
+ }
+ // Release the memory of this MapVector as it is no longer needed.
+ MemProfRecordData.clear();
+
+ // The call to Emit invokes RecordWriterTrait::EmitData which destructs
+ // the memprof record copies owned by the RecordTableGenerator. This works
+ // because the RecordTableGenerator is not used after this point.
+ return RecordTableGenerator.Emit(OS.OS, *RecordWriter);
+}
+
+// Serialize MemProfFrameData. Return FrameTableOffset.
+static uint64_t writeMemProfFrames(
+ ProfOStream &OS,
+ llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData) {
+ auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>();
+ OnDiskChainedHashTableGenerator<memprof::FrameWriterTrait>
+ FrameTableGenerator;
+ for (auto &I : MemProfFrameData) {
+ // Insert the key (frame id) and value (frame contents).
+ FrameTableGenerator.insert(I.first, I.second);
+ }
+ // Release the memory of this MapVector as it is no longer needed.
+ MemProfFrameData.clear();
+
+ return FrameTableGenerator.Emit(OS.OS, *FrameWriter);
+}
+
+static Error writeMemProfV0(
+ ProfOStream &OS,
+ llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
+ &MemProfRecordData,
+ llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData) {
+ uint64_t HeaderUpdatePos = OS.tell();
+ OS.write(0ULL); // Reserve space for the memprof record table offset.
+ OS.write(0ULL); // Reserve space for the memprof frame payload offset.
+ OS.write(0ULL); // Reserve space for the memprof frame table offset.
+
+ auto Schema = memprof::PortableMemInfoBlock::getSchema();
+ writeMemProfSchema(OS, Schema);
+
+ uint64_t RecordTableOffset =
+ writeMemProfRecords(OS, MemProfRecordData, &Schema);
+
+ uint64_t FramePayloadOffset = OS.tell();
+ uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfFrameData);
+
+ uint64_t Header[] = {RecordTableOffset, FramePayloadOffset, FrameTableOffset};
+ OS.patch({{HeaderUpdatePos, Header, std::size(Header)}});
+
+ return Error::success();
+}
+
+static Error writeMemProfV1(
+ ProfOStream &OS,
+ llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
+ &MemProfRecordData,
+ llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData) {
+ OS.write(memprof::Version0);
+ uint64_t HeaderUpdatePos = OS.tell();
+ OS.write(0ULL); // Reserve space for the memprof record table offset.
+ OS.write(0ULL); // Reserve space for the memprof frame payload offset.
+ OS.write(0ULL); // Reserve space for the memprof frame table offset.
+
+ auto Schema = memprof::PortableMemInfoBlock::getSchema();
+ writeMemProfSchema(OS, Schema);
+
+ uint64_t RecordTableOffset =
+ writeMemProfRecords(OS, MemProfRecordData, &Schema);
+
+ uint64_t FramePayloadOffset = OS.tell();
+ uint64_t FrameTableOffset = writeMemProfFrames(OS, MemProfFrameData);
+
+ uint64_t Header[] = {RecordTableOffset, FramePayloadOffset, FrameTableOffset};
+ OS.patch({{HeaderUpdatePos, Header, std::size(Header)}});
+
+ return Error::success();
+}
+
+// The MemProf profile data includes a simple schema
+// with the format described below followed by the hashtable:
+// uint64_t Version
+// uint64_t RecordTableOffset = RecordTableGenerator.Emit
+// uint64_t FramePayloadOffset = Stream offset before emitting the frame table
+// uint64_t FrameTableOffset = FrameTableGenerator.Emit
+// uint64_t Num schema entries
+// uint64_t Schema entry 0
+// uint64_t Schema entry 1
+// ....
+// uint64_t Schema entry N - 1
+// OnDiskChainedHashTable MemProfRecordData
+// OnDiskChainedHashTable MemProfFrameData
+static Error writeMemProf(
+ ProfOStream &OS,
+ llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
+ &MemProfRecordData,
+ llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData,
+ memprof::IndexedVersion MemProfVersionRequested) {
+
+ switch (MemProfVersionRequested) {
+ case memprof::Version0:
+ return writeMemProfV0(OS, MemProfRecordData, MemProfFrameData);
+ case memprof::Version1:
+ return writeMemProfV1(OS, MemProfRecordData, MemProfFrameData);
+ case memprof::Version2:
+ // TODO: Implement. Fall through to the error handling below for now.
+ break;
+ }
+
+ return make_error<InstrProfError>(
+ instrprof_error::unsupported_version,
+ formatv("MemProf version {} not supported; "
+ "requires version between {} and {}, inclusive",
+ MemProfVersionRequested, memprof::MinimumSupportedVersion,
+ memprof::MaximumSupportedVersion));
+}
+
Error InstrProfWriter::writeImpl(ProfOStream &OS) {
using namespace IndexedInstrProf;
using namespace support;
@@ -517,84 +655,13 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) {
// Write the hash table.
uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj);
- // Write the MemProf profile data if we have it. This includes a simple schema
- // with the format described below followed by the hashtable:
- // uint64_t Version
- // uint64_t RecordTableOffset = RecordTableGenerator.Emit
- // uint64_t FramePayloadOffset = Stream offset before emitting the frame table
- // uint64_t FrameTableOffset = FrameTableGenerator.Emit
- // uint64_t Num schema entries
- // uint64_t Schema entry 0
- // uint64_t Schema entry 1
- // ....
- // uint64_t Schema entry N - 1
- // OnDiskChainedHashTable MemProfRecordData
- // OnDiskChainedHashTable MemProfFrameData
+ // Write the MemProf profile data if we have it.
uint64_t MemProfSectionStart = 0;
if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
- if (MemProfVersionRequested < memprof::MinimumSupportedVersion ||
- MemProfVersionRequested > memprof::MaximumSupportedVersion) {
- return make_error<InstrProfError>(
- instrprof_error::unsupported_version,
- formatv("MemProf version {} not supported; "
- "requires version between {} and {}, inclusive",
- MemProfVersionRequested, memprof::MinimumSupportedVersion,
- memprof::MaximumSupportedVersion));
- }
-
MemProfSectionStart = OS.tell();
-
- if (MemProfVersionRequested >= memprof::Version1)
- OS.write(MemProfVersionRequested);
-
- OS.write(0ULL); // Reserve space for the memprof record table offset.
- OS.write(0ULL); // Reserve space for the memprof frame payload offset.
- OS.write(0ULL); // Reserve space for the memprof frame table offset.
-
- auto Schema = memprof::PortableMemInfoBlock::getSchema();
- OS.write(static_cast<uint64_t>(Schema.size()));
- for (const auto Id : Schema) {
- OS.write(static_cast<uint64_t>(Id));
- }
-
- auto RecordWriter = std::make_unique<memprof::RecordWriterTrait>();
- RecordWriter->Schema = &Schema;
- OnDiskChainedHashTableGenerator<memprof::RecordWriterTrait>
- RecordTableGenerator;
- for (auto &I : MemProfRecordData) {
- // Insert the key (func hash) and value (memprof record).
- RecordTableGenerator.insert(I.first, I.second);
- }
- // Release the memory of this MapVector as it is no longer needed.
- MemProfRecordData.clear();
-
- // The call to Emit invokes RecordWriterTrait::EmitData which destructs
- // the memprof record copies owned by the RecordTableGenerator. This works
- // because the RecordTableGenerator is not used after this point.
- uint64_t RecordTableOffset =
- RecordTableGenerator.Emit(OS.OS, *RecordWriter);
-
- uint64_t FramePayloadOffset = OS.tell();
-
- auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>();
- OnDiskChainedHashTableGenerator<memprof::FrameWriterTrait>
- FrameTableGenerator;
- for (auto &I : MemProfFrameData) {
- // Insert the key (frame id) and value (frame contents).
- FrameTableGenerator.insert(I.first, I.second);
- }
- // Release the memory of this MapVector as it is no longer needed.
- MemProfFrameData.clear();
-
- uint64_t FrameTableOffset = FrameTableGenerator.Emit(OS.OS, *FrameWriter);
-
- uint64_t Header[] = {RecordTableOffset, FramePayloadOffset,
- FrameTableOffset};
- uint64_t HeaderUpdatePos = MemProfSectionStart;
- if (MemProfVersionRequested >= memprof::Version1)
- // The updates go just after the version field.
- HeaderUpdatePos += sizeof(uint64_t);
- OS.patch({{HeaderUpdatePos, Header, std::size(Header)}});
+ if (auto E = writeMemProf(OS, MemProfRecordData, MemProfFrameData,
+ MemProfVersionRequested))
+ return E;
}
// BinaryIdSection has two parts:
diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp
index 6c419811d59e..ac0a8702c3f9 100644
--- a/llvm/lib/ProfileData/MemProf.cpp
+++ b/llvm/lib/ProfileData/MemProf.cpp
@@ -10,15 +10,88 @@
namespace llvm {
namespace memprof {
+namespace {
+size_t serializedSizeV0(const IndexedAllocationInfo &IAI) {
+ size_t Size = 0;
+ // The number of frames to serialize.
+ Size += sizeof(uint64_t);
+ // The callstack frame ids.
+ Size += sizeof(FrameId) * IAI.CallStack.size();
+ // The size of the payload.
+ Size += PortableMemInfoBlock::serializedSize();
+ return Size;
+}
-void IndexedMemProfRecord::serialize(const MemProfSchema &Schema,
- raw_ostream &OS) {
+size_t serializedSizeV2(const IndexedAllocationInfo &IAI) {
+ size_t Size = 0;
+ // The CallStackId
+ Size += sizeof(CallStackId);
+ // The size of the payload.
+ Size += PortableMemInfoBlock::serializedSize();
+ return Size;
+}
+} // namespace
+
+size_t IndexedAllocationInfo::serializedSize(IndexedVersion Version) const {
+ switch (Version) {
+ case Version0:
+ case Version1:
+ return serializedSizeV0(*this);
+ case Version2:
+ return serializedSizeV2(*this);
+ }
+ llvm_unreachable("unsupported MemProf version");
+}
+
+namespace {
+size_t serializedSizeV0(const IndexedMemProfRecord &Record) {
+ size_t Result = sizeof(GlobalValue::GUID);
+ for (const IndexedAllocationInfo &N : Record.AllocSites)
+ Result += N.serializedSize(Version0);
+
+ // The number of callsites we have information for.
+ Result += sizeof(uint64_t);
+ for (const auto &Frames : Record.CallSites) {
+ // The number of frame ids to serialize.
+ Result += sizeof(uint64_t);
+ Result += Frames.size() * sizeof(FrameId);
+ }
+ return Result;
+}
+
+size_t serializedSizeV2(const IndexedMemProfRecord &Record) {
+ size_t Result = sizeof(GlobalValue::GUID);
+ for (const IndexedAllocationInfo &N : Record.AllocSites)
+ Result += N.serializedSize(Version2);
+
+ // The number of callsites we have information for.
+ Result += sizeof(uint64_t);
+ // The CallStackId
+ Result += Record.CallSiteIds.size() * sizeof(CallStackId);
+ return Result;
+}
+} // namespace
+
+size_t IndexedMemProfRecord::serializedSize(IndexedVersion Version) const {
+ switch (Version) {
+ case Version0:
+ case Version1:
+ return serializedSizeV0(*this);
+ case Version2:
+ return serializedSizeV2(*this);
+ }
+ llvm_unreachable("unsupported MemProf version");
+}
+
+namespace {
+void serializeV0(const IndexedMemProfRecord &Record,
+ const MemProfSchema &Schema, raw_ostream &OS) {
using namespace support;
endian::Writer LE(OS, llvm::endianness::little);
- LE.write<uint64_t>(AllocSites.size());
- for (const IndexedAllocationInfo &N : AllocSites) {
+ LE.write<uint64_t>(Record.AllocSites.size());
+ for (const IndexedAllocationInfo &N : Record.AllocSites) {
LE.write<uint64_t>(N.CallStack.size());
for (const FrameId &Id : N.CallStack)
LE.write<FrameId>(Id);
@@ -26,17 +99,50 @@ void IndexedMemProfRecord::serialize(const MemProfSchema &Schema,
}
// Related contexts.
- LE.write<uint64_t>(CallSites.size());
- for (const auto &Frames : CallSites) {
+ LE.write<uint64_t>(Record.CallSites.size());
+ for (const auto &Frames : Record.CallSites) {
LE.write<uint64_t>(Frames.size());
for (const FrameId &Id : Frames)
LE.write<FrameId>(Id);
}
}
-IndexedMemProfRecord
-IndexedMemProfRecord::deserialize(const MemProfSchema &Schema,
- const unsigned char *Ptr) {
+void serializeV2(const IndexedMemProfRecord &Record,
+ const MemProfSchema &Schema, raw_ostream &OS) {
+ using namespace support;
+
+ endian::Writer LE(OS, llvm::endianness::little);
+
+ LE.write<uint64_t>(Record.AllocSites.size());
+ for (const IndexedAllocationInfo &N : Record.AllocSites) {
+ LE.write<CallStackId>(N.CSId);
+ N.Info.serialize(Schema, OS);
+ }
+
+ // Related contexts.
+ LE.write<uint64_t>(Record.CallSiteIds.size());
+ for (const auto &CSId : Record.CallSiteIds)
+ LE.write<CallStackId>(CSId);
+}
+} // namespace
+
+void IndexedMemProfRecord::serialize(const MemProfSchema &Schema,
+ raw_ostream &OS, IndexedVersion Version) {
+ switch (Version) {
+ case Version0:
+ case Version1:
+ serializeV0(*this, Schema, OS);
+ return;
+ case Version2:
+ serializeV2(*this, Schema, OS);
+ return;
+ }
+ llvm_unreachable("unsupported MemProf version");
+}
+
+namespace {
+IndexedMemProfRecord deserializeV0(const MemProfSchema &Schema,
+ const unsigned char *Ptr) {
using namespace support;
IndexedMemProfRecord Record;
@@ -73,11 +179,57 @@ IndexedMemProfRecord::deserialize(const MemProfSchema &Schema,
Frames.push_back(Id);
}
Record.CallSites.push_back(Frames);
+ Record.CallSiteIds.push_back(hashCallStack(Frames));
}
return Record;
}
+IndexedMemProfRecord deserializeV2(const MemProfSchema &Schema,
+ const unsigned char *Ptr) {
+ using namespace support;
+
+ IndexedMemProfRecord Record;
+
+ // Read the meminfo nodes.
+ const uint64_t NumNodes =
+ endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
+ for (uint64_t I = 0; I < NumNodes; I++) {
+ IndexedAllocationInfo Node;
+ Node.CSId =
+ endian::readNext<CallStackId, llvm::endianness::little, unaligned>(Ptr);
+ Node.Info.deserialize(Schema, Ptr);
+ Ptr += PortableMemInfoBlock::serializedSize();
+ Record.AllocSites.push_back(Node);
+ }
+
+ // Read the callsite information.
+ const uint64_t NumCtxs =
+ endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Ptr);
+ for (uint64_t J = 0; J < NumCtxs; J++) {
+ CallStackId CSId =
+ endian::readNext<CallStackId, llvm::endianness::little, unaligned>(Ptr);
+ Record.CallSiteIds.push_back(CSId);
+ }
+
+ return Record;
+}
+} // namespace
+
+IndexedMemProfRecord
+IndexedMemProfRecord::deserialize(const MemProfSchema &Schema,
+ const unsigned char *Ptr,
+ IndexedVersion Version) {
+ switch (Version) {
+ case Version0:
+ case Version1:
+ return deserializeV0(Schema, Ptr);
+ case Version2:
+ return deserializeV2(Schema, Ptr);
+ }
+ llvm_unreachable("unsupported MemProf version");
+}
+
GlobalValue::GUID IndexedMemProfRecord::getGUID(const StringRef FunctionName) {
// Canonicalize the function name to drop suffixes such as ".llvm.". Note
// we do not drop any ".__uniq." suffixes, as getCanonicalFnName does not drop
generated by cgit v1.2.3 (git 2.25.1) at 2026-05-31 11:00:34 +0000