Revert "[clang][AST] Support AST files larger than 512M"

Bitcode file alignment is only 32-bit so 64-bit offsets need
special handling.
/b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/Serialization/ASTReader.cpp:6327:28: runtime error: load of misaligned address 0x7fca2bcfe54c for type 'const uint64_t' (aka 'const unsigned long'), which requires 8 byte alignment
0x7fca2bcfe54c: note: pointer points here
  00 00 00 00 5a a6 01 00  00 00 00 00 19 a7 01 00  00 00 00 00 48 a7 01 00  00 00 00 00 7d a7 01 00
              ^
    #0 0x3be2fe4 in clang::ASTReader::TypeCursorForIndex(unsigned int) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/Serialization/ASTReader.cpp:6327:28
    #1 0x3be30a0 in clang::ASTReader::readTypeRecord(unsigned int) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/Serialization/ASTReader.cpp:6348:24
    #2 0x3bd3d4a in clang::ASTReader::GetType(unsigned int) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/Serialization/ASTReader.cpp:6985:26
    #3 0x3c5d9ae in clang::ASTDeclReader::Visit(clang::Decl*) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp:533:31
    #4 0x3c91cac in clang::ASTReader::ReadDeclRecord(unsigned int) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/Serialization/ASTReaderDecl.cpp:4045:10
    #5 0x3bd4fb1 in clang::ASTReader::GetDecl(unsigned int) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/Serialization/ASTReader.cpp:7352:5
    #6 0x3bce2f9 in clang::ASTReader::ReadASTBlock(clang::serialization::ModuleFile&, unsigned int) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/Serialization/ASTReader.cpp:3625:22
    #7 0x3bd6d75 in clang::ASTReader::ReadAST(llvm::StringRef, clang::serialization::ModuleKind, clang::SourceLocation, unsigned int, llvm::SmallVectorImpl<clang::ASTReader::ImportedSubmodule>*) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/Serialization/ASTReader.cpp:4230:32
    #8 0x3a6b415 in clang::CompilerInstance::createPCHExternalASTSource(llvm::StringRef, llvm::StringRef, bool, bool, clang::Preprocessor&, clang::InMemoryModuleCache&, clang::ASTContext&, clang::PCHContainerReader const&, llvm::ArrayRef<std::shared_ptr<clang::ModuleFileExtension> >, llvm::ArrayRef<std::shared_ptr<clang::DependencyCollector> >, void*, bool, bool, bool) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/Frontend/CompilerInstance.cpp:539:19
    #9 0x3a6b00e in clang::CompilerInstance::createPCHExternalASTSource(llvm::StringRef, bool, bool, void*, bool) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/Frontend/CompilerInstance.cpp:501:18
    #10 0x3abac80 in clang::FrontendAction::BeginSourceFile(clang::CompilerInstance&, clang::FrontendInputFile const&) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/Frontend/FrontendAction.cpp:865:12
    #11 0x3a6e61c in clang::CompilerInstance::ExecuteAction(clang::FrontendAction&) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/Frontend/CompilerInstance.cpp:972:13
    #12 0x3ba74bf in clang::ExecuteCompilerInvocation(clang::CompilerInstance*) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/lib/FrontendTool/ExecuteCompilerInvocation.cpp:282:25
    #13 0xa3f753 in cc1_main(llvm::ArrayRef<char const*>, char const*, void*) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/tools/driver/cc1_main.cpp:240:15
    #14 0xa3a68a in ExecuteCC1Tool(llvm::SmallVectorImpl<char const*>&) /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/tools/driver/driver.cpp:330:12
    #15 0xa37f31 in main /b/sanitizer-x86_64-linux-fast/build/llvm-project/clang/tools/driver/driver.cpp:407:12
    #16 0x7fca2a7032e0 in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x202e0)
    #17 0xa21029 in _start (/b/sanitizer-x86_64-linux-fast/build/llvm_build_ubsan/bin/clang-11+0xa21029)

This reverts commit 30d5946db9.
This commit is contained in:
Dmitry Polukhin 2020-04-16 09:05:40 -07:00
parent dfcc403b2d
commit a8f85da9f5
8 changed files with 40 additions and 81 deletions

View File

@ -41,7 +41,7 @@ namespace serialization {
/// Version 4 of AST files also requires that the version control branch and /// Version 4 of AST files also requires that the version control branch and
/// revision match exactly, since there is no backward compatibility of /// revision match exactly, since there is no backward compatibility of
/// AST files at this time. /// AST files at this time.
const unsigned VERSION_MAJOR = 10; const unsigned VERSION_MAJOR = 9;
/// AST file minor version number supported by this version of /// AST file minor version number supported by this version of
/// Clang. /// Clang.
@ -181,7 +181,7 @@ namespace serialization {
/// Raw source location of end of range. /// Raw source location of end of range.
unsigned End; unsigned End;
/// Offset in the AST file relative to ModuleFile::MacroOffsetsBase. /// Offset in the AST file.
uint32_t BitOffset; uint32_t BitOffset;
PPEntityOffset(SourceRange R, uint32_t BitOffset) PPEntityOffset(SourceRange R, uint32_t BitOffset)
@ -221,18 +221,12 @@ namespace serialization {
/// Raw source location. /// Raw source location.
unsigned Loc = 0; unsigned Loc = 0;
/// Offset in the AST file. Split 64-bit integer into low/high parts /// Offset in the AST file.
/// to keep structure alignment 32-bit and don't have padding gap. uint32_t BitOffset = 0;
/// This structure is serialized "as is" to the AST file and undefined
/// value in the padding affects AST hash.
uint32_t BitOffsetLow = 0;
uint32_t BitOffsetHigh = 0;
DeclOffset() = default; DeclOffset() = default;
DeclOffset(SourceLocation Loc, uint64_t BitOffset) { DeclOffset(SourceLocation Loc, uint32_t BitOffset)
setLocation(Loc); : Loc(Loc.getRawEncoding()), BitOffset(BitOffset) {}
setBitOffset(BitOffset);
}
void setLocation(SourceLocation L) { void setLocation(SourceLocation L) {
Loc = L.getRawEncoding(); Loc = L.getRawEncoding();
@ -241,15 +235,6 @@ namespace serialization {
SourceLocation getLocation() const { SourceLocation getLocation() const {
return SourceLocation::getFromRawEncoding(Loc); return SourceLocation::getFromRawEncoding(Loc);
} }
void setBitOffset(uint64_t Offset) {
BitOffsetLow = Offset;
BitOffsetHigh = Offset >> 32;
}
uint64_t getBitOffset() const {
return BitOffsetLow | (uint64_t(BitOffsetHigh) << 32);
}
}; };
/// The number of predefined preprocessed entity IDs. /// The number of predefined preprocessed entity IDs.

View File

@ -723,10 +723,9 @@ private:
struct PendingMacroInfo { struct PendingMacroInfo {
ModuleFile *M; ModuleFile *M;
/// Offset relative to ModuleFile::MacroOffsetsBase. uint64_t MacroDirectivesOffset;
uint32_t MacroDirectivesOffset;
PendingMacroInfo(ModuleFile *M, uint32_t MacroDirectivesOffset) PendingMacroInfo(ModuleFile *M, uint64_t MacroDirectivesOffset)
: M(M), MacroDirectivesOffset(MacroDirectivesOffset) {} : M(M), MacroDirectivesOffset(MacroDirectivesOffset) {}
}; };
@ -2206,7 +2205,7 @@ public:
/// \param MacroDirectivesOffset Offset of the serialized macro directive /// \param MacroDirectivesOffset Offset of the serialized macro directive
/// history. /// history.
void addPendingMacro(IdentifierInfo *II, ModuleFile *M, void addPendingMacro(IdentifierInfo *II, ModuleFile *M,
uint32_t MacroDirectivesOffset); uint64_t MacroDirectivesOffset);
/// Read the set of macros defined by this external macro source. /// Read the set of macros defined by this external macro source.
void ReadDefinedMacros() override; void ReadDefinedMacros() override;

View File

@ -243,7 +243,7 @@ private:
/// Offset of each type in the bitstream, indexed by /// Offset of each type in the bitstream, indexed by
/// the type's ID. /// the type's ID.
std::vector<uint64_t> TypeOffsets; std::vector<uint32_t> TypeOffsets;
/// The first ID number we can use for our own identifiers. /// The first ID number we can use for our own identifiers.
serialization::IdentID FirstIdentID = serialization::NUM_PREDEF_IDENT_IDS; serialization::IdentID FirstIdentID = serialization::NUM_PREDEF_IDENT_IDS;
@ -277,8 +277,7 @@ private:
/// The macro infos to emit. /// The macro infos to emit.
std::vector<MacroInfoToEmitData> MacroInfosToEmit; std::vector<MacroInfoToEmitData> MacroInfosToEmit;
llvm::DenseMap<const IdentifierInfo *, uint32_t> llvm::DenseMap<const IdentifierInfo *, uint64_t> IdentMacroDirectivesOffsetMap;
IdentMacroDirectivesOffsetMap;
/// @name FlushStmt Caches /// @name FlushStmt Caches
/// @{ /// @{
@ -465,8 +464,7 @@ private:
const Preprocessor &PP); const Preprocessor &PP);
void WritePreprocessor(const Preprocessor &PP, bool IsModule); void WritePreprocessor(const Preprocessor &PP, bool IsModule);
void WriteHeaderSearch(const HeaderSearch &HS); void WriteHeaderSearch(const HeaderSearch &HS);
void WritePreprocessorDetail(PreprocessingRecord &PPRec, void WritePreprocessorDetail(PreprocessingRecord &PPRec);
uint64_t MacroOffsetsBase);
void WriteSubmodules(Module *WritingModule); void WriteSubmodules(Module *WritingModule);
void WritePragmaDiagnosticMappings(const DiagnosticsEngine &Diag, void WritePragmaDiagnosticMappings(const DiagnosticsEngine &Diag,
@ -590,7 +588,7 @@ public:
/// Determine the ID of an already-emitted macro. /// Determine the ID of an already-emitted macro.
serialization::MacroID getMacroID(MacroInfo *MI); serialization::MacroID getMacroID(MacroInfo *MI);
uint32_t getMacroDirectivesOffset(const IdentifierInfo *Name); uint64_t getMacroDirectivesOffset(const IdentifierInfo *Name);
/// Emit a reference to a type. /// Emit a reference to a type.
void AddTypeRef(QualType T, RecordDataImpl &Record); void AddTypeRef(QualType T, RecordDataImpl &Record);

View File

@ -251,10 +251,6 @@ public:
/// The base offset in the source manager's view of this module. /// The base offset in the source manager's view of this module.
unsigned SLocEntryBaseOffset = 0; unsigned SLocEntryBaseOffset = 0;
/// Base file offset for the offsets in SLocEntryOffsets. Real file offset
/// for the entry is SLocEntryOffsetsBase + SLocEntryOffsets[i].
uint64_t SLocEntryOffsetsBase = 0;
/// Offsets for all of the source location entries in the /// Offsets for all of the source location entries in the
/// AST file. /// AST file.
const uint32_t *SLocEntryOffsets = nullptr; const uint32_t *SLocEntryOffsets = nullptr;
@ -306,10 +302,6 @@ public:
/// The number of macros in this AST file. /// The number of macros in this AST file.
unsigned LocalNumMacros = 0; unsigned LocalNumMacros = 0;
/// Base file offset for the offsets in MacroOffsets. Real file offset for
/// the entry is MacroOffsetsBase + MacroOffsets[i].
uint64_t MacroOffsetsBase = 0;
/// Offsets of macros in the preprocessor block. /// Offsets of macros in the preprocessor block.
/// ///
/// This array is indexed by the macro ID (-1), and provides /// This array is indexed by the macro ID (-1), and provides
@ -458,7 +450,7 @@ public:
/// Offset of each type within the bitstream, indexed by the /// Offset of each type within the bitstream, indexed by the
/// type ID, or the representation of a Type*. /// type ID, or the representation of a Type*.
const uint64_t *TypeOffsets = nullptr; const uint32_t *TypeOffsets = nullptr;
/// Base type ID for types local to this module as represented in /// Base type ID for types local to this module as represented in
/// the global type ID space. /// the global type ID space.

View File

@ -1470,7 +1470,6 @@ bool ASTReader::ReadSLocEntry(int ID) {
ModuleFile *F = GlobalSLocEntryMap.find(-ID)->second; ModuleFile *F = GlobalSLocEntryMap.find(-ID)->second;
if (llvm::Error Err = F->SLocEntryCursor.JumpToBit( if (llvm::Error Err = F->SLocEntryCursor.JumpToBit(
F->SLocEntryOffsetsBase +
F->SLocEntryOffsets[ID - F->SLocEntryBaseID])) { F->SLocEntryOffsets[ID - F->SLocEntryBaseID])) {
Error(std::move(Err)); Error(std::move(Err));
return true; return true;
@ -1933,8 +1932,9 @@ HeaderFileInfoTrait::ReadData(internal_key_ref key, const unsigned char *d,
return HFI; return HFI;
} }
void ASTReader::addPendingMacro(IdentifierInfo *II, ModuleFile *M, void ASTReader::addPendingMacro(IdentifierInfo *II,
uint32_t MacroDirectivesOffset) { ModuleFile *M,
uint64_t MacroDirectivesOffset) {
assert(NumCurrentElementsDeserializing > 0 &&"Missing deserialization guard"); assert(NumCurrentElementsDeserializing > 0 &&"Missing deserialization guard");
PendingMacroIDs[II].push_back(PendingMacroInfo(M, MacroDirectivesOffset)); PendingMacroIDs[II].push_back(PendingMacroInfo(M, MacroDirectivesOffset));
} }
@ -2099,8 +2099,7 @@ void ASTReader::resolvePendingMacro(IdentifierInfo *II,
BitstreamCursor &Cursor = M.MacroCursor; BitstreamCursor &Cursor = M.MacroCursor;
SavedStreamPosition SavedPosition(Cursor); SavedStreamPosition SavedPosition(Cursor);
if (llvm::Error Err = if (llvm::Error Err = Cursor.JumpToBit(PMInfo.MacroDirectivesOffset)) {
Cursor.JumpToBit(M.MacroOffsetsBase + PMInfo.MacroDirectivesOffset)) {
Error(std::move(Err)); Error(std::move(Err));
return; return;
} }
@ -3099,7 +3098,7 @@ ASTReader::ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
Error("duplicate TYPE_OFFSET record in AST file"); Error("duplicate TYPE_OFFSET record in AST file");
return Failure; return Failure;
} }
F.TypeOffsets = reinterpret_cast<const uint64_t *>(Blob.data()); F.TypeOffsets = (const uint32_t *)Blob.data();
F.LocalNumTypes = Record[0]; F.LocalNumTypes = Record[0];
unsigned LocalBaseTypeIndex = Record[1]; unsigned LocalBaseTypeIndex = Record[1];
F.BaseTypeIndex = getTotalNumTypes(); F.BaseTypeIndex = getTotalNumTypes();
@ -3377,7 +3376,6 @@ ASTReader::ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
F.SLocEntryOffsets = (const uint32_t *)Blob.data(); F.SLocEntryOffsets = (const uint32_t *)Blob.data();
F.LocalNumSLocEntries = Record[0]; F.LocalNumSLocEntries = Record[0];
unsigned SLocSpaceSize = Record[1]; unsigned SLocSpaceSize = Record[1];
F.SLocEntryOffsetsBase = Record[2];
std::tie(F.SLocEntryBaseID, F.SLocEntryBaseOffset) = std::tie(F.SLocEntryBaseID, F.SLocEntryBaseOffset) =
SourceMgr.AllocateLoadedSLocEntries(F.LocalNumSLocEntries, SourceMgr.AllocateLoadedSLocEntries(F.LocalNumSLocEntries,
SLocSpaceSize); SLocSpaceSize);
@ -3696,7 +3694,6 @@ ASTReader::ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
F.MacroOffsets = (const uint32_t *)Blob.data(); F.MacroOffsets = (const uint32_t *)Blob.data();
F.LocalNumMacros = Record[0]; F.LocalNumMacros = Record[0];
unsigned LocalBaseMacroID = Record[1]; unsigned LocalBaseMacroID = Record[1];
F.MacroOffsetsBase = Record[2];
F.BaseMacroID = getTotalNumMacros(); F.BaseMacroID = getTotalNumMacros();
if (F.LocalNumMacros > 0) { if (F.LocalNumMacros > 0) {
@ -5910,8 +5907,8 @@ PreprocessedEntity *ASTReader::ReadPreprocessedEntity(unsigned Index) {
} }
SavedStreamPosition SavedPosition(M.PreprocessorDetailCursor); SavedStreamPosition SavedPosition(M.PreprocessorDetailCursor);
if (llvm::Error Err = M.PreprocessorDetailCursor.JumpToBit( if (llvm::Error Err =
M.MacroOffsetsBase + PPOffs.BitOffset)) { M.PreprocessorDetailCursor.JumpToBit(PPOffs.BitOffset)) {
Error(std::move(Err)); Error(std::move(Err));
return nullptr; return nullptr;
} }
@ -8430,8 +8427,7 @@ MacroInfo *ASTReader::getMacro(MacroID ID) {
assert(I != GlobalMacroMap.end() && "Corrupted global macro map"); assert(I != GlobalMacroMap.end() && "Corrupted global macro map");
ModuleFile *M = I->second; ModuleFile *M = I->second;
unsigned Index = ID - M->BaseMacroID; unsigned Index = ID - M->BaseMacroID;
MacrosLoaded[ID] = MacrosLoaded[ID] = ReadMacroRecord(*M, M->MacroOffsets[Index]);
ReadMacroRecord(*M, M->MacroOffsetsBase + M->MacroOffsets[Index]);
if (DeserializationListener) if (DeserializationListener)
DeserializationListener->MacroRead(ID + NUM_PREDEF_MACRO_IDS, DeserializationListener->MacroRead(ID + NUM_PREDEF_MACRO_IDS,

View File

@ -2870,7 +2870,7 @@ ASTReader::DeclCursorForID(DeclID ID, SourceLocation &Loc) {
const DeclOffset &DOffs = const DeclOffset &DOffs =
M->DeclOffsets[ID - M->BaseDeclID - NUM_PREDEF_DECL_IDS]; M->DeclOffsets[ID - M->BaseDeclID - NUM_PREDEF_DECL_IDS];
Loc = TranslateSourceLocation(*M, DOffs.getLocation()); Loc = TranslateSourceLocation(*M, DOffs.getLocation());
return RecordLocation(M, DOffs.getBitOffset()); return RecordLocation(M, DOffs.BitOffset);
} }
ASTReader::RecordLocation ASTReader::getLocalBitOffset(uint64_t GlobalOffset) { ASTReader::RecordLocation ASTReader::getLocalBitOffset(uint64_t GlobalOffset) {

View File

@ -1893,7 +1893,6 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
// Write out the source location entry table. We skip the first // Write out the source location entry table. We skip the first
// entry, which is always the same dummy entry. // entry, which is always the same dummy entry.
std::vector<uint32_t> SLocEntryOffsets; std::vector<uint32_t> SLocEntryOffsets;
uint64_t SLocEntryOffsetsBase = Stream.GetCurrentBitNo();
RecordData PreloadSLocs; RecordData PreloadSLocs;
SLocEntryOffsets.reserve(SourceMgr.local_sloc_entry_size() - 1); SLocEntryOffsets.reserve(SourceMgr.local_sloc_entry_size() - 1);
for (unsigned I = 1, N = SourceMgr.local_sloc_entry_size(); for (unsigned I = 1, N = SourceMgr.local_sloc_entry_size();
@ -1904,9 +1903,7 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
assert(&SourceMgr.getSLocEntry(FID) == SLoc); assert(&SourceMgr.getSLocEntry(FID) == SLoc);
// Record the offset of this source-location entry. // Record the offset of this source-location entry.
uint64_t Offset = Stream.GetCurrentBitNo() - SLocEntryOffsetsBase; SLocEntryOffsets.push_back(Stream.GetCurrentBitNo());
assert((Offset >> 32) == 0 && "SLocEntry offset too large");
SLocEntryOffsets.push_back(Offset);
// Figure out which record code to use. // Figure out which record code to use.
unsigned Code; unsigned Code;
@ -2014,14 +2011,12 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
Abbrev->Add(BitCodeAbbrevOp(SOURCE_LOCATION_OFFSETS)); Abbrev->Add(BitCodeAbbrevOp(SOURCE_LOCATION_OFFSETS));
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // # of slocs Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // # of slocs
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // total size Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // total size
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 32)); // base offset
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // offsets Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // offsets
unsigned SLocOffsetsAbbrev = Stream.EmitAbbrev(std::move(Abbrev)); unsigned SLocOffsetsAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
{ {
RecordData::value_type Record[] = { RecordData::value_type Record[] = {
SOURCE_LOCATION_OFFSETS, SLocEntryOffsets.size(), SOURCE_LOCATION_OFFSETS, SLocEntryOffsets.size(),
SourceMgr.getNextLocalOffset() - 1 /* skip dummy */, SourceMgr.getNextLocalOffset() - 1 /* skip dummy */};
SLocEntryOffsetsBase};
Stream.EmitRecordWithBlob(SLocOffsetsAbbrev, Record, Stream.EmitRecordWithBlob(SLocOffsetsAbbrev, Record,
bytes(SLocEntryOffsets)); bytes(SLocEntryOffsets));
} }
@ -2098,11 +2093,9 @@ static bool shouldIgnoreMacro(MacroDirective *MD, bool IsModule,
/// Writes the block containing the serialized form of the /// Writes the block containing the serialized form of the
/// preprocessor. /// preprocessor.
void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) { void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) {
uint64_t MacroOffsetsBase = Stream.GetCurrentBitNo();
PreprocessingRecord *PPRec = PP.getPreprocessingRecord(); PreprocessingRecord *PPRec = PP.getPreprocessingRecord();
if (PPRec) if (PPRec)
WritePreprocessorDetail(*PPRec, MacroOffsetsBase); WritePreprocessorDetail(*PPRec);
RecordData Record; RecordData Record;
RecordData ModuleMacroRecord; RecordData ModuleMacroRecord;
@ -2163,8 +2156,7 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) {
// identifier they belong to. // identifier they belong to.
for (const IdentifierInfo *Name : MacroIdentifiers) { for (const IdentifierInfo *Name : MacroIdentifiers) {
MacroDirective *MD = PP.getLocalMacroDirectiveHistory(Name); MacroDirective *MD = PP.getLocalMacroDirectiveHistory(Name);
uint64_t StartOffset = Stream.GetCurrentBitNo() - MacroOffsetsBase; auto StartOffset = Stream.GetCurrentBitNo();
assert((StartOffset >> 32) == 0 && "Macro identifiers offset too large");
// Emit the macro directives in reverse source order. // Emit the macro directives in reverse source order.
for (; MD; MD = MD->getPrevious()) { for (; MD; MD = MD->getPrevious()) {
@ -2237,12 +2229,14 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) {
// Record the local offset of this macro. // Record the local offset of this macro.
unsigned Index = ID - FirstMacroID; unsigned Index = ID - FirstMacroID;
if (Index >= MacroOffsets.size()) if (Index == MacroOffsets.size())
MacroOffsets.resize(Index + 1); MacroOffsets.push_back(Stream.GetCurrentBitNo());
else {
if (Index > MacroOffsets.size())
MacroOffsets.resize(Index + 1);
uint64_t Offset = Stream.GetCurrentBitNo() - MacroOffsetsBase; MacroOffsets[Index] = Stream.GetCurrentBitNo();
assert((Offset >> 32) == 0 && "Macro offset too large"); }
MacroOffsets[Index] = Offset;
AddIdentifierRef(Name, Record); AddIdentifierRef(Name, Record);
AddSourceLocation(MI->getDefinitionLoc(), Record); AddSourceLocation(MI->getDefinitionLoc(), Record);
@ -2293,20 +2287,17 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) {
Abbrev->Add(BitCodeAbbrevOp(MACRO_OFFSET)); Abbrev->Add(BitCodeAbbrevOp(MACRO_OFFSET));
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of macros Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of macros
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // first ID Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // first ID
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 32)); // base offset
Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
unsigned MacroOffsetAbbrev = Stream.EmitAbbrev(std::move(Abbrev)); unsigned MacroOffsetAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
{ {
RecordData::value_type Record[] = {MACRO_OFFSET, MacroOffsets.size(), RecordData::value_type Record[] = {MACRO_OFFSET, MacroOffsets.size(),
FirstMacroID - NUM_PREDEF_MACRO_IDS, FirstMacroID - NUM_PREDEF_MACRO_IDS};
MacroOffsetsBase};
Stream.EmitRecordWithBlob(MacroOffsetAbbrev, Record, bytes(MacroOffsets)); Stream.EmitRecordWithBlob(MacroOffsetAbbrev, Record, bytes(MacroOffsets));
} }
} }
void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec, void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) {
uint64_t MacroOffsetsBase) {
if (PPRec.local_begin() == PPRec.local_end()) if (PPRec.local_begin() == PPRec.local_end())
return; return;
@ -2343,10 +2334,8 @@ void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec,
(void)++E, ++NumPreprocessingRecords, ++NextPreprocessorEntityID) { (void)++E, ++NumPreprocessingRecords, ++NextPreprocessorEntityID) {
Record.clear(); Record.clear();
uint64_t Offset = Stream.GetCurrentBitNo() - MacroOffsetsBase;
assert((Offset >> 32) == 0 && "Preprocessed entity offset too large");
PreprocessedEntityOffsets.push_back( PreprocessedEntityOffsets.push_back(
PPEntityOffset((*E)->getSourceRange(), Offset)); PPEntityOffset((*E)->getSourceRange(), Stream.GetCurrentBitNo()));
if (auto *MD = dyn_cast<MacroDefinitionRecord>(*E)) { if (auto *MD = dyn_cast<MacroDefinitionRecord>(*E)) {
// Record this macro definition's ID. // Record this macro definition's ID.
@ -5155,7 +5144,7 @@ MacroID ASTWriter::getMacroID(MacroInfo *MI) {
return MacroIDs[MI]; return MacroIDs[MI];
} }
uint32_t ASTWriter::getMacroDirectivesOffset(const IdentifierInfo *Name) { uint64_t ASTWriter::getMacroDirectivesOffset(const IdentifierInfo *Name) {
return IdentMacroDirectivesOffsetMap.lookup(Name); return IdentMacroDirectivesOffsetMap.lookup(Name);
} }

View File

@ -2434,12 +2434,12 @@ void ASTWriter::WriteDecl(ASTContext &Context, Decl *D) {
SourceLocation Loc = D->getLocation(); SourceLocation Loc = D->getLocation();
unsigned Index = ID - FirstDeclID; unsigned Index = ID - FirstDeclID;
if (DeclOffsets.size() == Index) if (DeclOffsets.size() == Index)
DeclOffsets.emplace_back(Loc, Offset); DeclOffsets.push_back(DeclOffset(Loc, Offset));
else if (DeclOffsets.size() < Index) { else if (DeclOffsets.size() < Index) {
// FIXME: Can/should this happen? // FIXME: Can/should this happen?
DeclOffsets.resize(Index+1); DeclOffsets.resize(Index+1);
DeclOffsets[Index].setLocation(Loc); DeclOffsets[Index].setLocation(Loc);
DeclOffsets[Index].setBitOffset(Offset); DeclOffsets[Index].BitOffset = Offset;
} else { } else {
llvm_unreachable("declarations should be emitted in ID order"); llvm_unreachable("declarations should be emitted in ID order");
} }