OSchip
/
llvm-project
mirror of https://github.com/THU-DSP-LAB/llvm-project.git
13
0
Fork 2
Code Issues Packages Projects Releases Wiki Activity

[llvm-objcopy] Support --{,de}compress-debug-sections for zstd 

Also, add ELFCOMPRESS_ZSTD (2) from the approved generic-abi proposal:
https://groups.google.com/g/generic-abi/c/satyPkuMisk
("Add new ch_type value: ELFCOMPRESS_ZSTD")

Link: https://discourse.llvm.org/t/rfc-zstandard-as-a-second-compression-method-to-llvm/63399
("[RFC] Zstandard as a second compression method to LLVM")

Reviewed By: jhenderson, dblaikie

Differential Revision: https://reviews.llvm.org/D130458
This commit is contained in:
Fangrui Song 2022-09-08 00:59:14 -07:00
parent a41977dd0f
commit b6e1fd761d
12 changed files with 149 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
llvm/docs/CommandGuide/llvm-objcopy.rst
View File

@ -299,7 +299,7 @@ them.
.. option:: --compress-debug-sections [<format>]
Compress DWARF debug sections in the output, using the specified format.
Supported formats are ``zlib``. Use ``zlib`` if ``<format>`` is omitted.
Supported formats are ``zlib`` and ``zstd``. Use ``zlib`` if ``<format>`` is omitted.
.. option:: --decompress-debug-sections

1
llvm/include/llvm/BinaryFormat/ELF.h
View File

@ -1800,6 +1800,7 @@ struct Elf64_Nhdr {
// Legal values for ch_type field of compressed section header.
enum {
ELFCOMPRESS_ZLIB = 1, // ZLIB/DEFLATE algorithm.
ELFCOMPRESS_ZSTD = 2, // Zstandard algorithm
ELFCOMPRESS_LOOS = 0x60000000, // Start of OS-specific.
ELFCOMPRESS_HIOS = 0x6fffffff, // End of OS-specific.
ELFCOMPRESS_LOPROC = 0x70000000, // Start of processor-specific.

40
llvm/lib/ObjCopy/ELF/ELFObject.cpp
View File

@ -438,14 +438,29 @@ template <class ELFT>
Error ELFSectionWriter<ELFT>::visit(const DecompressedSection &Sec) {
ArrayRef<uint8_t> Compressed =
Sec.OriginalData.slice(sizeof(Elf_Chdr_Impl<ELFT>));
SmallVector<uint8_t, 128> DecompressedContent;
if (Error Err = compression::zlib::uncompress(Compressed, DecompressedContent,
static_cast<size_t>(Sec.Size)))
SmallVector<uint8_t, 128> Decompressed;
DebugCompressionType Type;
switch (Sec.ChType) {
case ELFCOMPRESS_ZLIB:
Type = DebugCompressionType::Z;
break;
case ELFCOMPRESS_ZSTD:
Type = DebugCompressionType::Zstd;
break;
default:
return createStringError(errc::invalid_argument,
"'" + Sec.Name + "': " + toString(std::move(Err)));
"--decompress-debug-sections: ch_type (" +
Twine(Sec.ChType) + ") of section '" +
Sec.Name + "' is unsupported");
}
if (Error E = compression::decompress(Type, Compressed, Decompressed,
static_cast<size_t>(Sec.Size)))
return createStringError(errc::invalid_argument,
"failed to decompress section '" + Sec.Name +
"': " + toString(std::move(E)));
uint8_t *Buf = reinterpret_cast<uint8_t *>(Out.getBufferStart()) + Sec.Offset;
std::copy(DecompressedContent.begin(), DecompressedContent.end(), Buf);
std::copy(Decompressed.begin(), Decompressed.end(), Buf);
return Error::success();
}
@ -498,6 +513,9 @@ Error ELFSectionWriter<ELFT>::visit(const CompressedSection &Sec) {
case DebugCompressionType::Z:
Chdr.ch_type = ELF::ELFCOMPRESS_ZLIB;
break;
case DebugCompressionType::Zstd:
Chdr.ch_type = ELF::ELFCOMPRESS_ZSTD;
break;
}
Chdr.ch_size = Sec.DecompressedSize;
Chdr.ch_addralign = Sec.DecompressedAlign;
@ -512,9 +530,9 @@ CompressedSection::CompressedSection(const SectionBase &Sec,
DebugCompressionType CompressionType)
: SectionBase(Sec), CompressionType(CompressionType),
DecompressedSize(Sec.OriginalData.size()), DecompressedAlign(Sec.Align) {
compression::zlib::compress(OriginalData, CompressedData);
compression::compress(compression::Params(CompressionType), OriginalData,
CompressedData);
assert(CompressionType != DebugCompressionType::None);
Flags |= ELF::SHF_COMPRESSED;
size_t ChdrSize =
std::max(std::max(sizeof(object::Elf_Chdr_Impl<object::ELF64LE>),
@ -526,9 +544,9 @@ CompressedSection::CompressedSection(const SectionBase &Sec,
}
CompressedSection::CompressedSection(ArrayRef<uint8_t> CompressedData,
uint64_t DecompressedSize,
uint32_t ChType, uint64_t DecompressedSize,
uint64_t DecompressedAlign)
: CompressionType(DebugCompressionType::None),
: ChType(ChType), CompressionType(DebugCompressionType::None),
DecompressedSize(DecompressedSize), DecompressedAlign(DecompressedAlign) {
OriginalData = CompressedData;
}
@ -1706,8 +1724,8 @@ Expected<SectionBase &> ELFBuilder<ELFT>::makeSection(const Elf_Shdr &Shdr) {
if (!(Shdr.sh_flags & ELF::SHF_COMPRESSED))
return Obj.addSection<Section>(*Data);
auto *Chdr = reinterpret_cast<const Elf_Chdr_Impl<ELFT> *>(Data->data());
return Obj.addSection<CompressedSection>(
CompressedSection(*Data, Chdr->ch_size, Chdr->ch_addralign));
return Obj.addSection<CompressedSection>(CompressedSection(
*Data, Chdr->ch_type, Chdr->ch_size, Chdr->ch_addralign));
}
}
}

9
llvm/lib/ObjCopy/ELF/ELFObject.h
View File

@ -536,6 +536,7 @@ public:
class CompressedSection : public SectionBase {
MAKE_SEC_WRITER_FRIEND
uint32_t ChType = 0;
DebugCompressionType CompressionType;
uint64_t DecompressedSize;
uint64_t DecompressedAlign;
@ -544,11 +545,12 @@ class CompressedSection : public SectionBase {
public:
CompressedSection(const SectionBase &Sec,
DebugCompressionType CompressionType);
CompressedSection(ArrayRef<uint8_t> CompressedData, uint64_t DecompressedSize,
uint64_t DecompressedAlign);
CompressedSection(ArrayRef<uint8_t> CompressedData, uint32_t ChType,
uint64_t DecompressedSize, uint64_t DecompressedAlign);
uint64_t getDecompressedSize() const { return DecompressedSize; }
uint64_t getDecompressedAlign() const { return DecompressedAlign; }
uint64_t getChType() const { return ChType; }
Error accept(SectionVisitor &Visitor) const override;
Error accept(MutableSectionVisitor &Visitor) override;
@ -562,8 +564,9 @@ class DecompressedSection : public SectionBase {
MAKE_SEC_WRITER_FRIEND
public:
uint32_t ChType;
explicit DecompressedSection(const CompressedSection &Sec)
: SectionBase(Sec) {
: SectionBase(Sec), ChType(Sec.getChType()) {
Size = Sec.getDecompressedSize();
Align = Sec.getDecompressedAlign();
Flags = OriginalFlags = (Flags & ~ELF::SHF_COMPRESSED);

5
llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-default.test
View File

@ -2,12 +2,15 @@
# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t.o
# RUN: llvm-objcopy --compress-debug-sections %t.o %t-compressed.o
# RUN: llvm-readobj -S %t-compressed.o | FileCheck %s
# RUN: llvm-readobj -S --sd %t-compressed.o | FileCheck %s
# CHECK: Name: .debug_foo
# CHECK-NEXT: Type: SHT_PROGBITS
# CHECK-NEXT: Flags [
# CHECK-NEXT: SHF_COMPRESSED
# CHECK-NEXT: ]
# CHECK: SectionData (
## ch_type = ELFCOMPRESS_ZLIB (1)
# CHECK-NEXT: 0000: 01000000 {{.*}}
# CHECK-NOT: Name: .debug_foo

18
llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test
View File

@ -28,3 +28,21 @@
## --compress-debug-sections does not update a compressed section.
# RUN: llvm-objcopy --compress-debug-sections=zlib %t-zlib.o %t-zlib-zlib.o
# RUN: cmp %t-zlib.o %t-zlib-zlib.o
# RUN: yaml2obj %s -o %t-corrupted
# RUN: not llvm-objcopy --decompress-debug-sections %t-corrupted /dev/null 2>&1 | FileCheck %s -DFILE=%t-corrupted --check-prefix=ERR
# ERR: error: '[[FILE]]': failed to decompress section '.debug_info': zlib error: Z_DATA_ERROR
--- !ELF
FileHeader:
Class: ELFCLASS64
Data: ELFDATA2LSB
Type: ET_REL
Machine: EM_X86_64
Sections:
- Type: SHT_PROGBITS
Name: .debug_info
Flags: [ SHF_COMPRESSED ]
AddressAlign: 8
Content: "010000000000000004000000000000000100000000000000ffffffff"

5
llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd-err.test Normal file
View File

@ -0,0 +1,5 @@
# UNSUPPORTED: zstd
# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t
# RUN: not llvm-objcopy --compress-debug-sections=zstd %t /dev/null 2>&1 | FileCheck %s
# CHECK: error: LLVM was not built with LLVM_ENABLE_ZSTD or did not find zstd at build time

51
llvm/test/tools/llvm-objcopy/ELF/compress-debug-sections-zstd.test Normal file
View File

@ -0,0 +1,51 @@
# REQUIRES: zstd
## Test --compress-debug-sections=zstd and decompression.
# RUN: yaml2obj %p/Inputs/compress-debug-sections.yaml -o %t && llvm-objcopy %t
# RUN: llvm-objcopy --compress-debug-sections=zstd %t %t-zstd
# RUN: llvm-objcopy --decompress-debug-sections %t-zstd %t-de
# RUN: cmp %t %t-de
# RUN: llvm-readelf -S -r -x .debug_foo %t-zstd | FileCheck %s --check-prefixes=CHECK,COMPRESSED
# RUN: llvm-readelf -S -r -x .debug_foo %t-de | FileCheck %s --check-prefixes=CHECK,DECOMPRESSED
# CHECK: Name Type Address Off Size ES Flg Lk Inf Al
# COMPRESSED: .debug_foo PROGBITS 0000000000000000 000040 {{.*}} 00 C 0 0 8
# COMPRESSED-NEXT: .notdebug_foo PROGBITS 0000000000000000 {{.*}} 000008 00 0 0 0
# DECOMPRESSED: .debug_foo PROGBITS 0000000000000000 000040 000008 00 0 0 0
# DECOMPRESSED-NEXT: .notdebug_foo PROGBITS 0000000000000000 {{.*}} 000008 00 0 0 0
## Relocations do not change.
# CHECK: Relocation section '.rela.debug_foo' at offset {{.*}} contains 2 entries:
# CHECK-NEXT: Offset
# CHECK-NEXT: 0000000000000001 000000010000000a R_X86_64_32 0000000000000000 .debug_foo + 0
# CHECK-NEXT: 0000000000000002 000000020000000a R_X86_64_32 0000000000000000 .notdebug_foo + 0
# COMPRESSED: Hex dump of section '.debug_foo':
## ch_type == ELFCOMPRESS_ZSTD (2)
# COMPRESSED-NEXT: 0x00000000 02000000 00000000 08000000 00000000
# COMPRESSED-NEXT: 0x00000010 00000000 00000000 {{.*}}
## --compress-debug-sections does not update a compressed section. Its compression
## type does not change.
# RUN: llvm-objcopy --compress-debug-sections=zstd %t-zstd %t-zstd-zstd
# RUN: cmp %t-zstd %t-zstd-zstd
# RUN: %if zlib %{ llvm-objcopy --compress-debug-sections=zlib %t-zstd %t-zstd-zlib && cmp %t-zstd %t-zstd-zlib %}
# RUN: yaml2obj %s -o %t-corrupted
# RUN: not llvm-objcopy --decompress-debug-sections %t-corrupted /dev/null 2>&1 | FileCheck %s -DFILE=%t-corrupted --check-prefix=ERR
# ERR: error: '[[FILE]]': failed to decompress section '.debug_info': Src size is incorrect
--- !ELF
FileHeader:
Class: ELFCLASS64
Data: ELFDATA2LSB
Type: ET_REL
Machine: EM_X86_64
Sections:
- Type: SHT_PROGBITS
Name: .debug_info
Flags: [ SHF_COMPRESSED ]
AddressAlign: 8
Content: "020000000000000004000000000000000100000000000000ffffffff"

24
llvm/test/tools/llvm-objcopy/ELF/decompress-debug-sections-err.test Normal file
View File

@ -0,0 +1,24 @@
# REQUIRES: zlib
# RUN: yaml2obj %s -o %t
# RUN: not llvm-objcopy --decompress-debug-sections %t /dev/null 2>&1 | FileCheck %s -DFILE=%t
# CHECK: error: '[[FILE]]': --decompress-debug-sections: ch_type (3) of section '.debug_info' is unsupported
# CHECK-EMPTY:
--- !ELF
FileHeader:
Class: ELFCLASS64
Data: ELFDATA2LSB
Type: ET_REL
Machine: EM_X86_64
Sections:
- Name: .debug_info
Type: SHT_PROGBITS
Flags: [ SHF_COMPRESSED ]
Content: 030000000000000004000000000000000000000000000000789c6360
AddressAlign: 8
- Name: .debug_str
Type: SHT_PROGBITS
Flags: [ SHF_COMPRESSED ]
Content: 030000000000000004000000000000000000000000000000789c6360
AddressAlign: 8

11
llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
View File

@ -721,16 +721,17 @@ objcopy::parseObjcopyOptions(ArrayRef<const char *> RawArgsArr,
if (const auto *A = InputArgs.getLastArg(OBJCOPY_compress_debug_sections)) {
Config.CompressionType = StringSwitch<DebugCompressionType>(A->getValue())
.Case("zlib", DebugCompressionType::Z)
.Case("zstd", DebugCompressionType::Zstd)
.Default(DebugCompressionType::None);
if (Config.CompressionType == DebugCompressionType::None)
if (Config.CompressionType == DebugCompressionType::None) {
return createStringError(
errc::invalid_argument,
"invalid or unsupported --compress-debug-sections format: %s",
A->getValue());
if (!compression::zlib::isAvailable())
return createStringError(
errc::invalid_argument,
"LLVM was not compiled with LLVM_ENABLE_ZLIB: can not compress");
}
if (const char *Reason = compression::getReasonIfUnsupported(
compression::formatFor(Config.CompressionType)))
return createStringError(errc::invalid_argument, Reason);
}
Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink);

2
llvm/tools/llvm-objcopy/ObjcopyOpts.td
View File

@ -33,7 +33,7 @@ def compress_debug_sections
: Joined<["--"], "compress-debug-sections=">,
MetaVarName<"format">,
HelpText<"Compress DWARF debug sections using specified format. Supported "
"formats: zlib">;
"formats: zlib, zstd. Select zlib if <format> is omitted">;
def : Flag<["--"], "compress-debug-sections">, Alias<compress_debug_sections>,
AliasArgs<["zlib"]>;
def decompress_debug_sections : Flag<["--"], "decompress-debug-sections">,

3
llvm/utils/lit/lit/llvm/config.py
View File

@ -112,6 +112,9 @@ class LLVMConfig(object):
have_zlib = getattr(config, 'have_zlib', None)
if have_zlib:
features.add('zlib')
have_zstd = getattr(config, 'have_zstd', None)
if have_zstd:
features.add('zstd')
# Check if we should run long running tests.
long_tests = lit_config.params.get('run_long_tests', None)