OSchip
/
llvm-project
mirror of https://github.com/THU-DSP-LAB/llvm-project.git
13
0
Fork 2
Code Issues Packages Projects Releases Wiki Activity

[SystemZ][z/OS] Distinguish between text and binary files on z/OS 

This patch consists of the initial changes to help distinguish between text and binary content correctly on z/OS. I would like to get feedback from Windows users on setting OF_None for all ToolOutputFiles. This seems to have been done as an optimization to prevent CRLF translation on Windows in the past.

Reviewed By: zibi

Differential Revision: https://reviews.llvm.org/D97785
This commit is contained in:
Abhina Sreeskantharajan 2021-03-19 08:09:01 -04:00
parent c2313a4530
commit 4f750f6ebc
12 changed files with 117 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
clang/lib/Frontend/CompilerInstance.cpp
View File

@ -814,15 +814,18 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary,
TempPath += OutputExtension;
TempPath += ".tmp";
int fd;
std::error_code EC =
llvm::sys::fs::createUniqueFile(TempPath, fd, TempPath);
std::error_code EC = llvm::sys::fs::createUniqueFile(
TempPath, fd, TempPath,
Binary ? llvm::sys::fs::OF_None : llvm::sys::fs::OF_Text);
if (CreateMissingDirectories &&
EC == llvm::errc::no_such_file_or_directory) {
StringRef Parent = llvm::sys::path::parent_path(OutputPath);
EC = llvm::sys::fs::create_directories(Parent);
if (!EC) {
EC = llvm::sys::fs::createUniqueFile(TempPath, fd, TempPath);
EC = llvm::sys::fs::createUniqueFile(TempPath, fd, TempPath,
Binary ? llvm::sys::fs::OF_None
: llvm::sys::fs::OF_Text);
}
}

11
clang/lib/Frontend/FrontendActions.cpp
View File

@ -795,7 +795,7 @@ void PreprocessOnlyAction::ExecuteAction() {
void PrintPreprocessedAction::ExecuteAction() {
CompilerInstance &CI = getCompilerInstance();
// Output file may need to be set to 'Binary', to avoid converting Unix style
// line feeds (<LF>) to Microsoft style line feeds (<CR><LF>).
// line feeds (<LF>) to Microsoft style line feeds (<CR><LF>) on Windows.
//
// Look to see what type of line endings the file uses. If there's a
// CRLF, then we won't open the file up in binary mode. If there is
@ -807,7 +807,10 @@ void PrintPreprocessedAction::ExecuteAction() {
// all of their source code on a single line. However, that is still a
// concern, so if we scan for too long, we'll just assume the file should
// be opened in binary mode.
bool BinaryMode = true;
bool BinaryMode = false;
if (llvm::Triple(LLVM_HOST_TRIPLE).isOSWindows()) {
BinaryMode = true;
const SourceManager &SM = CI.getSourceManager();
if (llvm::Optional<llvm::MemoryBufferRef> Buffer =
SM.getBufferOrNone(SM.getMainFileID())) {
@ -818,7 +821,8 @@ void PrintPreprocessedAction::ExecuteAction() {
// Limit ourselves to only scanning 256 characters into the source
// file. This is mostly a sanity check in case the file has no
// newlines whatsoever.
if (end - cur > 256) end = cur + 256;
if (end - cur > 256)
end = cur + 256;
while (next < end) {
if (*cur == 0x0D) { // CR
@ -833,6 +837,7 @@ void PrintPreprocessedAction::ExecuteAction() {
++next;
}
}
}
std::unique_ptr<raw_ostream> OS =
CI.createDefaultOutputFile(BinaryMode, getCurrentFileOrBufferName());

4
clang/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
View File

@ -275,8 +275,8 @@ void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
<< "' absolute: " << EC.message() << '\n';
return;
}
if (std::error_code EC =
llvm::sys::fs::createUniqueFile(Model, FD, ResultPath)) {
if (std::error_code EC = llvm::sys::fs::createUniqueFile(
Model, FD, ResultPath, llvm::sys::fs::OF_Text)) {
llvm::errs() << "warning: could not create file in '" << Directory
<< "': " << EC.message() << '\n';
return;

12
clang/tools/arcmt-test/arcmt-test.cpp
View File

@ -207,11 +207,15 @@ static bool performTransformations(StringRef resourcesPath,
static bool filesCompareEqual(StringRef fname1, StringRef fname2) {
using namespace llvm;
ErrorOr<std::unique_ptr<MemoryBuffer>> file1 = MemoryBuffer::getFile(fname1);
ErrorOr<std::unique_ptr<MemoryBuffer>> file1 = MemoryBuffer::getFile(
fname1, /*FileSize*/ -1, /*RequiresNullTerminator*/ true,
/*IsVolatile*/ false, /*IsText*/ true);
if (!file1)
return false;
ErrorOr<std::unique_ptr<MemoryBuffer>> file2 = MemoryBuffer::getFile(fname2);
ErrorOr<std::unique_ptr<MemoryBuffer>> file2 = MemoryBuffer::getFile(
fname2, /*FileSize*/ -1, /*RequiresNullTerminator*/ true,
/*IsVolatile*/ false, /*IsText*/ true);
if (!file2)
return false;
@ -240,7 +244,9 @@ static bool verifyTransformedFiles(ArrayRef<std::string> resultFiles) {
if (RemappingsFile.empty())
inputBuf = MemoryBuffer::getSTDIN();
else
inputBuf = MemoryBuffer::getFile(RemappingsFile);
inputBuf = MemoryBuffer::getFile(RemappingsFile, /*FileSize*/ -1,
/*RequiresNullTerminator*/ true,
/*IsVolatile*/ false, /*IsText*/ true);
if (!inputBuf) {
errs() << "error: could not read remappings input\n";
return true;

9
llvm/include/llvm/Support/FileSystem.h
View File

@ -802,10 +802,13 @@ void createUniquePath(const Twine &Model, SmallVectorImpl<char> &ResultPath,
/// @param Model Name to base unique path off of.
/// @param ResultFD Set to the opened file's file descriptor.
/// @param ResultPath Set to the opened file's absolute path.
/// @param Flags Set to the opened file's flags.
/// @param Mode Set to the opened file's permissions.
/// @returns errc::success if Result{FD,Path} have been successfully set,
/// otherwise a platform-specific error_code.
std::error_code createUniqueFile(const Twine &Model, int &ResultFD,
SmallVectorImpl<char> &ResultPath,
OpenFlags Flags = OF_None,
unsigned Mode = all_read | all_write);
/// Simpler version for clients that don't want an open file. An empty
@ -862,12 +865,14 @@ public:
/// running the assembler.
std::error_code createTemporaryFile(const Twine &Prefix, StringRef Suffix,
int &ResultFD,
SmallVectorImpl<char> &ResultPath);
SmallVectorImpl<char> &ResultPath,
OpenFlags Flags = OF_None);
/// Simpler version for clients that don't want an open file. An empty
/// file will still be created.
std::error_code createTemporaryFile(const Twine &Prefix, StringRef Suffix,
SmallVectorImpl<char> &ResultPath);
SmallVectorImpl<char> &ResultPath,
OpenFlags Flags = OF_None);
std::error_code createUniqueDirectory(const Twine &Prefix,
SmallVectorImpl<char> &ResultPath);

8
llvm/include/llvm/Support/MemoryBuffer.h
View File

@ -82,9 +82,13 @@ public:
/// \param IsVolatile Set to true to indicate that the contents of the file
/// can change outside the user's control, e.g. when libclang tries to parse
/// while the user is editing/updating the file or if the file is on an NFS.
///
/// \param IsText Set to true to indicate that the file should be read in
/// text mode.
static ErrorOr<std::unique_ptr<MemoryBuffer>>
getFile(const Twine &Filename, int64_t FileSize = -1,
bool RequiresNullTerminator = true, bool IsVolatile = false);
bool RequiresNullTerminator = true, bool IsVolatile = false,
bool IsText = false);
/// Read all of the specified file into a MemoryBuffer as a stream
/// (i.e. until EOF reached). This is useful for special files that
@ -130,7 +134,7 @@ public:
/// is "-".
static ErrorOr<std::unique_ptr<MemoryBuffer>>
getFileOrSTDIN(const Twine &Filename, int64_t FileSize = -1,
bool RequiresNullTerminator = true);
bool RequiresNullTerminator = true, bool IsText = false);
/// Map a subrange of the specified file as a MemoryBuffer.
static ErrorOr<std::unique_ptr<MemoryBuffer>>

4
llvm/lib/IRReader/IRReader.cpp
View File

@ -92,7 +92,9 @@ std::unique_ptr<Module>
llvm::parseIRFile(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context,
DataLayoutCallbackTy DataLayoutCallback) {
ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
MemoryBuffer::getFileOrSTDIN(Filename);
MemoryBuffer::getFileOrSTDIN(Filename, /*FileSize*/ -1,
/*RequiresNullTerminator*/ true,
/*IsText*/ true);
if (std::error_code EC = FileOrErr.getError()) {
Err = SMDiagnostic(Filename, SourceMgr::DK_Error,
"Could not open input file: " + EC.message());

26
llvm/lib/Support/MemoryBuffer.cpp
View File

@ -106,7 +106,8 @@ public:
template <typename MB>
static ErrorOr<std::unique_ptr<MB>>
getFileAux(const Twine &Filename, int64_t FileSize, uint64_t MapSize,
uint64_t Offset, bool RequiresNullTerminator, bool IsVolatile);
uint64_t Offset, bool RequiresNullTerminator, bool IsVolatile,
bool IsText);
std::unique_ptr<MemoryBuffer>
MemoryBuffer::getMemBuffer(StringRef InputData, StringRef BufferName,
@ -141,20 +142,20 @@ MemoryBuffer::getMemBufferCopy(StringRef InputData, const Twine &BufferName) {
ErrorOr<std::unique_ptr<MemoryBuffer>>
MemoryBuffer::getFileOrSTDIN(const Twine &Filename, int64_t FileSize,
bool RequiresNullTerminator) {
bool RequiresNullTerminator, bool IsText) {
SmallString<256> NameBuf;
StringRef NameRef = Filename.toStringRef(NameBuf);
if (NameRef == "-")
return getSTDIN();
return getFile(Filename, FileSize, RequiresNullTerminator);
return getFile(Filename, FileSize, RequiresNullTerminator, false, IsText);
}
ErrorOr<std::unique_ptr<MemoryBuffer>>
MemoryBuffer::getFileSlice(const Twine &FilePath, uint64_t MapSize,
uint64_t Offset, bool IsVolatile) {
return getFileAux<MemoryBuffer>(FilePath, -1, MapSize, Offset, false,
IsVolatile);
IsVolatile, false);
}
//===----------------------------------------------------------------------===//
@ -240,12 +241,12 @@ getMemoryBufferForStream(sys::fs::file_t FD, const Twine &BufferName) {
return getMemBufferCopyImpl(Buffer, BufferName);
}
ErrorOr<std::unique_ptr<MemoryBuffer>>
MemoryBuffer::getFile(const Twine &Filename, int64_t FileSize,
bool RequiresNullTerminator, bool IsVolatile) {
bool RequiresNullTerminator, bool IsVolatile,
bool IsText) {
return getFileAux<MemoryBuffer>(Filename, FileSize, FileSize, 0,
RequiresNullTerminator, IsVolatile);
RequiresNullTerminator, IsVolatile, IsText);
}
template <typename MB>
@ -257,9 +258,10 @@ getOpenFileImpl(sys::fs::file_t FD, const Twine &Filename, uint64_t FileSize,
template <typename MB>
static ErrorOr<std::unique_ptr<MB>>
getFileAux(const Twine &Filename, int64_t FileSize, uint64_t MapSize,
uint64_t Offset, bool RequiresNullTerminator, bool IsVolatile) {
Expected<sys::fs::file_t> FDOrErr =
sys::fs::openNativeFileForRead(Filename, sys::fs::OF_None);
uint64_t Offset, bool RequiresNullTerminator, bool IsVolatile,
bool IsText) {
Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
Filename, IsText ? sys::fs::OF_Text : sys::fs::OF_None);
if (!FDOrErr)
return errorToErrorCode(FDOrErr.takeError());
sys::fs::file_t FD = *FDOrErr;
@ -274,14 +276,14 @@ WritableMemoryBuffer::getFile(const Twine &Filename, int64_t FileSize,
bool IsVolatile) {
return getFileAux<WritableMemoryBuffer>(Filename, FileSize, FileSize, 0,
/*RequiresNullTerminator*/ false,
IsVolatile);
IsVolatile, false);
}
ErrorOr<std::unique_ptr<WritableMemoryBuffer>>
WritableMemoryBuffer::getFileSlice(const Twine &Filename, uint64_t MapSize,
uint64_t Offset, bool IsVolatile) {
return getFileAux<WritableMemoryBuffer>(Filename, -1, MapSize, Offset, false,
IsVolatile);
IsVolatile, false);
}
std::unique_ptr<WritableMemoryBuffer>

48
llvm/lib/Support/Path.cpp
View File

@ -167,8 +167,8 @@ enum FSEntity {
static std::error_code
createUniqueEntity(const Twine &Model, int &ResultFD,
SmallVectorImpl<char> &ResultPath, bool MakeAbsolute,
unsigned Mode, FSEntity Type,
sys::fs::OpenFlags Flags = sys::fs::OF_None) {
FSEntity Type, sys::fs::OpenFlags Flags = sys::fs::OF_None,
unsigned Mode = 0) {
// Limit the number of attempts we make, so that we don't infinite loop. E.g.
// "permission denied" could be for a specific file (so we retry with a
@ -816,22 +816,16 @@ void createUniquePath(const Twine &Model, SmallVectorImpl<char> &ResultPath,
std::error_code createUniqueFile(const Twine &Model, int &ResultFd,
SmallVectorImpl<char> &ResultPath,
unsigned Mode) {
return createUniqueEntity(Model, ResultFd, ResultPath, false, Mode, FS_File);
}
static std::error_code createUniqueFile(const Twine &Model, int &ResultFd,
SmallVectorImpl<char> &ResultPath,
unsigned Mode, OpenFlags Flags) {
return createUniqueEntity(Model, ResultFd, ResultPath, false, Mode, FS_File,
Flags);
OpenFlags Flags, unsigned Mode) {
return createUniqueEntity(Model, ResultFd, ResultPath, false, FS_File, Flags,
Mode);
}
std::error_code createUniqueFile(const Twine &Model,
SmallVectorImpl<char> &ResultPath,
unsigned Mode) {
int FD;
auto EC = createUniqueFile(Model, FD, ResultPath, Mode);
auto EC = createUniqueFile(Model, FD, ResultPath, OF_None, Mode);
if (EC)
return EC;
// FD is only needed to avoid race conditions. Close it right away.
@ -841,34 +835,39 @@ std::error_code createUniqueFile(const Twine &Model,
static std::error_code
createTemporaryFile(const Twine &Model, int &ResultFD,
llvm::SmallVectorImpl<char> &ResultPath, FSEntity Type) {
llvm::SmallVectorImpl<char> &ResultPath, FSEntity Type,
sys::fs::OpenFlags Flags = sys::fs::OF_None) {
SmallString<128> Storage;
StringRef P = Model.toNullTerminatedStringRef(Storage);
assert(P.find_first_of(separators(Style::native)) == StringRef::npos &&
"Model must be a simple filename.");
// Use P.begin() so that createUniqueEntity doesn't need to recreate Storage.
return createUniqueEntity(P.begin(), ResultFD, ResultPath, true,
owner_read | owner_write, Type);
return createUniqueEntity(P.begin(), ResultFD, ResultPath, true, Type, Flags,
owner_read | owner_write);
}
static std::error_code
createTemporaryFile(const Twine &Prefix, StringRef Suffix, int &ResultFD,
llvm::SmallVectorImpl<char> &ResultPath, FSEntity Type) {
llvm::SmallVectorImpl<char> &ResultPath, FSEntity Type,
sys::fs::OpenFlags Flags = sys::fs::OF_None) {
const char *Middle = Suffix.empty() ? "-%%%%%%" : "-%%%%%%.";
return createTemporaryFile(Prefix + Middle + Suffix, ResultFD, ResultPath,
Type);
Type, Flags);
}
std::error_code createTemporaryFile(const Twine &Prefix, StringRef Suffix,
int &ResultFD,
SmallVectorImpl<char> &ResultPath) {
return createTemporaryFile(Prefix, Suffix, ResultFD, ResultPath, FS_File);
SmallVectorImpl<char> &ResultPath,
sys::fs::OpenFlags Flags) {
return createTemporaryFile(Prefix, Suffix, ResultFD, ResultPath, FS_File,
Flags);
}
std::error_code createTemporaryFile(const Twine &Prefix, StringRef Suffix,
SmallVectorImpl<char> &ResultPath) {
SmallVectorImpl<char> &ResultPath,
sys::fs::OpenFlags Flags) {
int FD;
auto EC = createTemporaryFile(Prefix, Suffix, FD, ResultPath);
auto EC = createTemporaryFile(Prefix, Suffix, FD, ResultPath, Flags);
if (EC)
return EC;
// FD is only needed to avoid race conditions. Close it right away.
@ -876,13 +875,12 @@ std::error_code createTemporaryFile(const Twine &Prefix, StringRef Suffix,
return EC;
}
// This is a mkdtemp with a different pattern. We use createUniqueEntity mostly
// for consistency. We should try using mkdtemp.
std::error_code createUniqueDirectory(const Twine &Prefix,
SmallVectorImpl<char> &ResultPath) {
int Dummy;
return createUniqueEntity(Prefix + "-%%%%%%", Dummy, ResultPath, true, 0,
return createUniqueEntity(Prefix + "-%%%%%%", Dummy, ResultPath, true,
FS_Dir);
}
@ -890,7 +888,7 @@ std::error_code
getPotentiallyUniqueFileName(const Twine &Model,
SmallVectorImpl<char> &ResultPath) {
int Dummy;
return createUniqueEntity(Model, Dummy, ResultPath, false, 0, FS_Name);
return createUniqueEntity(Model, Dummy, ResultPath, false, FS_Name);
}
std::error_code
@ -1279,7 +1277,7 @@ Expected<TempFile> TempFile::create(const Twine &Model, unsigned Mode) {
int FD;
SmallString<128> ResultPath;
if (std::error_code EC =
createUniqueFile(Model, FD, ResultPath, Mode, OF_Delete))
createUniqueFile(Model, FD, ResultPath, OF_Delete, Mode))
return errorCodeToError(EC);
TempFile Ret(ResultPath, FD);

8
llvm/lib/Support/ToolOutputFile.cpp
View File

@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/Support/ToolOutputFile.h"
#include "llvm/ADT/Triple.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Signals.h"
using namespace llvm;
@ -45,7 +46,12 @@ ToolOutputFile::ToolOutputFile(StringRef Filename, std::error_code &EC,
EC = std::error_code();
return;
}
OSHolder.emplace(Filename, EC, Flags);
// On Windows, we set the OF_None flag even for text files to avoid
// CRLF translation.
OSHolder.emplace(
Filename, EC,
llvm::Triple(LLVM_HOST_TRIPLE).isOSWindows() ? sys::fs::OF_None : Flags);
OS = OSHolder.getPointer();
// If open fails, no cleanup is needed.
if (EC)

9
llvm/lib/TableGen/Main.cpp
View File

@ -70,7 +70,7 @@ static int createDependencyFile(const TGParser &Parser, const char *argv0) {
return reportError(argv0, "the option -d must be used together with -o\n");
std::error_code EC;
ToolOutputFile DepOut(DependFilename, EC, sys::fs::OF_None);
ToolOutputFile DepOut(DependFilename, EC, sys::fs::OF_Text);
if (EC)
return reportError(argv0, "error opening " + DependFilename + ":" +
EC.message() + "\n");
@ -93,7 +93,7 @@ int llvm::TableGenMain(const char *argv0, TableGenMainFn *MainFn) {
Records.startTimer("Parse, build records");
ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
MemoryBuffer::getFileOrSTDIN(InputFilename);
MemoryBuffer::getFileOrSTDIN(InputFilename, -1, true, true);
if (std::error_code EC = FileOrErr.getError())
return reportError(argv0, "Could not open input file '" + InputFilename +
"': " + EC.message() + "\n");
@ -137,13 +137,14 @@ int llvm::TableGenMain(const char *argv0, TableGenMainFn *MainFn) {
// Only updates the real output file if there are any differences.
// This prevents recompilation of all the files depending on it if there
// aren't any.
if (auto ExistingOrErr = MemoryBuffer::getFile(OutputFilename))
if (auto ExistingOrErr =
MemoryBuffer::getFile(OutputFilename, -1, true, false, true))
if (std::move(ExistingOrErr.get())->getBuffer() == Out.str())
WriteFile = false;
}
if (WriteFile) {
std::error_code EC;
ToolOutputFile OutFile(OutputFilename, EC, sys::fs::OF_None);
ToolOutputFile OutFile(OutputFilename, EC, sys::fs::OF_Text);
if (EC)
return reportError(argv0, "error opening " + OutputFilename + ": " +
EC.message() + "\n");

8
llvm/utils/FileCheck/FileCheck.cpp
View File

@ -821,7 +821,9 @@ int main(int argc, char **argv) {
// Read the expected strings from the check file.
ErrorOr<std::unique_ptr<MemoryBuffer>> CheckFileOrErr =
MemoryBuffer::getFileOrSTDIN(CheckFilename);
MemoryBuffer::getFileOrSTDIN(CheckFilename, /*FileSize*/ -1,
/*RequiresNullTerminator*/ true,
/*IsText*/ true);
if (std::error_code EC = CheckFileOrErr.getError()) {
errs() << "Could not open check file '" << CheckFilename
<< "': " << EC.message() << '\n';
@ -843,7 +845,9 @@ int main(int argc, char **argv) {
// Open the file to check and add it to SourceMgr.
ErrorOr<std::unique_ptr<MemoryBuffer>> InputFileOrErr =
MemoryBuffer::getFileOrSTDIN(InputFilename);
MemoryBuffer::getFileOrSTDIN(InputFilename, /*FileSize*/ -1,
/*RequiresNullTerminator*/ true,
/*IsText*/ true);
if (InputFilename == "-")
InputFilename = "<stdin>"; // Overwrite for improved diagnostic messages
if (std::error_code EC = InputFileOrErr.getError()) {