Skip to content

[clang-offload-bundler] unification with the upstream LLVM [NFC] #2823

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Dec 1, 2020
Merged

[clang-offload-bundler] unification with the upstream LLVM [NFC] #2823

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 42 additions & 93 deletions clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,18 +163,6 @@ class FileHandler {
/// Read the current bundle and write the result into the stream \a OS.
virtual Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0;

/// Read the current bundle and write the result into the file \a FileName.
/// The meaning of \a FileName depends on unbundling type - in some
/// cases (type="oo") it will contain a list of actual outputs.
virtual Error ReadBundle(StringRef FileName, MemoryBuffer &Input) {
std::error_code EC;
raw_fd_ostream OS(FileName, EC);

if (EC)
return createFileError(FileName, EC);
return ReadBundle(OS, Input);
}

/// Write the header of the bundled file to \a OS based on the information
/// gathered from \a Inputs.
virtual Error WriteHeader(raw_fd_ostream &OS,
Expand Down Expand Up @@ -354,8 +342,6 @@ class BinaryFileHandler final : public FileHandler {
return Error::success();
}

using FileHandler::ReadBundle; // to avoid hiding via the overload below

Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
StringRef FC = Input.getBuffer();
Expand Down Expand Up @@ -638,23 +624,14 @@ class ObjectFileHandler final : public FileHandler {

Error ReadBundleEnd(MemoryBuffer &Input) final { return Error::success(); }

Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) {
llvm_unreachable("must not be called for the ObjectFileHandler");
}

Error ReadBundle(StringRef OutName, MemoryBuffer &Input) final {
Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
assert(CurBundle != TripleToBundleInfo.end() &&
"all bundles have been read already");

// TODO: temporary workaround to copy fat object to the host output until
// driver is fixed to correctly handle list file for the host bundle in
// 'oo' mode.
if (FilesType == "oo" && hasHostKind(CurBundle->getKey())) {
std::error_code EC;
raw_fd_ostream OS(OutName, EC);

if (EC)
return createFileError(OutName, EC);
OS.write(Input.getBufferStart(), Input.getBufferSize());
return Error::success();
}
Expand All @@ -676,35 +653,13 @@ class ObjectFileHandler final : public FileHandler {
errc::invalid_argument,
"'o' file type is requested, but the fat object contains multiple "
"device objects; use 'oo' instead");
std::string FileList;

// Iterate through individual objects and extract them
for (size_t I = 0; I < NumObjects; ++I) {
uint64_t ObjSize = SizeVec[I];
// Flag for the special case used to "unbundle" host target object.
bool HostTriple = ObjSize == 1;

StringRef ObjFileName = OutName;
SmallString<128> Path;

// If not in file list mode there is no need in a temporary file - output
// goes directly to what was specified in -outputs. The same is true for
// the host triple.
if (FileListMode && !HostTriple) {
std::error_code EC =
sys::fs::createTemporaryFile(TempFileNameBase, "devo", Path);
ObjFileName = Path.data();

if (EC)
return createFileError(ObjFileName, EC);
}
std::error_code EC;
raw_fd_ostream OS(ObjFileName, EC);

if (EC)
return createFileError(ObjFileName, EC);

if (HostTriple) {
// Flag for the special case used to "unbundle" host target object.
if (ObjSize == 1) {
// Copy fat object contents to the output when extracting host bundle.

// In the partially linked fat object multiple dummy host bundles were
Expand All @@ -716,26 +671,31 @@ class ObjectFileHandler final : public FileHandler {
OS.write(Input.getBufferStart(), Input.getBufferSize());
break;
}
OS.write(ObjData, ObjSize);

if (FileListMode) {
SmallString<128> ObjFileName;
std::error_code EC =
sys::fs::createTemporaryFile(TempFileNameBase, "devo", ObjFileName);
if (EC)
return createFileError(ObjFileName, EC);

raw_fd_ostream ObjOS(ObjFileName, EC);
if (EC)
return createFileError(ObjFileName, EC);

ObjOS.write(ObjData, ObjSize);
if (ObjOS.has_error())
return createFileError(ObjFileName, ObjOS.error());

// add the written file name to the output list of files
FileList = (Twine(FileList) + Twine(ObjFileName) + Twine("\n")).str();
}
OS << ObjFileName << "\n";
} else
OS.write(ObjData, ObjSize);

// Move "object data" pointer to the next object within the concatenated
// bundle.
ObjData += ObjSize;
}
if (FileListMode) {
// dump the list of files into the file list specified in -outputs for the
// current target
std::error_code EC;
raw_fd_ostream OS1(OutName, EC);

if (EC)
return createFileError(OutName, EC);
OS1.write(FileList.data(), FileList.size());
}
return Error::success();
}

Expand Down Expand Up @@ -939,8 +899,6 @@ class TextFileHandler final : public FileHandler {
return Error::success();
}

using FileHandler::ReadBundle; // to avoid hiding via the overload below

Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
StringRef FC = Input.getBuffer();
size_t BundleStart = ReadChars;
Expand Down Expand Up @@ -1056,7 +1014,7 @@ class ArchiveFileHandler final : public FileHandler {

Error ReadBundleEnd(MemoryBuffer &Input) override { return Error::success(); }

Error ReadBundle(StringRef OutName, MemoryBuffer &Input) override {
Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) override {
assert(CurrBundle->second && "attempt to extract nonexistent bundle");

bool FileListMode = FilesType == "aoo";
Expand All @@ -1068,11 +1026,6 @@ class ArchiveFileHandler final : public FileHandler {
"'ao' file type is requested, but the archive contains multiple "
"device objects; use 'aoo' instead");

// In file-list mode archive unbundling produces multiple files, so output
// file is a file list where we write the unbundled object names.
SmallVector<char, 0u> FileListBuf;
raw_svector_ostream FileList{FileListBuf};

// Read all children.
Error Err = Error::success();
for (auto &C : Ar->children(Err)) {
Expand Down Expand Up @@ -1102,25 +1055,30 @@ class ArchiveFileHandler final : public FileHandler {
// This is the bundle we are looking for. Create temporary file where
// the device part will be extracted if we are in the file-list mode,
// or write directly to the output file otherwise.
SmallString<128u> ChildFileName;
if (FileListMode) {
SmallString<128u> ChildFileName;
auto EC = sys::fs::createTemporaryFile(TempFileNameBase, "o",
ChildFileName);
if (EC)
return createFileError(ChildFileName, EC);
} else
ChildFileName = OutName;

// And extract the bundle.
if (Error Err = OFH.ReadBundle(ChildFileName, *Buf))
return Err;
if (Error Err = OFH.ReadBundleEnd(*Buf))
return Err;
raw_fd_ostream ChildOS(ChildFileName, EC);
if (EC)
return createFileError(ChildFileName, EC);

if (Error Err = OFH.ReadBundle(ChildOS, *Buf))
return Err;

if (ChildOS.has_error())
return createFileError(ChildFileName, ChildOS.error());

if (FileListMode)
// Add temporary file name with the device part to the output file
// list.
FileList << ChildFileName << "\n";
OS << ChildFileName << "\n";
} else if (Error Err = OFH.ReadBundle(OS, *Buf))
return Err;
if (Error Err = OFH.ReadBundleEnd(*Buf))
return Err;
}
NameOrErr = OFH.ReadBundleStart(*Buf);
if (!NameOrErr)
Expand All @@ -1129,22 +1087,9 @@ class ArchiveFileHandler final : public FileHandler {
}
if (Err)
return Err;

if (FileListMode) {
// Dump file list to the output file.
std::error_code EC;
raw_fd_ostream OS(OutName, EC);
if (EC)
return createFileError(OutName, EC);
OS << FileList.str();
}
return Error::success();
}

Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) override {
llvm_unreachable("must not be called for the ArchiveFileHandler");
}

Error WriteHeader(raw_fd_ostream &OS,
ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) override {
llvm_unreachable("unsupported for the ArchiveFileHandler");
Expand Down Expand Up @@ -1323,7 +1268,11 @@ static Error UnbundleFiles() {
continue;

// Check if the output file can be opened and copy the bundle to it.
if (Error Err = FH->ReadBundle(Output->second, Input))
std::error_code EC;
raw_fd_ostream OutputFile(Output->second, EC, sys::fs::OF_None);
if (EC)
return createFileError(Output->second, EC);
if (Error Err = FH->ReadBundle(OutputFile, Input))
return Err;
if (Error Err = FH->ReadBundleEnd(Input))
return Err;
Expand Down