diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td index 268d73833d81c..c435da92bae46 100644 --- a/clang/include/clang/Basic/DiagnosticDriverKinds.td +++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td @@ -352,6 +352,8 @@ def warn_drv_sycl_offload_target_duplicate : Warning< def warn_drv_sycl_target_missing : Warning< "linked binaries do not contain expected '%0' target; found targets: '%1'">, InGroup; +def err_drv_no_rdc_sycl_target_missing : Error< + "linked binaries do not contain expected '%0' target; found targets: '%1', this is not supported with '-fno-sycl-rdc'">; def err_drv_multiple_target_with_forced_target : Error< "multiple target usage with '%0' is not supported with '%1'">; def err_drv_failed_to_deduce_target_from_arch : Error< diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 58ae955f41a99..cdaa48aa5c666 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -11791,6 +11791,9 @@ def err_sycl_restrict : Error< def err_sycl_external_global : Error< "invalid reference to 'device_global' variable; external 'device_global'" " variable must be marked with SYCL_EXTERNAL macro">; +def err_sycl_external_no_rdc : Error< + "invalid %select{declaration|definition}0 of SYCL_EXTERNAL function in non-relocatable " + "device code mode">; def warn_sycl_kernel_too_big_args : Warning< "size of kernel arguments (%0 bytes) may exceed the supported maximum " "of %1 bytes on some devices">, InGroup, ShowInSystemHeader; diff --git a/clang/include/clang/Driver/Action.h b/clang/include/clang/Driver/Action.h index cc27008eebde5..c7916b101bac0 100644 --- a/clang/include/clang/Driver/Action.h +++ b/clang/include/clang/Driver/Action.h @@ -14,6 +14,7 @@ #include "clang/Driver/Util.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/iterator_range.h" @@ -817,7 +818,8 @@ class FileTableTformJobAction : public JobAction { REPLACE, REPLACE_CELL, RENAME, - COPY_SINGLE_FILE + COPY_SINGLE_FILE, + MERGE }; Tform() = default; @@ -855,6 +857,10 @@ class FileTableTformJobAction : public JobAction { // output file. void addCopySingleFileTform(StringRef ColumnName, int Row); + // Merges all tables from filename listed at column into a + // single output table. + void addMergeTform(StringRef ColumnName); + static bool classof(const Action *A) { return A->getKind() == FileTableTformJobClass; } @@ -937,6 +943,14 @@ class ForEachWrappingAction : public Action { static bool classof(const Action *A) { return A->getKind() == ForEachWrappingClass; } + + void addSerialAction(const Action *A) { SerialActions.insert(A); } + const llvm::SmallSetVector &getSerialActions() const { + return SerialActions; + } + +private: + llvm::SmallSetVector SerialActions; }; } // namespace driver diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 8127d6d2e146c..8d1e21d54b7d9 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -1005,8 +1005,9 @@ def fno_cuda_flush_denormals_to_zero : Flag<["-"], "fno-cuda-flush-denormals-to- Alias; defm gpu_rdc : BoolFOption<"gpu-rdc", LangOpts<"GPURelocatableDeviceCode">, DefaultFalse, - PosFlag, - NegFlag>; + PosFlag, + NegFlag, + BothFlags<[CC1Option]>>; def : Flag<["-"], "fcuda-rdc">, Alias; def : Flag<["-"], "fno-cuda-rdc">, Alias; defm cuda_short_ptr : BoolFOption<"cuda-short-ptr", @@ -2990,6 +2991,8 @@ def fsycl_max_parallel_jobs_EQ : Joined<["-"], "fsycl-max-parallel-link-jobs=">, HelpText<"Experimental feature: Controls the maximum parallelism of actions performed " "on SYCL device code post-link, i.e. the generation of SPIR-V device images " "or AOT compilation of each device image.">; +def : Flag<["-"], "fsycl-rdc">, Alias; +def : Flag<["-"], "fno-sycl-rdc">, Alias; def fsyntax_only : Flag<["-"], "fsyntax-only">, Flags<[NoXarchOption,CoreOption,CC1Option,FC1Option,FlangOption]>, Group, HelpText<"Run the preprocessor, parser and semantic analysis stages">; diff --git a/clang/lib/Driver/Action.cpp b/clang/lib/Driver/Action.cpp index f9af7a8f3f970..73312dcf28744 100644 --- a/clang/lib/Driver/Action.cpp +++ b/clang/lib/Driver/Action.cpp @@ -567,6 +567,10 @@ void FileTableTformJobAction::addCopySingleFileTform(StringRef ColumnName, Tform(Tform::COPY_SINGLE_FILE, {ColumnName, std::to_string(Row)})); } +void FileTableTformJobAction::addMergeTform(StringRef ColumnName) { + Tforms.emplace_back(Tform(Tform::MERGE, {ColumnName})); +} + void AppendFooterJobAction::anchor() {} AppendFooterJobAction::AppendFooterJobAction(Action *Input, types::ID Type) diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index e6626b807c107..ec8609bdf51dd 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -1095,10 +1095,11 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C, Diag(clang::diag::err_drv_invalid_argument_to_option) << ArgValue << A->getOption().getName(); }; + Arg *DeviceCodeSplit = + C.getInputArgs().getLastArg(options::OPT_fsycl_device_code_split_EQ); checkSingleArgValidity(SYCLLink, {"early", "image"}); - checkSingleArgValidity( - C.getInputArgs().getLastArg(options::OPT_fsycl_device_code_split_EQ), - {"per_kernel", "per_source", "auto", "off"}); + checkSingleArgValidity(DeviceCodeSplit, + {"per_kernel", "per_source", "auto", "off"}); Arg *SYCLForceTarget = getArgRequiringSYCLRuntime(options::OPT_fsycl_force_target_EQ); @@ -5374,6 +5375,7 @@ class OffloadingActionBuilder final { // Current list is empty, nothing to process. continue; + ActionList DeviceLibs; ActionList DeviceLibObjects; ActionList LinkObjects; auto TT = TC->getTriple(); @@ -5439,6 +5441,8 @@ class OffloadingActionBuilder final { // n - target is NVPTX/AMDGCN // a - SPIRV AOT compilation is requested // s - device code split requested + // r - relocatable device code is requested + // f - link object output type is TY_Tempfilelist (fat archive) // * - "all other cases" // - no condition means output/input is "always" present // First symbol indicates output/input type @@ -5449,34 +5453,63 @@ class OffloadingActionBuilder final { // .-----------------. // |Link(LinkObjects)| // .-----------------. - // | - // .--------------------------------------. - // | PostLink | - // .--------------------------------------. - // [+*] [+] - // | | - // .-----------------. | - // | FileTableTform | | - // | (extract "Code")| | - // .-----------------. | - // [-] | - // --------------------| | - // [.] [-*] | - // .---------------. .-------------------. | - // | finalizeNVPTX | | SPIRVTranslator | | - // | finalizeAMDGCN | | | | - // .---------------. .-------------------. | - // [.] [-as] [-!a] | - // | | | | - // | [-s] | | - // | .----------------. | | - // | | BackendCompile | | | - // | .----------------. | | - // | [-s] | | - // | | | | - // | [-a] [-!a] [+] - // | .--------------------. - // -----------[-n]| FileTableTform | + // ----[-!rf] [*] + // [-!rf] | + // .-------------. | + // | llvm-foreach| | + // .-------------. | + // [.] | + // | | + // | | + // .---------------------------------------. + // | PostLink | + // .---------------------------------------. + // [+*] [+] + // | | + // | | + // |--------- | + // | | | + // | | | + // | [+!rf] | + // | .-------------. | + // | | llvm-foreach| | + // | .-------------. | + // | | | + // [+*] [+!rf] | + // .-----------------. | + // | FileTableTform | | + // | (extract "Code")| | + // .-----------------. | + // [-] |----------- + // --------------------| | + // | | | + // | |----------------- | + // | | | | + // | | [-!rf] | + // | | .--------------. | + // | | |FileTableTform| | + // | | | (merge) | | + // | | .--------------. | + // | | [-] |------- + // | | | | | + // | | | ------| | + // | | --------| | | + // [.] [-*] [-!rf] [+!rf] | + // .---------------. .-------------------. .--------------. | + // | finalizeNVPTX | | SPIRVTranslator | |FileTableTform| | + // | finalizeAMDGCN | | | | (merge) | | + // .---------------. .-------------------. . -------------. | + // [.] [-as] [-!a] | | + // | | | | | + // | [-s] | | | + // | .----------------. | | | + // | | BackendCompile | | | | + // | .----------------. | ------| | + // | [-s] | | | + // | | | | | + // | [-a] [-!a] [-!rf] | + // | .--------------------. | + // -----------[-n]| FileTableTform |[+*]--------------| // | (replace "Code") | // .--------------------. // | @@ -5485,16 +5518,20 @@ class OffloadingActionBuilder final { // | OffloadWrapper | // .--------------------------------------. // - Action *DeviceLinkAction = - C.MakeAction(LinkObjects, types::TY_LLVM_BC); ActionList FullLinkObjects; - bool SYCLDeviceLibLinked = false; - FullLinkObjects.push_back(DeviceLinkAction); + bool IsRDC = !tools::SYCL::shouldDoPerObjectFileLinking(C); + if (IsRDC) { + Action *DeviceLinkAction = + C.MakeAction(LinkObjects, types::TY_LLVM_BC); + FullLinkObjects.push_back(DeviceLinkAction); + } else + FullLinkObjects = LinkObjects; // FIXME: Link all wrapper and fallback device libraries as default, // When spv online link is supported by all backends, the fallback // device libraries are only needed when current toolchain is using // AOT compilation. + bool SYCLDeviceLibLinked = false; if (isSPIR || isNVPTX) { bool UseJitLink = isSPIR && @@ -5502,118 +5539,209 @@ class OffloadingActionBuilder final { options::OPT_fno_sycl_device_lib_jit_link, false); bool UseAOTLink = isSPIR && (isSpirvAOT || !UseJitLink); SYCLDeviceLibLinked = addSYCLDeviceLibs( - TC, FullLinkObjects, UseAOTLink, + TC, DeviceLibs, UseAOTLink, C.getDefaultToolChain().getTriple().isWindowsMSVCEnvironment()); } + JobAction *LinkSYCLLibs = + C.MakeAction(DeviceLibs, types::TY_LLVM_BC); + for (Action *FullLinkObject : FullLinkObjects) { + if (FullLinkObject->getKind() == + clang::driver::Action::OffloadDepsJobClass) + continue; + Action *FullDeviceLinkAction = nullptr; + ActionList WrapperInputs; + + if (SYCLDeviceLibLinked) { + if (IsRDC) { + // First object has to be non-DeviceLib for only-needed to be + // passed. + DeviceLibs.insert(DeviceLibs.begin(), FullLinkObject); + FullDeviceLinkAction = + C.MakeAction(DeviceLibs, types::TY_LLVM_BC); + } else { + FullDeviceLinkAction = FullLinkObject; + + ActionList DeviceCodeAndSYCLLibs{FullDeviceLinkAction, LinkSYCLLibs}; + JobAction *LinkDeviceCode = + C.MakeAction(DeviceCodeAndSYCLLibs, types::TY_LLVM_BC); + + if (FullDeviceLinkAction->getType() == types::TY_Tempfilelist) { + // If our compiler input outputs a temp file list (eg. fat + // static archive), we need to link the device code against + // each entry in the static archive. + auto *ParallelLinkDeviceCode = + C.MakeAction( + cast(FullDeviceLinkAction), LinkDeviceCode); + // The SYCL device library action tree should not be + // for-eached, it only needs to happen once total. The + // for-each action should start linking device code with the + // device libraries. + std::function traverseActionTree = + [&](const Action *Act) { + ParallelLinkDeviceCode->addSerialAction(Act); + for (const auto &Input : Act->getInputs()) { + traverseActionTree(Input); + } + }; + traverseActionTree(LinkSYCLLibs); + ActionList TformInputs{FullDeviceLinkAction, + ParallelLinkDeviceCode}; + auto *ReplaceFilesAction = + C.MakeAction( + TformInputs, types::TY_Tempfilelist, + types::TY_Tempfilelist); + ReplaceFilesAction->addReplaceColumnTform( + FileTableTformJobAction::COL_ZERO, + FileTableTformJobAction::COL_ZERO); + ReplaceFilesAction->addExtractColumnTform( + FileTableTformJobAction::COL_ZERO, false /*drop titles*/); + FullDeviceLinkAction = ReplaceFilesAction; + } else { + // If our compiler input is singular, just do a single link. + FullDeviceLinkAction = LinkDeviceCode; + } + } + } else + FullDeviceLinkAction = FullLinkObject; + + // reflects whether current target is ahead-of-time and can't + // support runtime setting of specialization constants + bool isAOT = isNVPTX || isAMDGCN || isSpirvAOT; + + // post link is not optional - even if not splitting, always need to + // process specialization constants + types::ID PostLinkOutType = isSPIR ? types::TY_Tempfiletable + : FullDeviceLinkAction->getType(); + auto createPostLinkAction = [&]() { + // For SPIR-V targets, force TY_Tempfiletable. + auto TypedPostLinkAction = C.MakeAction( + FullDeviceLinkAction, PostLinkOutType, types::TY_Tempfiletable); + TypedPostLinkAction->setRTSetsSpecConstants(!isAOT); + return TypedPostLinkAction; + }; + Action *PostLinkAction = createPostLinkAction(); + bool NoRDCFatStaticArchive = + !IsRDC && + FullDeviceLinkAction->getType() == types::TY_Tempfilelist; + if (NoRDCFatStaticArchive) + PostLinkAction = C.MakeAction( + cast(FullDeviceLinkAction), + cast(PostLinkAction)); + + auto createExtractIRFilesAction = [&]() { + auto *TypedExtractIRFilesAction = + C.MakeAction( + PostLinkAction, + isSPIR ? types::TY_Tempfilelist : PostLinkAction->getType(), + types::TY_Tempfilelist); + // single column w/o title fits TY_Tempfilelist format + TypedExtractIRFilesAction->addExtractColumnTform( + FileTableTformJobAction::COL_CODE, false /*drop titles*/); + return TypedExtractIRFilesAction; + }; - Action *FullDeviceLinkAction = nullptr; - if (SYCLDeviceLibLinked) - FullDeviceLinkAction = - C.MakeAction(FullLinkObjects, types::TY_LLVM_BC); - else - FullDeviceLinkAction = DeviceLinkAction; - - // reflects whether current target is ahead-of-time and can't support - // runtime setting of specialization constants - bool isAOT = isNVPTX || isAMDGCN || isSpirvAOT; - - ActionList WrapperInputs; - // post link is not optional - even if not splitting, always need to - // process specialization constants - - types::ID PostLinkOutType = - isSPIR ? types::TY_Tempfiletable : FullDeviceLinkAction->getType(); - // For SPIR-V targets, force TY_Tempfiletable. - auto *PostLinkAction = C.MakeAction( - FullDeviceLinkAction, PostLinkOutType, types::TY_Tempfiletable); - PostLinkAction->setRTSetsSpecConstants(!isAOT); - - auto *ExtractIRFilesAction = C.MakeAction( - PostLinkAction, - isSPIR ? types::TY_Tempfilelist : PostLinkAction->getType(), - types::TY_Tempfilelist); - // single column w/o title fits TY_Tempfilelist format - ExtractIRFilesAction->addExtractColumnTform( - FileTableTformJobAction::COL_CODE, false /*drop titles*/); - - if (isNVPTX || isAMDGCN) { - JobAction *FinAction = - isNVPTX ? finalizeNVPTXDependences(ExtractIRFilesAction, - TC->getTriple()) - : finalizeAMDGCNDependences(ExtractIRFilesAction, - TC->getTriple()); - auto *ForEachWrapping = C.MakeAction( - ExtractIRFilesAction, FinAction); - - ActionList TformInputs{PostLinkAction, ForEachWrapping}; - auto *ReplaceFilesAction = C.MakeAction( - TformInputs, types::TY_Tempfiletable, types::TY_Tempfiletable); - ReplaceFilesAction->addReplaceColumnTform( - FileTableTformJobAction::COL_CODE, - FileTableTformJobAction::COL_CODE); - - WrapperInputs.push_back(ReplaceFilesAction); - } else { - // For SPIRV-based targets - translate to SPIRV then optionally - // compile ahead-of-time to native architecture - Action *BuildCodeAction = - (Action *)C.MakeAction( - ExtractIRFilesAction, types::TY_Tempfilelist); + Action *ExtractIRFilesAction = createExtractIRFilesAction(); - // After the Link, wrap the files before the final host link - if (isAOT) { - types::ID OutType = types::TY_Tempfilelist; - if (!DeviceCodeSplit) { - OutType = (TT.getSubArch() == llvm::Triple::SPIRSubArch_fpga) - ? FPGAOutType - : types::TY_Image; + if (isNVPTX || isAMDGCN) { + JobAction *FinAction = + isNVPTX ? finalizeNVPTXDependences(ExtractIRFilesAction, + TC->getTriple()) + : finalizeAMDGCNDependences(ExtractIRFilesAction, + TC->getTriple()); + auto *ForEachWrapping = C.MakeAction( + cast(ExtractIRFilesAction), FinAction); + + ActionList TformInputs{PostLinkAction, ForEachWrapping}; + auto *ReplaceFilesAction = C.MakeAction( + TformInputs, types::TY_Tempfiletable, types::TY_Tempfiletable); + ReplaceFilesAction->addReplaceColumnTform( + FileTableTformJobAction::COL_CODE, + FileTableTformJobAction::COL_CODE); + + WrapperInputs.push_back(ReplaceFilesAction); + } else { + if (NoRDCFatStaticArchive) { + ExtractIRFilesAction = C.MakeAction( + cast(FullDeviceLinkAction), + cast(ExtractIRFilesAction)); + + auto *MergeAllTablesIntoOne = + C.MakeAction(ExtractIRFilesAction, + types::TY_Tempfilelist, + types::TY_Tempfilelist); + MergeAllTablesIntoOne->addMergeTform( + FileTableTformJobAction::COL_ZERO); + ExtractIRFilesAction = MergeAllTablesIntoOne; + } + // For SPIRV-based targets - translate to SPIRV then optionally + // compile ahead-of-time to native architecture + Action *BuildCodeAction = C.MakeAction( + ExtractIRFilesAction, types::TY_Tempfilelist); + + // After the Link, wrap the files before the final host link + if (isAOT) { + types::ID OutType = types::TY_Tempfilelist; + if (!DeviceCodeSplit) { + OutType = (TT.getSubArch() == llvm::Triple::SPIRSubArch_fpga) + ? FPGAOutType + : types::TY_Image; + } + // Do the additional Ahead of Time compilation when the specific + // triple calls for it (provided a valid subarch). + ActionList BEInputs; + BEInputs.push_back(BuildCodeAction); + auto unbundleAdd = [&](Action *A, types::ID T) { + ActionList AL; + AL.push_back(A); + Action *UnbundleAction = + C.MakeAction(AL, T); + BEInputs.push_back(UnbundleAction); + }; + // Send any known objects/archives through the unbundler to grab + // the dependency file associated. This is only done for + // -fintelfpga. + for (Action *A : FPGAObjectInputs) + unbundleAdd(A, types::TY_FPGA_Dependencies); + for (Action *A : FPGAArchiveInputs) + unbundleAdd(A, types::TY_FPGA_Dependencies_List); + for (const auto &A : DeviceLibObjects) + BEInputs.push_back(A); + BuildCodeAction = + C.MakeAction(BEInputs, OutType); + } + if (NoRDCFatStaticArchive) { + auto *MergeAllTablesIntoOne = + C.MakeAction(PostLinkAction, + types::TY_Tempfilelist, + types::TY_Tempfilelist); + MergeAllTablesIntoOne->addMergeTform( + FileTableTformJobAction::COL_ZERO); + PostLinkAction = MergeAllTablesIntoOne; } - // Do the additional Ahead of Time compilation when the specific - // triple calls for it (provided a valid subarch). - ActionList BEInputs; - BEInputs.push_back(BuildCodeAction); - auto unbundleAdd = [&](Action *A, types::ID T) { - ActionList AL; - AL.push_back(A); - Action *UnbundleAction = - C.MakeAction(AL, T); - BEInputs.push_back(UnbundleAction); - }; - // Send any known objects/archives through the unbundler to grab the - // dependency file associated. This is only done for -fintelfpga. - for (Action *A : FPGAObjectInputs) - unbundleAdd(A, types::TY_FPGA_Dependencies); - for (Action *A : FPGAArchiveInputs) - unbundleAdd(A, types::TY_FPGA_Dependencies_List); - for (const auto &A : DeviceLibObjects) - BEInputs.push_back(A); - BuildCodeAction = - C.MakeAction(BEInputs, OutType); + ActionList TformInputs{PostLinkAction, BuildCodeAction}; + auto *ReplaceFilesAction = C.MakeAction( + TformInputs, types::TY_Tempfiletable, types::TY_Tempfiletable); + ReplaceFilesAction->addReplaceColumnTform( + FileTableTformJobAction::COL_CODE, + FileTableTformJobAction::COL_CODE); + WrapperInputs.push_back(ReplaceFilesAction); } - ActionList TformInputs{PostLinkAction, BuildCodeAction}; - auto *ReplaceFilesAction = C.MakeAction( - TformInputs, types::TY_Tempfiletable, types::TY_Tempfiletable); - ReplaceFilesAction->addReplaceColumnTform( - FileTableTformJobAction::COL_CODE, - FileTableTformJobAction::COL_CODE); - WrapperInputs.push_back(ReplaceFilesAction); - } - // After the Link, wrap the files before the final host link - auto *DeviceWrappingAction = C.MakeAction( - WrapperInputs, types::TY_Object); + // After the Link, wrap the files before the final host link + auto *DeviceWrappingAction = C.MakeAction( + WrapperInputs, types::TY_Object); - if (isSpirvAOT) { - bool AddBA = (TT.getSubArch() == llvm::Triple::SPIRSubArch_gen && - BoundArch != nullptr); - DA.add(*DeviceWrappingAction, *TC, AddBA ? BoundArch : nullptr, - Action::OFK_SYCL); - } else - withBoundArchForToolChain(TC, [&](const char *BoundArch) { - DA.add(*DeviceWrappingAction, *TC, BoundArch, Action::OFK_SYCL); - }); + if (isSpirvAOT) { + bool AddBA = (TT.getSubArch() == llvm::Triple::SPIRSubArch_gen && + BoundArch != nullptr); + DA.add(*DeviceWrappingAction, *TC, AddBA ? BoundArch : nullptr, + Action::OFK_SYCL); + } else + withBoundArchForToolChain(TC, [&](const char *BoundArch) { + DA.add(*DeviceWrappingAction, *TC, BoundArch, Action::OFK_SYCL); + }); + } } - for (auto &SAI : SYCLAOTInputs) { // Extract binary file name std::string FN(SAI.second); @@ -5799,8 +5927,14 @@ class OffloadingActionBuilder final { ArchListStr += Section; Cnt++; } - C.getDriver().Diag(diag::warn_drv_sycl_target_missing) - << SectionTriple << ArchListStr; + if (tools::SYCL::shouldDoPerObjectFileLinking(C)) { + C.getDriver().Diag(diag::err_drv_no_rdc_sycl_target_missing) + << SectionTriple << ArchListStr; + C.setContainsError(); + } else { + C.getDriver().Diag(diag::warn_drv_sycl_target_missing) + << SectionTriple << ArchListStr; + } } } @@ -7990,7 +8124,8 @@ static void CollectForEachInputs( InputInfoList &InputInfos, const Action *SourceAction, const ToolChain *TC, StringRef BoundArch, Action::OffloadKind TargetDeviceOffloadKind, const std::map, InputInfoList> - &CachedResults) { + &CachedResults, + const ForEachWrappingAction *FEA) { for (const Action *Input : SourceAction->getInputs()) { // Search for the Input, if not in the cache assume actions were collapsed // so recurse. @@ -7998,10 +8133,12 @@ static void CollectForEachInputs( {Input, GetTriplePlusArchString(TC, BoundArch, TargetDeviceOffloadKind)}); if (Lookup != CachedResults.end()) { - InputInfos.append(Lookup->second); + if (!FEA->getSerialActions().count(Input)) { + InputInfos.append(Lookup->second); + } } else { CollectForEachInputs(InputInfos, Input, TC, BoundArch, - TargetDeviceOffloadKind, CachedResults); + TargetDeviceOffloadKind, CachedResults, FEA); } } } @@ -8172,18 +8309,25 @@ InputInfoList Driver::BuildJobsForActionNoCache( llvm::make_range(std::make_move_iterator(JobsToWrap.begin()), std::make_move_iterator(JobsToWrap.end()))) { const JobAction *SourceAction = cast(&Cmd->getSource()); - + if (FEA->getSerialActions().count(SourceAction)) { + C.addCommand(std::move(Cmd)); + continue; + } ActionResult = CachedResults.at( {SourceAction, GetTriplePlusArchString(TC, BoundArch, TargetDeviceOffloadKind)}).front(); InputInfoList InputInfos; CollectForEachInputs(InputInfos, SourceAction, TC, BoundArch, - TargetDeviceOffloadKind, CachedResults); + TargetDeviceOffloadKind, CachedResults, FEA); const Tool *Creator = &Cmd->getCreator(); + StringRef ParallelJobs; + if (TargetDeviceOffloadKind == Action::OFK_SYCL) + ParallelJobs = C.getArgs().getLastArgValue( + options::OPT_fsycl_max_parallel_jobs_EQ); tools::SYCL::constructLLVMForeachCommand( C, *SourceAction, std::move(Cmd), InputInfos, ActionResult, Creator, - "", types::getTypeTempSuffix(ActionResult.getType())); + "", types::getTypeTempSuffix(ActionResult.getType()), ParallelJobs); } return { ActionResult }; } diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index bcfc8332714cb..f38a773413553 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -4869,7 +4869,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, options::OPT_no_offload_new_driver, false)); bool IsRDCMode = - Args.hasFlag(options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc, false); + Args.hasFlag(options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc, IsSYCL); bool IsUsingLTO = D.isUsingLTO(IsDeviceOffloadAction); auto LTOMode = D.getLTOMode(IsDeviceOffloadAction); bool IsFPGASYCLOffloadDevice = @@ -6993,9 +6993,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, options::OPT_fno_hip_kernel_arg_name); } - if (IsCuda || IsHIP) { + if (IsCuda || IsHIP || IsSYCL) { if (IsRDCMode) CmdArgs.push_back("-fgpu-rdc"); + else + CmdArgs.push_back("-fno-gpu-rdc"); + } + if (IsCuda || IsHIP) { if (Args.hasFlag(options::OPT_fgpu_defer_diag, options::OPT_fno_gpu_defer_diag, false)) CmdArgs.push_back("-fgpu-defer-diag"); @@ -9842,6 +9846,13 @@ void FileTableTform::ConstructJob(Compilation &C, const JobAction &JA, addArgs(CmdArgs, TCArgs, {Arg}); break; } + case FileTableTformJobAction::Tform::MERGE: { + assert(Tf.TheArgs.size() == 1 && "column name expected"); + SmallString<128> Arg("-merge="); + Arg += Tf.TheArgs[0]; + addArgs(CmdArgs, TCArgs, {Arg}); + break; + } } } diff --git a/clang/lib/Driver/ToolChains/SYCL.cpp b/clang/lib/Driver/ToolChains/SYCL.cpp index 54bafca712b47..be7a92200c1ab 100644 --- a/clang/lib/Driver/ToolChains/SYCL.cpp +++ b/clang/lib/Driver/ToolChains/SYCL.cpp @@ -127,6 +127,11 @@ void SYCL::constructLLVMForeachCommand(Compilation &C, const JobAction &JA, C.addCommand(std::move(Cmd)); } +bool SYCL::shouldDoPerObjectFileLinking(const Compilation &C) { + return !C.getArgs().hasFlag(options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc, + /*default=*/true); +} + // The list should match pre-built SYCL device library files located in // compiler package. Once we add or remove any SYCL device library files, // the list should be updated accordingly. @@ -163,12 +168,25 @@ const char *SYCL::Linker::constructLLVMLinkCommand( // an actual object/archive. Take that list and pass those to the linker // instead of the original object. if (JA.isDeviceOffloading(Action::OFK_SYCL)) { - auto isSYCLDeviceLib = [&C, this](const InputInfo &II) { + bool IsRDC = !shouldDoPerObjectFileLinking(C); + auto isNoRDCDeviceCodeLink = [&](const InputInfo &II) { + if (IsRDC) + return false; + if (II.getType() != clang::driver::types::TY_LLVM_BC) + return false; + if (InputFiles.size() != 2) + return false; + return &II == &InputFiles[1]; + }; + auto isSYCLDeviceLib = [&](const InputInfo &II) { const ToolChain *HostTC = C.getSingleOffloadToolChain(); StringRef LibPostfix = ".o"; if (HostTC->getTriple().isWindowsMSVCEnvironment() && C.getDriver().IsCLMode()) LibPostfix = ".obj"; + else if (isNoRDCDeviceCodeLink(II)) + LibPostfix = ".bc"; + std::string FileName = this->getToolChain().getInputFilename(II); StringRef InputFilename = llvm::sys::path::filename(FileName); if (this->getToolChain().getTriple().isNVPTX()) { @@ -183,9 +201,21 @@ const char *SYCL::Linker::constructLLVMLinkCommand( !InputFilename.endswith(LibPostfix) || (InputFilename.count('-') < 2)) return false; // Skip the prefix "libsycl-" - StringRef PureLibName = InputFilename.substr(LibSyclPrefix.size()); + std::string PureLibName = + InputFilename.substr(LibSyclPrefix.size()).str(); + if (isNoRDCDeviceCodeLink(II)) { + // Skip the final - until the . because we linked all device libs into a + // single BC in a previous action so we have a temp file name. + auto FinalDashPos = PureLibName.find_last_of('-'); + auto DotPos = PureLibName.find_last_of('.'); + assert((FinalDashPos != std::string::npos && + DotPos != std::string::npos) && + "Unexpected filename"); + PureLibName = + PureLibName.substr(0, FinalDashPos) + PureLibName.substr(DotPos); + } for (const auto &L : SYCLDeviceLibList) { - if (PureLibName.startswith(L)) + if (StringRef(PureLibName).startswith(L)) return true; } return false; @@ -203,8 +233,17 @@ const char *SYCL::Linker::constructLLVMLinkCommand( for (const auto &II : InputFiles) { std::string FileName = getToolChain().getInputFilename(II); if (II.getType() == types::TY_Tempfilelist) { - // Pass the unbundled list with '@' to be processed. - Libs.push_back(C.getArgs().MakeArgString("@" + FileName)); + if (IsRDC) { + // Pass the unbundled list with '@' to be processed. + Libs.push_back(C.getArgs().MakeArgString("@" + FileName)); + } else { + assert(InputFiles.size() == 2 && + "Unexpected inputs for no-RDC with temp file list"); + // If we're in no-RDC mode and the input is a temp file list, + // we want to link multiple object files each against device libs, + // so we should consider this input as an object and not pass '@'. + Objs.push_back(C.getArgs().MakeArgString(FileName)); + } } else if (II.getType() == types::TY_Archive && !LinkSYCLDeviceLibs) { Libs.push_back(C.getArgs().MakeArgString(FileName)); } else diff --git a/clang/lib/Driver/ToolChains/SYCL.h b/clang/lib/Driver/ToolChains/SYCL.h index fb78d2ab7e270..f5aef6c9e5ea8 100644 --- a/clang/lib/Driver/ToolChains/SYCL.h +++ b/clang/lib/Driver/ToolChains/SYCL.h @@ -38,7 +38,7 @@ void constructLLVMForeachCommand(Compilation &C, const JobAction &JA, const InputInfo &Output, const Tool *T, StringRef Increment, StringRef Ext = "out", StringRef ParallelJobs = ""); - +bool shouldDoPerObjectFileLinking(const Compilation &C); // Runs llvm-spirv to convert spirv to bc, llvm-link, which links multiple LLVM // bitcode. Converts generated bc back to spirv using llvm-spirv, wraps with // offloading information. Finally compiles to object using llc diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 10bfe7caf0991..cc023f93ece19 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -3605,6 +3605,9 @@ void CompilerInvocation::GenerateLangArgs(const LangOptions &Opts, if (!Opts.RandstructSeed.empty()) GenerateArg(Args, OPT_frandomize_layout_seed_EQ, Opts.RandstructSeed, SA); + + if (!Opts.GPURelocatableDeviceCode) + GenerateArg(Args, OPT_fno_gpu_rdc, SA); } bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args, @@ -4200,6 +4203,11 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args, Diags.Report(diag::err_drv_hlsl_unsupported_target) << T.str(); } + // GPURelocatableDeviceCode should be true for SYCL if not specified. + if (Args.hasArg(OPT_fsycl_is_device) || Args.hasArg(OPT_fsycl_is_host)) + Opts.GPURelocatableDeviceCode = Args.hasFlag( + options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc, /*default=*/true); + return Diags.getNumErrors() == NumErrorsBefore; } diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index 8d74734b024ab..44aff4b6c0dff 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -10239,6 +10239,15 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC, } } + if (getLangOpts().SYCLIsDevice && !getLangOpts().GPURelocatableDeviceCode && + NewFD->hasAttr() && + !getSourceManager().isInSystemHeader(NewFD->getLocation())) { + Diag(NewFD->getLocation(), diag::err_sycl_external_no_rdc) + << (D.getFunctionDefinitionKind() == + clang::FunctionDefinitionKind::Definition); + NewFD->setInvalidDecl(); + } + if (!getLangOpts().CPlusPlus) { // Perform semantic checking on the function declaration. if (!NewFD->isInvalidDecl() && NewFD->isMain()) diff --git a/clang/test/Driver/sycl-no-rdc-errors.cpp b/clang/test/Driver/sycl-no-rdc-errors.cpp new file mode 100644 index 0000000000000..3506333efcc9e --- /dev/null +++ b/clang/test/Driver/sycl-no-rdc-errors.cpp @@ -0,0 +1,12 @@ +/// Tests driver errors for -no-sycl-rdc + +// RUN: %clang -target %itanium_abi_triple -c %s -o %t.o +// RUN: %clang -target spir64_gen -emit-llvm -c %s -o %t +// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,sycl-spir64_gen-unknown-unknown -input=%t -input=%t.o -output=%t.fat.o +// RUN: not %clang -### -fsycl -fno-sycl-rdc %t.fat.o 2>&1 | FileCheck -check-prefix=CHECK-ARCH %s + +// CHECK-ARCH: error: linked binaries do not contain expected 'spir64-unknown-unknown' target; found targets: 'spir64_gen-unknown-unknown', this is not supported with '-fno-sycl-rdc' + +// Some code so that we can create a binary out of this file. +void test_func(void) { +} diff --git a/clang/test/Driver/sycl-no-rdc-fat-archive.cpp b/clang/test/Driver/sycl-no-rdc-fat-archive.cpp new file mode 100644 index 0000000000000..9caac166bccac --- /dev/null +++ b/clang/test/Driver/sycl-no-rdc-fat-archive.cpp @@ -0,0 +1,29 @@ +/// test behaviors of passing a fat static lib with -fno-sycl-rdc +// Build a fat static lib that will be used for all tests +// RUN: echo "void foo(void) {}" > %t1.cpp +// RUN: %clangxx -target x86_64-unknown-linux-gnu -fsycl %t1.cpp -c -o %t1_bundle.o +// RUN: llvm-ar cr %t_lib.a %t1_bundle.o +// RUN: %clang -### -fsycl -fno-sycl-rdc -fsycl-device-code-split=none --sysroot=%S/Inputs/SYCL %t_lib.a 2>&1 -ccc-print-phases | FileCheck %s +// RUN: %clang -### -fsycl -fno-sycl-rdc -fsycl-device-code-split=auto --sysroot=%S/Inputs/SYCL %t_lib.a 2>&1 -ccc-print-phases | FileCheck %s +// RUN: %clang -### -fsycl -fno-sycl-rdc -fsycl-device-code-split=per_kernel --sysroot=%S/Inputs/SYCL %t_lib.a 2>&1 -ccc-print-phases | FileCheck %s +// RUN: %clang -### -fsycl -fno-sycl-rdc -fsycl-device-code-split=per_source --sysroot=%S/Inputs/SYCL %t_lib.a 2>&1 -ccc-print-phases | FileCheck %s +// CHECK: 2: input, "{{.*}}_lib.a", archive +// CHECK: 3: clang-offload-unbundler, {2}, tempfilelist +// CHECK: 4: spirv-to-ir-wrapper, {3}, tempfilelist, (device-sycl) +// CHECK: 5: input, "{{.*}}libsycl-crt{{.*}}", object +// CHECK: 6: clang-offload-unbundler, {5}, object +// CHECK: 7: offload, " (spir64-unknown-unknown)" {6}, object +// CHECK: 65: linker, {7, {{.*}}}, ir, (device-sycl) +// CHECK: 66: linker, {4, 65}, ir, (device-sycl) +// CHECK: 67: foreach, {4, 66}, ir, (device-sycl) +// CHECK: 68: file-table-tform, {4, 67}, tempfilelist, (device-sycl) +// CHECK: 69: sycl-post-link, {68}, tempfiletable, (device-sycl) +// CHECK: 70: foreach, {68, 69}, tempfiletable, (device-sycl) +// CHECK: 71: file-table-tform, {70}, tempfilelist, (device-sycl) +// CHECK: 72: file-table-tform, {70}, tempfilelist, (device-sycl) +// CHECK: 73: foreach, {68, 72}, tempfilelist, (device-sycl) +// CHECK: 74: file-table-tform, {73}, tempfilelist, (device-sycl) +// CHECK: 75: llvm-spirv, {74}, tempfilelist, (device-sycl) +// CHECK: 76: file-table-tform, {71, 75}, tempfiletable, (device-sycl) +// CHECK: 77: clang-offload-wrapper, {76}, object, (device-sycl) +// CHECK: 78: offload, "host-sycl (x86_64-unknown-linux-gnu)" {1}, "device-sycl (spir64-unknown-unknown)" {77}, image diff --git a/clang/test/Driver/sycl-no-rdc.cpp b/clang/test/Driver/sycl-no-rdc.cpp new file mode 100644 index 0000000000000..22f8135f987fc --- /dev/null +++ b/clang/test/Driver/sycl-no-rdc.cpp @@ -0,0 +1,31 @@ +/// Tests for -fno-sycl-rdc +// RUN: touch %t1.cpp +// RUN: touch %t2.cpp +// RUN: %clang -### -fsycl -fno-sycl-rdc --sysroot=%S/Inputs/SYCL %t1.cpp %t2.cpp 2>&1 -ccc-print-phases | FileCheck %s + +// CHECK: 3: input, "{{.*}}1.cpp", c++, (device-sycl) +// CHECK: 4: preprocessor, {3}, c++-cpp-output, (device-sycl) +// CHECK: 5: compiler, {4}, ir, (device-sycl) +// CHECK: 13: input, "{{.*}}2.cpp", c++, (device-sycl) +// CHECK: 14: preprocessor, {13}, c++-cpp-output, (device-sycl) +// CHECK: 15: compiler, {14}, ir, (device-sycl) + +// CHECK: 21: input, {{.*}}libsycl-crt{{.*}}, object +// CHECK: 22: clang-offload-unbundler, {21}, object +// CHECK: 23: offload, " (spir64-unknown-unknown)" {22}, object +// CHECK: 81: linker, {23, {{.*}}}, ir, (device-sycl) +// CHECK: 82: linker, {5, 81}, ir, (device-sycl) +// CHECK: 83: sycl-post-link, {82}, tempfiletable, (device-sycl) +// CHECK: 84: file-table-tform, {83}, tempfilelist, (device-sycl) +// CHECK: 85: llvm-spirv, {84}, tempfilelist, (device-sycl) +// CHECK: 86: file-table-tform, {83, 85}, tempfiletable, (device-sycl) +// CHECK: 87: clang-offload-wrapper, {86}, object, (device-sycl) + +// CHECK: 88: linker, {15, 81}, ir, (device-sycl) +// CHECK: 89: sycl-post-link, {88}, tempfiletable, (device-sycl) +// CHECK: 90: file-table-tform, {89}, tempfilelist, (device-sycl) +// CHECK: 91: llvm-spirv, {90}, tempfilelist, (device-sycl) +// CHECK: 92: file-table-tform, {89, 91}, tempfiletable, (device-sycl) +// CHECK: 93: clang-offload-wrapper, {92}, object, (device-sycl) + +// CHECK: 94: offload, "host-sycl (x86_64-unknown-linux-gnu)" {{{.*}}}, "device-sycl (spir64-unknown-unknown)" {87}, "device-sycl (spir64-unknown-unknown)" {93}, image diff --git a/clang/test/SemaSYCL/sycl-no-rdc.cpp b/clang/test/SemaSYCL/sycl-no-rdc.cpp new file mode 100644 index 0000000000000..b3466ee8fa396 --- /dev/null +++ b/clang/test/SemaSYCL/sycl-no-rdc.cpp @@ -0,0 +1,24 @@ +// RUN: %clang_cc1 -fsycl-is-device -verify -fsyntax-only -fno-gpu-rdc -internal-isystem %S/Inputs %s + +// Check that declarations of SYCL_EXTERNAL functions throw an error if -fno-gpu-rdc is passed +#include "sycl.hpp" + +// expected-error@+1{{invalid declaration of SYCL_EXTERNAL function in non-relocatable device code mode}} +SYCL_EXTERNAL void syclExternalDecl(); + +// expected-error@+1{{invalid definition of SYCL_EXTERNAL function in non-relocatable device code mode}} +SYCL_EXTERNAL void syclExternalDefn() {} + +using namespace sycl; +queue q; + +void kernel_wrapper() { + q.submit([&](handler &h) { + h.single_task([=] { + }); + }); +} + +int main() { + kernel_wrapper(); +} diff --git a/llvm/include/llvm/Support/SimpleTable.h b/llvm/include/llvm/Support/SimpleTable.h index e91f288a6bc26..1881f080736c4 100644 --- a/llvm/include/llvm/Support/SimpleTable.h +++ b/llvm/include/llvm/Support/SimpleTable.h @@ -132,6 +132,9 @@ class SimpleTable { Row &operator[](int I) { return Rows[I]; } const Row &operator[](int I) const { return Rows[I]; } + // Merge another table into this table + Error merge(const SimpleTable &Other); + private: Error addColumnName(StringRef ColName); void rebuildName2NumMapping(); diff --git a/llvm/lib/Support/SimpleTable.cpp b/llvm/lib/Support/SimpleTable.cpp index 41aeb57b15d28..b75f65d3bf6d6 100644 --- a/llvm/lib/Support/SimpleTable.cpp +++ b/llvm/lib/Support/SimpleTable.cpp @@ -251,5 +251,14 @@ Expected SimpleTable::read(const Twine &FileName, return read(MemBuf->get(), ColSep); } +Error SimpleTable::merge(const SimpleTable &Other) { + if (getNumColumns() != Other.getNumColumns()) + return makeError("different number of columns"); + if (ColumnNames != Other.ColumnNames) + return makeError("different column names"); + Rows.insert(Rows.end(), Other.Rows.begin(), Other.Rows.end()); + return Error::success(); +} + } // namespace util } // namespace llvm diff --git a/llvm/test/tools/file-table-tform/Inputs/a1.txt b/llvm/test/tools/file-table-tform/Inputs/a1.txt new file mode 100644 index 0000000000000..a26424e50e980 --- /dev/null +++ b/llvm/test/tools/file-table-tform/Inputs/a1.txt @@ -0,0 +1,3 @@ +[11|12] +44|55 +66|77 diff --git a/llvm/test/tools/file-table-tform/Inputs/merge-gold.txt b/llvm/test/tools/file-table-tform/Inputs/merge-gold.txt new file mode 100644 index 0000000000000..e81ea3c87ecfe --- /dev/null +++ b/llvm/test/tools/file-table-tform/Inputs/merge-gold.txt @@ -0,0 +1,5 @@ +[11|12] +00|11 +22|33 +44|55 +66|77 diff --git a/llvm/test/tools/file-table-tform/Inputs/merge-input.txt b/llvm/test/tools/file-table-tform/Inputs/merge-input.txt new file mode 100644 index 0000000000000..67866fe6602ea --- /dev/null +++ b/llvm/test/tools/file-table-tform/Inputs/merge-input.txt @@ -0,0 +1,3 @@ +[One|Two] +0|a0.txt +1|a1.txt diff --git a/llvm/test/tools/file-table-tform/file-table-tform-merge.test b/llvm/test/tools/file-table-tform/file-table-tform-merge.test new file mode 100644 index 0000000000000..f81f34b2d33c9 --- /dev/null +++ b/llvm/test/tools/file-table-tform/file-table-tform-merge.test @@ -0,0 +1,6 @@ +-- Merge two tables listed in merge-input.txt +RUN: cp %S/Inputs/a{0,1}.txt . +RUN: file-table-tform --merge=Two %S/Inputs/merge-input.txt -o merge-output.txt + +-- Verify result +RUN: diff %S/Inputs/merge-gold.txt merge-output.txt diff --git a/llvm/tools/file-table-tform/file-table-tform.cpp b/llvm/tools/file-table-tform/file-table-tform.cpp index 1ded228953a00..c1add76ad1616 100644 --- a/llvm/tools/file-table-tform/file-table-tform.cpp +++ b/llvm/tools/file-table-tform/file-table-tform.cpp @@ -82,6 +82,7 @@ static constexpr char OPT_REPLACE_CELL[] = "replace_cell"; static constexpr char OPT_RENAME[] = "rename"; static constexpr char OPT_EXTRACT[] = "extract"; static constexpr char OPT_COPY_SINGLE_FILE[] = "copy_single_file"; +static constexpr char OPT_MERGE[] = "merge"; static cl::list TformReplace{ OPT_REPLACE, cl::ZeroOrMore, cl::desc("replace a column"), @@ -107,6 +108,10 @@ static cl::list TformCopySingleFile{ cl::value_desc(","), cl::cat(FileTableTformCat)}; +static cl::list TformMerge{ + OPT_MERGE, cl::Optional, cl::desc("merge all input tables"), + cl::value_desc(""), cl::cat(FileTableTformCat)}; + static cl::opt DropTitles{"drop_titles", cl::Optional, cl::desc("drop column titles"), cl::cat(FileTableTformCat)}; @@ -172,7 +177,8 @@ struct TformCmd { .Case(OPT_RENAME, [&](TformCmd *Cmd) { return Error::success(); }) .Case(OPT_EXTRACT, [&](TformCmd *Cmd) { return Error::success(); }) .Case(OPT_COPY_SINGLE_FILE, - [&](TformCmd *Cmd) { return Error::success(); }); + [&](TformCmd *Cmd) { return Error::success(); }) + .Case(OPT_MERGE, [&](TformCmd *Cmd) { return Error::success(); }); return F(this); } @@ -234,15 +240,24 @@ struct TformCmd { std::back_inserter(Args)); return Error::success(); }) - .Case(OPT_COPY_SINGLE_FILE, [&](TformCmd *Cmd) -> Error { - // argument is , + .Case(OPT_COPY_SINGLE_FILE, + [&](TformCmd *Cmd) -> Error { + // argument is , + if (Arg.empty()) + return makeUserError("empty argument in " + + Twine(OPT_COPY_SINGLE_FILE)); + Arg.split(Args, ','); + if (Args.size() != 2 || Args[0].empty() || Args[1].empty()) + return makeUserError("invalid argument in " + + Twine(OPT_COPY_SINGLE_FILE)); + return Error::success(); + }) + .Case(OPT_MERGE, [&](TformCmd *Cmd) -> Error { if (Arg.empty()) - return makeUserError("empty argument in " + - Twine(OPT_COPY_SINGLE_FILE)); + return makeUserError("empty argument in " + Twine(OPT_MERGE)); Arg.split(Args, ','); - if (Args.size() != 2 || Args[0].empty() || Args[1].empty()) - return makeUserError("invalid argument in " + - Twine(OPT_COPY_SINGLE_FILE)); + if (Args.size() != 1 || Args[0].empty()) + return makeUserError("invalid argument in " + Twine(OPT_MERGE)); return Error::success(); }); return F(this); @@ -286,23 +301,63 @@ struct TformCmd { Error Res = Table.peelColumns(Args); return Res ? std::move(Res) : std::move(Error::success()); }) - .Case(OPT_COPY_SINGLE_FILE, [&](TformCmd *Cmd) -> Error { - // argument is , - assert(Args.size() == 2); - const int Row = std::stoi(Args[1].str()); - if (Row >= Table.getNumRows()) - return makeUserError("row index is out of bounds"); + .Case(OPT_COPY_SINGLE_FILE, + [&](TformCmd *Cmd) -> Error { + // argument is , + assert(Args.size() == 2); + const int Row = std::stoi(Args[1].str()); + if (Row >= Table.getNumRows()) + return makeUserError("row index is out of bounds"); - // Copy the file from the only remaining row at specified - // column - StringRef FileToCopy = Table[Row].getCell(Args[0], ""); + // Copy the file from the only remaining row at specified + // column + StringRef FileToCopy = Table[Row].getCell(Args[0], ""); - if (FileToCopy.empty()) - return makeUserError("no file found in specified column"); + if (FileToCopy.empty()) + return makeUserError("no file found in specified column"); + std::error_code EC = sys::fs::copy_file(FileToCopy, Output); + return EC ? createFileError(Output, EC) + : std::move(Error::success()); + }) + .Case(OPT_MERGE, [&](TformCmd *Cmd) -> Error { + StringRef FileToCopy = Table[0].getCell(Args[0]); + if (!llvm::sys::fs::exists(FileToCopy)) + return makeUserError("first file not found"); std::error_code EC = sys::fs::copy_file(FileToCopy, Output); - return EC ? createFileError(Output, EC) - : std::move(Error::success()); + if (EC) + return createFileError(Output, EC); + + Expected OutputTable = + util::SimpleTable::read(Output); + if (!OutputTable) + return OutputTable.takeError(); + + // The first table will be the table merged into, so start merging + // from the second table. + for (int Row = 1; Row < Table.getNumRows(); Row++) { + // Each cell in the input table is a path + // to another table. + Expected Table1 = + util::SimpleTable::read(Table[Row].getCell(Args[0])); + if (!Table1) + return Table1.takeError(); + + Error Err = (*OutputTable)->merge(*Table1.get()); + if (Err) + return Err; + } + raw_fd_ostream Out{Output, EC, sys::fs::OpenFlags::OF_None}; + + if (EC) + return createFileError(Output, EC); + + (*OutputTable)->write(Out, (*OutputTable)->getNumColumns() > 1); + + if (Out.has_error()) + return createFileError(Output, Out.error()); + Out.close(); + return Error::success(); }); return F(this); } @@ -344,9 +399,10 @@ int main(int argc, char **argv) { // yet, as an order across all command line options-commands needs to be // established first to properly map inputs to commands. - auto Lists = {std::addressof(TformReplace), std::addressof(TformReplaceCell), - std::addressof(TformRename), std::addressof(TformExtract), - std::addressof(TformCopySingleFile)}; + auto Lists = { + std::addressof(TformReplace), std::addressof(TformReplaceCell), + std::addressof(TformRename), std::addressof(TformExtract), + std::addressof(TformCopySingleFile), std::addressof(TformMerge)}; for (const auto *L : Lists) { for (auto It = L->begin(); It != L->end(); It++) { @@ -369,12 +425,17 @@ int main(int argc, char **argv) { // ensure that if copy_single_file is specified, it must be the last tform bool HasCopySingleFileTform = false; + bool HasMerge = false; for (auto &P : Cmds) { if (HasCopySingleFileTform) { CHECK_AND_EXIT( makeUserError("copy_single_file must be the last transformation")); } + if (HasMerge) { + CHECK_AND_EXIT(makeUserError("merge must be the last transformation")); + } HasCopySingleFileTform = P.second->Kind == OPT_COPY_SINGLE_FILE; + HasMerge = P.second->Kind == OPT_MERGE; } for (auto &P : Cmds) { @@ -397,9 +458,9 @@ int main(int argc, char **argv) { CHECK_AND_EXIT(std::move(Res)); } - // If copy_single_file was specified the output file is generated by the - // corresponding transformation. - if (!HasCopySingleFileTform) { + // If copy_single_file or merge was specified the output file is generated by + // the corresponding transformation. + if (!HasCopySingleFileTform && !HasMerge) { // Write the transformed table to file std::error_code EC; raw_fd_ostream Out{Output, EC, sys::fs::OpenFlags::OF_None};