[LLD] Implement --enable-non-contiguous-regions

When enabled, input sections that would otherwise overflow a memory
region are instead spilled to the next matching output section.

This feature parallels the one in GNU LD, but there are some differences
from its documented behavior:

- /DISCARD/ only matches previously-unmatched sections (i.e., the flag
  does not affect it).

- If a section fails to fit at any of its matches, the link fails
  instead of discarding the section.

- The flag --enable-non-contiguous-regions-warnings is not implemented,
  as it exists to warn about such occurrences.

The implementation places stubs at possible spill locations, and
replaces them with the original input section when effecting spills.
Spilling decisions occur after address assignment. Sections are spilled
in reverse order of assignment, with each spill naively decreasing the
size of the affected memory regions. This continues until the memory
regions are brought back under size. Spilling anything causes another
pass of address assignment, and this continues to fixed point.

Spilling after rather than during assignment allows the algorithm to
consider the size effects of unspillable input sections that appear
later in the assignment. Otherwise, such sections (e.g. thunks) may
force an overflow, even if spilling something earlier could have avoided
it.

A few notable feature interactions occur:

- Stubs affect alignment, ONLY_IF_RO, etc, broadly as if a copy of the
  input section were actually placed there.

- SHF_MERGE synthetic sections use the spill list of their first
  contained input section (the one that gives the section its name).

- ICF occurs oblivious to spill sections; spill lists for merged-away
  sections become inert and are removed after assignment.

- SHF_LINK_ORDER and .ARM.exidx are ordered according to the final
  section ordering, after all spilling has completed.

- INSERT BEFORE/AFTER and OVERWRITE_SECTIONS are explicitly disallowed.

Author: mysterymath

lld/ELF/Config.h

@@ -237,6 +237,7 @@ struct Config {
   bool emitLLVM;
   bool emitRelocs;
   bool enableNewDtags;
+  bool enableNonContiguousRegions;
   bool executeOnly;
   bool exportDynamic;
   bool fixCortexA53Errata843419;

lld/ELF/Driver.cpp

@@ -1250,6 +1250,8 @@ static void readConfigs(opt::InputArgList &args) {
   config->emitRelocs = args.hasArg(OPT_emit_relocs);
   config->enableNewDtags =
       args.hasFlag(OPT_enable_new_dtags, OPT_disable_new_dtags, true);
+  config->enableNonContiguousRegions =
+      args.hasArg(OPT_enable_non_contiguous_regions);
   config->entry = args.getLastArgValue(OPT_entry);
 
   errorHandler().errorHandlingScript =
@@ -3077,7 +3079,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
     // sectionBases.
     for (SectionCommand *cmd : script->sectionCommands)
       if (auto *osd = dyn_cast<OutputDesc>(cmd))
-        osd->osec.finalizeInputSections();
+        osd->osec.finalizeInputSections(script.get());
   }
 
   // Two input sections with different output sections should not be folded.

lld/ELF/ICF.cpp

@@ -75,7 +75,7 @@
 #include "ICF.h"
 #include "Config.h"
 #include "InputFiles.h"
-#include "LinkerScript.h"
+#include "InputSection.h"
 #include "OutputSections.h"
 #include "SymbolTable.h"
 #include "Symbols.h"

lld/ELF/InputSection.cpp

@@ -161,6 +161,7 @@ uint64_t SectionBase::getOffset(uint64_t offset) const {
   }
   case Regular:
   case Synthetic:
+  case Spill:
     return cast<InputSection>(this)->outSecOff + offset;
   case EHFrame: {
     // Two code paths may reach here. First, clang_rt.crtbegin.o and GCC
@@ -309,6 +310,12 @@ std::string InputSectionBase::getObjMsg(uint64_t off) const {
       .str();
 }
 
+SpillInputSection::SpillInputSection(InputSectionBase *source,
+                                     InputSectionDescription *isd)
+    : InputSection(source->file, source->flags, source->type, source->addralign,
+                   {}, source->name, SectionBase::Spill),
+      isd(isd) {}
+
 InputSection InputSection::discarded(nullptr, 0, 0, 0, ArrayRef<uint8_t>(), "");
 
 InputSection::InputSection(InputFile *f, uint64_t flags, uint32_t type,

lld/ELF/InputSection.h

@@ -48,7 +48,7 @@ template <class ELFT> struct RelsOrRelas {
 // sections.
 class SectionBase {
 public:
-  enum Kind { Regular, Synthetic, EHFrame, Merge, Output };
+  enum Kind { Regular, Synthetic, EHFrame, Merge, Output, Spill };
 
   Kind kind() const { return (Kind)sectionKind; }
 
@@ -382,7 +382,8 @@ class InputSection : public InputSectionBase {
 
   static bool classof(const SectionBase *s) {
     return s->kind() == SectionBase::Regular ||
-           s->kind() == SectionBase::Synthetic;
+           s->kind() == SectionBase::Synthetic ||
+           s->kind() == SectionBase::Spill;
   }
 
   // Write this section to a mmap'ed file, assuming Buf is pointing to
@@ -425,6 +426,25 @@ class InputSection : public InputSectionBase {
   template <class ELFT> void copyShtGroup(uint8_t *buf);
 };
 
+// A marker for a potential spill location for another input section. This
+// broadly acts as if it were the original section until address assignment.
+// Then it is either replaced with the real input section or removed.
+class SpillInputSection : public InputSection {
+public:
+  // The containing input section description; used to quickly replace this stub
+  // with the actual section.
+  InputSectionDescription *isd;
+
+  // Next spill location for the same source input section.
+  SpillInputSection *next = nullptr;
+
+  SpillInputSection(InputSectionBase *source, InputSectionDescription *cmd);
+
+  static bool classof(const SectionBase *sec) {
+    return sec->kind() == InputSectionBase::Spill;
+  }
+};
+
 static_assert(sizeof(InputSection) <= 160, "InputSection is too big");
 
 class SyntheticSection : public InputSection {

lld/ELF/LinkerScript.cpp

@@ -304,6 +304,9 @@ getChangedSymbolAssignment(const SymbolAssignmentMap &oldValues) {
 void LinkerScript::processInsertCommands() {
   SmallVector<OutputDesc *, 0> moves;
   for (const InsertCommand &cmd : insertCommands) {
+    if (config->enableNonContiguousRegions)
+      error("INSERT cannot be used with --enable-non-contiguous-regions");
+
     for (StringRef name : cmd.names) {
       // If base is empty, it may have been discarded by
       // adjustOutputSections(). We do not handle such output sections.
@@ -486,10 +489,12 @@ static void sortInputSections(MutableArrayRef<InputSectionBase *> vec,
 // Compute and remember which sections the InputSectionDescription matches.
 SmallVector<InputSectionBase *, 0>
 LinkerScript::computeInputSections(const InputSectionDescription *cmd,
-                                   ArrayRef<InputSectionBase *> sections) {
+                                   ArrayRef<InputSectionBase *> sections,
+                                   const OutputSection &outCmd) {
   SmallVector<InputSectionBase *, 0> ret;
   SmallVector<size_t, 0> indexes;
   DenseSet<size_t> seen;
+  DenseSet<InputSectionBase *> spills;
   auto sortByPositionThenCommandLine = [&](size_t begin, size_t end) {
     llvm::sort(MutableArrayRef<size_t>(indexes).slice(begin, end - begin));
     for (size_t i = begin; i != end; ++i)
@@ -505,12 +510,33 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
     size_t sizeBeforeCurrPat = ret.size();
 
     for (size_t i = 0, e = sections.size(); i != e; ++i) {
-      // Skip if the section is dead or has been matched by a previous input
-      // section description or a previous pattern.
+      // Skip if the section is dead or has been matched by a previous pattern
+      // in this input section description.
       InputSectionBase *sec = sections[i];
-      if (!sec->isLive() || sec->parent || seen.contains(i))
+      if (!sec->isLive() || seen.contains(i))
         continue;
 
+      if (sec->parent) {
+        // Skip if not allowing multiple matches.
+        if (!config->enableNonContiguousRegions)
+          continue;
+
+        // Disallow spilling into /DISCARD/; special handling would be needed
+        // for this in address assignment, and the semantics are nebulous.
+        if (outCmd.name == "/DISCARD/")
+          continue;
+
+        // Skip if the section's first match was /DISCARD/; such sections are
+        // always discarded.
+        if (sec->parent->name == "/DISCARD/")
+          continue;
+
+        // Skip if the section was already matched by a different input section
+        // description within this output section.
+        if (sec->parent == &outCmd)
+          continue;
+      }
+
       // For --emit-relocs we have to ignore entries like
       //   .rela.dyn : { *(.rela.data) }
       // which are common because they are in the default bfd script.
@@ -530,6 +556,8 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
         continue;
 
       ret.push_back(sec);
+      if (sec->parent)
+        spills.insert(sec);
       indexes.push_back(i);
       seen.insert(i);
     }
@@ -555,6 +583,28 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd,
   // Matched sections after the last SORT* are sorted by (--sort-alignment,
   // input order).
   sortByPositionThenCommandLine(sizeAfterPrevSort, ret.size());
+
+  // Replace matches after the first with potential spill sections.
+  if (!spills.empty()) {
+    for (InputSectionBase *&sec : ret) {
+      if (!spills.contains(sec))
+        continue;
+
+      SpillInputSection *sis = make<SpillInputSection>(
+          sec, const_cast<InputSectionDescription *>(cmd));
+
+      // Append the spill input section to the list for the input section,
+      // creating it if necessary.
+      auto res = spillLists.try_emplace(sec, SpillList{sis, sis});
+      if (!res.second) {
+        SpillInputSection *&tail = res.first->second.tail;
+        tail = tail->next = sis;
+      }
+
+      sec = sis;
+    }
+  }
+
   return ret;
 }
 
@@ -577,7 +627,7 @@ void LinkerScript::discardSynthetic(OutputSection &outCmd) {
         part.armExidx->exidxSections.end());
     for (SectionCommand *cmd : outCmd.commands)
       if (auto *isd = dyn_cast<InputSectionDescription>(cmd))
-        for (InputSectionBase *s : computeInputSections(isd, secs))
+        for (InputSectionBase *s : computeInputSections(isd, secs, outCmd))
           discard(*s);
   }
 }
@@ -588,7 +638,7 @@ LinkerScript::createInputSectionList(OutputSection &outCmd) {
 
   for (SectionCommand *cmd : outCmd.commands) {
     if (auto *isd = dyn_cast<InputSectionDescription>(cmd)) {
-      isd->sectionBases = computeInputSections(isd, ctx.inputSections);
+      isd->sectionBases = computeInputSections(isd, ctx.inputSections, outCmd);
       for (InputSectionBase *s : isd->sectionBases)
         s->parent = &outCmd;
       ret.insert(ret.end(), isd->sectionBases.begin(), isd->sectionBases.end());
@@ -644,6 +694,9 @@ void LinkerScript::processSectionCommands() {
 
   // Process OVERWRITE_SECTIONS first so that it can overwrite the main script
   // or orphans.
+  if (config->enableNonContiguousRegions && !overwriteSections.empty())
+    error("OVERWRITE_SECTIONS cannot be used with "
+          "--enable-non-contiguous-regions");
   DenseMap<CachedHashStringRef, OutputDesc *> map;
   size_t i = 0;
   for (OutputDesc *osd : overwriteSections) {
@@ -911,6 +964,13 @@ void LinkerScript::diagnoseMissingSGSectionAddress() const {
     error("no address assigned to the veneers output section " + sec->name);
 }
 
+void LinkerScript::copySpillList(InputSectionBase *dst, InputSectionBase *src) {
+  auto i = spillLists.find(src);
+  if (i == spillLists.end())
+    return;
+  spillLists.try_emplace(dst, i->second);
+}
+
 // This function searches for a memory region to place the given output
 // section in. If found, a pointer to the appropriate memory region is
 // returned in the first member of the pair. Otherwise, a nullptr is returned.
@@ -1066,8 +1126,16 @@ void LinkerScript::assignOffsets(OutputSection *sec) {
     // Handle a single input section description command.
     // It calculates and assigns the offsets for each section and also
     // updates the output section size.
-    for (InputSection *isec : cast<InputSectionDescription>(cmd)->sections) {
+
+    DenseSet<InputSection *> spills;
+    auto &sections = cast<InputSectionDescription>(cmd)->sections;
+    for (InputSection *isec : sections) {
       assert(isec->getParent() == sec);
+
+      // Skip all possible spills.
+      if (isa<SpillInputSection>(isec))
+        continue;
+
       const uint64_t pos = dot;
       dot = alignToPowerOf2(dot, isec->addralign);
       isec->outSecOff = dot - sec->addr;
@@ -1364,6 +1432,107 @@ const Defined *LinkerScript::assignAddresses() {
   return getChangedSymbolAssignment(oldValues);
 }
 
+static bool isRegionOverflowed(MemoryRegion *mr) {
+  if (!mr)
+    return false;
+  return mr->curPos - mr->getOrigin() > mr->getLength();
+}
+
+// Spill input sections in reverse order of address assignment to (potentially)
+// bring memory regions out of overflow. The size savings of a spill can only be
+// estimated, since general linker script arithmetic may occur afterwards.
+// Under-estimates may cause unnecessary spills, but over-estimates can always
+// be corrected on the next pass.
+bool LinkerScript::spillSections() {
+  if (!config->enableNonContiguousRegions)
+    return false;
+
+  bool spilled = false;
+  for (SectionCommand *cmd : reverse(sectionCommands)) {
+    auto *od = dyn_cast<OutputDesc>(cmd);
+    if (!od)
+      continue;
+    OutputSection *osec = &od->osec;
+    if (!osec->size || !osec->memRegion)
+      continue;
+
+    DenseSet<InputSection *> spills;
+    for (SectionCommand *cmd : reverse(osec->commands)) {
+      if (!isRegionOverflowed(osec->memRegion) &&
+          !isRegionOverflowed(osec->lmaRegion))
+        break;
+
+      auto *is = dyn_cast<InputSectionDescription>(cmd);
+      if (!is)
+        continue;
+      for (InputSection *isec : reverse(is->sections)) {
+        // Potential spill locations cannot be spilled.
+        if (isa<SpillInputSection>(isec))
+          continue;
+
+        // Find the next spill location.
+        auto it = spillLists.find(isec);
+        if (it == spillLists.end())
+          continue;
+
+        spilled = true;
+        SpillList &list = it->second;
+
+        SpillInputSection *spill = list.head;
+        if (!spill->next)
+          spillLists.erase(isec);
+        else
+          list.head = spill->next;
+
+        spills.insert(isec);
+
+        // Replace the next spill location with the spilled section and adjust
+        // its properties to match the new location.
+        *llvm::find(spill->isd->sections, spill) = isec;
+        isec->parent = spill->parent;
+        // The alignment of the spill section may have diverged from the
+        // original, but correct assignment requires the spill's alignment,
+        // not the original.
+        isec->addralign = spill->addralign;
+
+        // Record the reduction in overage.
+        osec->memRegion->curPos -= isec->getSize();
+        if (osec->lmaRegion)
+          osec->lmaRegion->curPos -= isec->getSize();
+        if (!isRegionOverflowed(osec->memRegion) &&
+            !isRegionOverflowed(osec->lmaRegion))
+          break;
+      }
+      // Remove any spilled sections.
+      if (!spills.empty())
+        llvm::erase_if(is->sections, [&](InputSection *isec) {
+          return spills.contains(isec);
+        });
+    }
+  }
+
+  return spilled;
+}
+
+// Erase any potential spill sections that were not used.
+void LinkerScript::eraseSpillSections() {
+  if (spillLists.empty())
+    return;
+
+  // Collect the set of input section descriptions that contain potential
+  // spills.
+  DenseSet<InputSectionDescription *> isds;
+  for (const auto &[_, list] : spillLists)
+    for (SpillInputSection *s = list.head; s; s = s->next)
+      isds.insert(s->isd);
+
+  for (InputSectionDescription *isd : isds)
+    llvm::erase_if(isd->sections,
+                   [](InputSection *s) { return isa<SpillInputSection>(s); });
+
+  spillLists.clear();
+}
+
 // Creates program headers as instructed by PHDRS linker script command.
 SmallVector<PhdrEntry *, 0> LinkerScript::createPhdrs() {
   SmallVector<PhdrEntry *, 0> ret;