Feature/merge upstream 20230828

.github/workflows/repo-lockdown.yml

@@ -14,6 +14,7 @@ on:
       - 'clang/docs/**'
       - 'clang/unitests/Format/**'
       - 'clang/tools/clang-format/**'
+      - 'llvm/utils/git/**'
 
 permissions:
   pull-requests: write

bolt/CMakeLists.txt

@@ -32,10 +32,11 @@ foreach (tgt ${BOLT_TARGETS_TO_BUILD})
 endforeach()
 
 set(BOLT_ENABLE_RUNTIME_default OFF)
-if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64"
+if ((CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64"
+    OR CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
     AND (CMAKE_SYSTEM_NAME STREQUAL "Linux"
       OR CMAKE_SYSTEM_NAME STREQUAL "Darwin")
-    AND "X86" IN_LIST BOLT_TARGETS_TO_BUILD)
+    AND (NOT CMAKE_CROSSCOMPILING))
   set(BOLT_ENABLE_RUNTIME_default ON)
 endif()
 option(BOLT_ENABLE_RUNTIME "Enable BOLT runtime" ${BOLT_ENABLE_RUNTIME_default})

bolt/include/bolt/Core/AddressMap.h

@@ -0,0 +1,59 @@
+//===- bolt/Core/AddressMap.h - Input-output address map --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Helper class to create a mapping from input to output addresses needed for
+// updating debugging symbols and BAT. We emit an MCSection containing
+// <Input address, Output MCSymbol> pairs to the object file and JITLink will
+// transform this in <Input address, Output address> pairs. The linker output
+// can then be parsed and used to establish the mapping.
+//
+//===----------------------------------------------------------------------===//
+//
+#ifndef BOLT_CORE_ADDRESS_MAP_H
+#define BOLT_CORE_ADDRESS_MAP_H
+
+#include "llvm/ADT/StringRef.h"
+
+#include <optional>
+#include <unordered_map>
+
+namespace llvm {
+
+class MCStreamer;
+
+namespace bolt {
+
+class BinaryContext;
+
+class AddressMap {
+  using MapTy = std::unordered_multimap<uint64_t, uint64_t>;
+  MapTy Map;
+
+public:
+  static const char *const SectionName;
+
+  static void emit(MCStreamer &Streamer, BinaryContext &BC);
+  static AddressMap parse(StringRef Buffer, const BinaryContext &BC);
+
+  std::optional<uint64_t> lookup(uint64_t InputAddress) const {
+    auto It = Map.find(InputAddress);
+    if (It != Map.end())
+      return It->second;
+    return std::nullopt;
+  }
+
+  std::pair<MapTy::const_iterator, MapTy::const_iterator>
+  lookupAll(uint64_t InputAddress) const {
+    return Map.equal_range(InputAddress);
+  }
+};
+
+} // namespace bolt
+} // namespace llvm
+
+#endif

bolt/include/bolt/Core/BinaryBasicBlock.h

@@ -100,16 +100,6 @@ class BinaryBasicBlock {
   using LocSymsTy = std::vector<std::pair<uint32_t, const MCSymbol *>>;
   std::unique_ptr<LocSymsTy> LocSyms;
 
-  /// After output/codegen, map output offsets of instructions in this basic
-  /// block to instruction offsets in the original function. Note that the
-  /// output basic block could be different from the input basic block.
-  /// We only map instruction of interest, such as calls and markers.
-  ///
-  /// We store the offset array in a basic block to facilitate BAT tables
-  /// generation. Otherwise, the mapping could be done at function level.
-  using OffsetTranslationTableTy = std::vector<std::pair<uint32_t, uint32_t>>;
-  std::unique_ptr<OffsetTranslationTableTy> OffsetTranslationTable;
-
   /// Alignment requirements for the block.
   uint32_t Alignment{1};
 
@@ -828,8 +818,7 @@ class BinaryBasicBlock {
     return OutputAddressRange;
   }
 
-  /// Update addresses of special instructions inside this basic block.
-  void updateOutputValues(const MCAsmLayout &Layout);
+  bool hasLocSyms() const { return LocSyms != nullptr; }
 
   /// Return mapping of input offsets to symbols in the output.
   LocSymsTy &getLocSyms() {
@@ -841,19 +830,6 @@ class BinaryBasicBlock {
     return const_cast<BinaryBasicBlock *>(this)->getLocSyms();
   }
 
-  /// Return offset translation table for the basic block.
-  OffsetTranslationTableTy &getOffsetTranslationTable() {
-    return OffsetTranslationTable
-               ? *OffsetTranslationTable
-               : *(OffsetTranslationTable =
-                       std::make_unique<OffsetTranslationTableTy>());
-  }
-
-  /// Return offset translation table for the basic block.
-  const OffsetTranslationTableTy &getOffsetTranslationTable() const {
-    return const_cast<BinaryBasicBlock *>(this)->getOffsetTranslationTable();
-  }
-
   /// Return size of the basic block in the output binary.
   uint64_t getOutputSize() const {
     return OutputAddressRange.second - OutputAddressRange.first;

bolt/include/bolt/Core/BinaryContext.h

@@ -13,6 +13,7 @@
 #ifndef BOLT_CORE_BINARY_CONTEXT_H
 #define BOLT_CORE_BINARY_CONTEXT_H
 
+#include "bolt/Core/AddressMap.h"
 #include "bolt/Core/BinaryData.h"
 #include "bolt/Core/BinarySection.h"
 #include "bolt/Core/DebugData.h"
@@ -221,6 +222,9 @@ class BinaryContext {
   bool ContainsDwarf5{false};
   bool ContainsDwarfLegacy{false};
 
+  /// Mapping from input to output addresses.
+  std::optional<AddressMap> IOAddressMap;
+
   /// Preprocess DWO debug information.
   void preprocessDWODebugInfo();
 
@@ -1343,6 +1347,12 @@ class BinaryContext {
         /* DWARFMustBeAtTheEnd */ false));
     return Streamer;
   }
+
+  void setIOAddressMap(AddressMap Map) { IOAddressMap = std::move(Map); }
+  const AddressMap &getIOAddressMap() const {
+    assert(IOAddressMap && "Address map not set yet");
+    return *IOAddressMap;
+  }
 };
 
 template <typename T, typename = std::enable_if_t<sizeof(T) == 1>>

bolt/include/bolt/Core/BinaryFunction.h

@@ -366,14 +366,15 @@ class BinaryFunction {
   std::string ColdCodeSectionName;
 
   /// Parent function fragment for split function fragments.
-  SmallPtrSet<BinaryFunction *, 1> ParentFragments;
+  using FragmentsSetTy = SmallPtrSet<BinaryFunction *, 1>;
+  FragmentsSetTy ParentFragments;
 
   /// Indicate if the function body was folded into another function.
   /// Used by ICF optimization.
   BinaryFunction *FoldedIntoFunction{nullptr};
 
   /// All fragments for a parent function.
-  SmallPtrSet<BinaryFunction *, 1> Fragments;
+  FragmentsSetTy Fragments;
 
   /// The profile data for the number of times the function was executed.
   uint64_t ExecutionCount{COUNT_NO_PROFILE};
@@ -576,9 +577,6 @@ class BinaryFunction {
   /// Count the number of functions created.
   static uint64_t Count;
 
-  /// Map offsets of special instructions to addresses in the output.
-  InputOffsetToAddressMapTy InputOffsetToAddressMap;
-
   /// Register alternative function name.
   void addAlternativeName(std::string NewName) {
     Aliases.push_back(std::move(NewName));
@@ -1193,7 +1191,7 @@ class BinaryFunction {
 
     if (!Islands->FunctionConstantIslandLabel) {
       Islands->FunctionConstantIslandLabel =
-          BC.Ctx->createNamedTempSymbol("func_const_island");
+          BC.Ctx->getOrCreateSymbol("func_const_island@" + getOneName());
     }
     return Islands->FunctionConstantIslandLabel;
   }
@@ -1203,7 +1201,7 @@ class BinaryFunction {
 
     if (!Islands->FunctionColdConstantIslandLabel) {
       Islands->FunctionColdConstantIslandLabel =
-          BC.Ctx->createNamedTempSymbol("func_cold_const_island");
+          BC.Ctx->getOrCreateSymbol("func_cold_const_island@" + getOneName());
     }
     return Islands->FunctionColdConstantIslandLabel;
   }
@@ -1223,14 +1221,7 @@ class BinaryFunction {
   }
 
   /// Update output values of the function based on the final \p Layout.
-  void updateOutputValues(const MCAsmLayout &Layout);
-
-  /// Return mapping of input to output addresses. Most users should call
-  /// translateInputToOutputAddress() for address translation.
-  InputOffsetToAddressMapTy &getInputOffsetToAddressMap() {
-    assert(isEmitted() && "cannot use address mapping before code emission");
-    return InputOffsetToAddressMap;
-  }
+  void updateOutputValues(const BOLTLinker &Linker);
 
   /// Register relocation type \p RelType at a given \p Address in the function
   /// against \p Symbol.
@@ -1776,9 +1767,18 @@ class BinaryFunction {
 
   /// Returns if this function is a parent of \p Other function.
   bool isParentOf(const BinaryFunction &Other) const {
-    return llvm::is_contained(Fragments, &Other);
+    return Fragments.contains(&Other);
   }
 
+  /// Return the child fragment form parent function
+  iterator_range<FragmentsSetTy::const_iterator> getFragments() const {
+    return iterator_range<FragmentsSetTy::const_iterator>(Fragments.begin(),
+                                                          Fragments.end());
+  }
+
+  /// Return the parent function for split function fragments.
+  FragmentsSetTy *getParentFragments() { return &ParentFragments; }
+
   /// Returns if this function is a parent or child of \p Other function.
   bool isParentOrChildOf(const BinaryFunction &Other) const {
     return isChildOf(Other) || isParentOf(Other);
@@ -2170,6 +2170,11 @@ class BinaryFunction {
   /// its code emission.
   bool requiresAddressTranslation() const;
 
+  /// Return true if the linker needs to generate an address map for this
+  /// function. Used for keeping track of the mapping from input to out
+  /// addresses of basic blocks.
+  bool requiresAddressMap() const;
+
   /// Adjust branch instructions to match the CFG.
   ///
   /// As it comes to internal branches, the CFG represents "the ultimate source

bolt/include/bolt/Core/BinarySection.h

@@ -97,6 +97,8 @@ class BinarySection {
   mutable bool IsReordered{false}; // Have the contents been reordered?
   bool IsAnonymous{false};         // True if the name should not be included
                                    // in the output file.
+  bool IsLinkOnly{false};          // True if the section should not be included
+                                   // in the output file.
 
   uint64_t hash(const BinaryData &BD,
                 std::map<const BinaryData *, uint64_t> &Cache) const;
@@ -452,6 +454,8 @@ class BinarySection {
   void setIndex(uint32_t I) { Index = I; }
   void setOutputName(const Twine &Name) { OutputName = Name.str(); }
   void setAnonymous(bool Flag) { IsAnonymous = Flag; }
+  bool isLinkOnly() const { return IsLinkOnly; }
+  void setLinkOnly() { IsLinkOnly = true; }
 
   /// Emit the section as data, possibly with relocations.
   /// Use name \p SectionName for the section during the emission.

bolt/include/bolt/Core/Linker.h

@@ -31,15 +31,28 @@ class BOLTLinker {
       std::function<void(const BinarySection &Section, uint64_t Address)>;
   using SectionsMapper = std::function<void(SectionMapper)>;
 
+  struct SymbolInfo {
+    uint64_t Address;
+    uint64_t Size;
+  };
+
   virtual ~BOLTLinker() = default;
 
   /// Load and link \p Obj. \p MapSections will be called before the object is
   /// linked to allow section addresses to be remapped. When called, the address
   /// of a section can be changed by calling the passed SectionMapper.
   virtual void loadObject(MemoryBufferRef Obj, SectionsMapper MapSections) = 0;
 
+  /// Return the address and size of a symbol or std::nullopt if it cannot be
+  /// found.
+  virtual std::optional<SymbolInfo> lookupSymbolInfo(StringRef Name) const = 0;
+
   /// Return the address of a symbol or std::nullopt if it cannot be found.
-  virtual std::optional<uint64_t> lookupSymbol(StringRef Name) const = 0;
+  std::optional<uint64_t> lookupSymbol(StringRef Name) const {
+    if (const auto Info = lookupSymbolInfo(Name))
+      return Info->Address;
+    return std::nullopt;
+  }
 };
 
 } // namespace bolt