[DTLTO][LLD][ELF] Add support for Integrated Distributed ThinLTO

cross-project-tests/CMakeLists.txt

@@ -94,6 +94,13 @@ add_lit_testsuite(check-cross-amdgpu "Running AMDGPU cross-project tests"
   DEPENDS clang
   )
 
+# DTLTO tests.
+add_lit_testsuite(check-cross-dtlto "Running DTLTO cross-project tests"
+  ${CMAKE_CURRENT_BINARY_DIR}/dtlto
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS ${CROSS_PROJECT_TEST_DEPS}
+  )
+
 # Add check-cross-project-* targets.
 add_lit_testsuites(CROSS_PROJECT ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${CROSS_PROJECT_TEST_DEPS}

cross-project-tests/dtlto/README.md

@@ -0,0 +1,3 @@
+Tests for DTLTO (Integrated Distributed ThinLTO) functionality.
+
+These are integration tests as DTLTO invokes `clang` for code-generation.

cross-project-tests/dtlto/ld-dtlto.c

@@ -0,0 +1,41 @@
+// REQUIRES: ld.lld
+
+/// Simple test that DTLTO works with a single input bitcode file and that
+/// --save-temps can be applied to the remote compilation.
+
+// RUN: rm -rf %t && mkdir %t && cd %t
+
+// RUN: %clang --target=x86_64-linux-gnu -c -flto=thin %s -o dtlto.o
+
+// RUN: ld.lld dtlto.o \
+// RUN:   --thinlto-distributor=%python \
+// RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/local.py \
+// RUN:   --thinlto-remote-compiler=%clang \
+// RUN:   --thinlto-remote-compiler-arg=--save-temps
+
+/// Check that the required output files have been created.
+// RUN: ls | sort | FileCheck %s
+
+/// No files are expected before.
+// CHECK-NOT: {{.}}
+
+/// Linked ELF.
+// CHECK: {{^}}a.out{{$}}
+
+/// Produced by the bitcode compilation.
+// CHECK-NEXT: {{^}}dtlto.o{{$}}
+
+/// --save-temps output for the backend compilation.
+// CHECK-NEXT: {{^}}dtlto.s{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.0.preopt.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.1.promote.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.2.internalize.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.3.import.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.4.opt.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.5.precodegen.bc{{$}}
+// CHECK-NEXT: {{^}}dtlto.s.resolution.txt{{$}}
+
+/// No files are expected after.
+// CHECK-NOT: {{.}}
+
+int _start() { return 0; }

cross-project-tests/dtlto/lit.local.cfg

@@ -0,0 +1,5 @@
+if any(
+    f not in config.available_features
+    for f in ("clang", "x86-registered-target")
+):
+    config.unsupported = True

lld/ELF/Config.h

@@ -275,6 +275,10 @@ struct Config {
   llvm::SmallVector<llvm::StringRef, 0> searchPaths;
   llvm::SmallVector<llvm::StringRef, 0> symbolOrderingFile;
   llvm::SmallVector<llvm::StringRef, 0> thinLTOModulesToCompile;
+  llvm::StringRef dtltoDistributor;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoDistributorArgs;
+  llvm::StringRef dtltoCompiler;
+  llvm::SmallVector<llvm::StringRef, 0> dtltoCompilerArgs;
   llvm::SmallVector<llvm::StringRef, 0> undefined;
   llvm::SmallVector<SymbolVersion, 0> dynamicList;
   llvm::SmallVector<uint8_t, 0> buildIdVector;

lld/ELF/Driver.cpp

@@ -1396,6 +1396,11 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
       args.hasFlag(OPT_dependent_libraries, OPT_no_dependent_libraries, true);
   ctx.arg.disableVerify = args.hasArg(OPT_disable_verify);
   ctx.arg.discard = getDiscard(args);
+  ctx.arg.dtltoDistributor = args.getLastArgValue(OPT_thinlto_distributor_eq);
+  ctx.arg.dtltoDistributorArgs =
+      args::getStrings(args, OPT_thinlto_distributor_arg);
+  ctx.arg.dtltoCompiler = args.getLastArgValue(OPT_thinlto_compiler_eq);
+  ctx.arg.dtltoCompilerArgs = args::getStrings(args, OPT_thinlto_compiler_arg);
   ctx.arg.dwoDir = args.getLastArgValue(OPT_plugin_opt_dwo_dir_eq);
   ctx.arg.dynamicLinker = getDynamicLinker(ctx, args);
   ctx.arg.ehFrameHdr =

lld/ELF/LTO.cpp

@@ -180,6 +180,13 @@ BitcodeCompiler::BitcodeCompiler(Ctx &ctx) : ctx(ctx) {
         std::string(ctx.arg.thinLTOPrefixReplaceNew),
         std::string(ctx.arg.thinLTOPrefixReplaceNativeObject),
         ctx.arg.thinLTOEmitImportsFiles, indexFile.get(), onIndexWrite);
+  } else if (!ctx.arg.dtltoDistributor.empty()) {
+    backend = lto::createOutOfProcessThinBackend(
+        llvm::hardware_concurrency(ctx.arg.thinLTOJobs), onIndexWrite,
+        ctx.arg.thinLTOEmitIndexFiles, ctx.arg.thinLTOEmitImportsFiles,
+        ctx.arg.outputFile, ctx.arg.dtltoDistributor,
+        ctx.arg.dtltoDistributorArgs, ctx.arg.dtltoCompiler,
+        ctx.arg.dtltoCompilerArgs, !ctx.arg.saveTempsArgs.empty());
   } else {
     backend = lto::createInProcessThinBackend(
         llvm::heavyweight_hardware_concurrency(ctx.arg.thinLTOJobs),

lld/ELF/Options.td

@@ -710,7 +710,17 @@ def thinlto_object_suffix_replace_eq: JJ<"thinlto-object-suffix-replace=">;
 def thinlto_prefix_replace_eq: JJ<"thinlto-prefix-replace=">;
 def thinlto_single_module_eq: JJ<"thinlto-single-module=">,
   HelpText<"Specify a single module to compile in ThinLTO mode, for debugging only">;
-
+def thinlto_distributor_eq: JJ<"thinlto-distributor=">,
+  HelpText<"Distributor to use for ThinLTO backend compilations. If specified, "
+  "ThinLTO backend compilations will be distributed">;
+defm thinlto_distributor_arg: EEq<"thinlto-distributor-arg", "Arguments to "
+  "pass to the ThinLTO distributor">;
+def thinlto_compiler_eq: JJ<"thinlto-remote-compiler=">,
+  HelpText<"Compiler for the ThinLTO distributor to invoke for ThinLTO backend "
+  "compilations">;
+defm thinlto_compiler_arg: EEq<"thinlto-remote-compiler-arg", "Compiler "
+  "arguments for the ThinLTO distributor to pass for ThinLTO backend "
+  "compilations">;
 defm fat_lto_objects: BB<"fat-lto-objects",
     "Use the .llvm.lto section, which contains LLVM bitcode, in fat LTO object files to perform LTO.",
     "Ignore the .llvm.lto section in relocatable object files (default).">;

lld/docs/DTLTO.rst

@@ -0,0 +1,42 @@
+Integrated Distributed ThinLTO (DTLTO)
+======================================
+
+Integrated Distributed ThinLTO (DTLTO) enables the distribution of backend
+ThinLTO compilations via external distribution systems, such as Incredibuild,
+during the traditional link step.
+
+The implementation is documented here: https://llvm.org/docs/DTLTO.html.
+
+Currently, DTLTO is only supported in ELF LLD. Support will be added to other
+LLD flavours in the future.
+
+ELF LLD
+-------
+
+The command-line interface is as follows:
+
+- ``--thinlto-distributor=<path>``  
+  Specifies the file to execute as the distributor process. If specified,
+  ThinLTO backend compilations will be distributed.
+
+- ``--thinlto-remote-compiler=<path>``  
+  Specifies the path to the compiler that the distributor process will use for
+  backend compilations. The compiler invoked must match the version of LLD.
+
+- ``--thinlto-distributor-arg=<arg>``  
+  Specifies ``<arg>`` on the command line when invoking the distributor.
+  Can be specified multiple times.
+
+- ``--thinlto-remote-compiler-arg=<arg>``  
+  Appends ``<arg>`` to the remote compiler's command line.
+  Can be specified multiple times.
+
+  Options that introduce extra input/output files may cause miscompilation if
+  the distribution system does not automatically handle pushing/fetching them to
+  remote nodes. In such cases, configure the distributor - possibly using
+  ``--thinlto-distributor-arg=`` - to manage these dependencies. See the
+  distributor documentation for details.
+
+Some LLD LTO options (e.g., ``--lto-sample-profile=<file>``) are supported.
+Currently, other options are silently accepted but do not have the intended
+effect. Support for such options will be expanded in the future.

lld/docs/index.rst

@@ -147,3 +147,4 @@ document soon.
    ELF/start-stop-gc
    ELF/warn_backrefs
    MachO/index
+   DTLTO

lld/test/ELF/dtlto/files.test

@@ -0,0 +1,99 @@
+# REQUIRES: x86
+
+## Test that the LLD options --save-temps, --thinlto-emit-index-files,
+## and --thinlto-emit-imports-files function correctly with DTLTO.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: sed 's/@t1/@t2/g' t1.ll > t2.ll
+
+## Generate ThinLTO bitcode files. Note that t3.bc will not be used by the
+## linker.
+RUN: opt -thinlto-bc t1.ll -o t1.bc
+RUN: opt -thinlto-bc t2.ll -o t2.bc
+RUN: cp t1.bc t3.bc
+
+## Generate object files for mock.py to return.
+RUN: llc t1.ll --filetype=obj -o t1.o
+RUN: llc t2.ll --filetype=obj -o t2.o
+
+## Create response file containing shared ThinLTO linker arguments.
+## --start-lib/--end-lib is used to test the special case where unused lazy
+## bitcode inputs result in empty index/imports files.
+## Note that mock.py does not do any compilation; instead, it simply writes
+## the contents of the object files supplied on the command line into the
+## output object files in job order.
+RUN: echo "t1.bc t2.bc --start-lib t3.bc --end-lib -o my.elf \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \
+RUN:   --thinlto-distributor-arg=t1.o \
+RUN:   --thinlto-distributor-arg=t2.o" > l.rsp
+
+## Check that without extra flags, no index/imports files are produced and
+## backend temp files are removed.
+RUN: ld.lld @l.rsp
+RUN: ls | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,NOINDEXFILES,NOIMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that index files are created with --thinlto-emit-index-files.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --thinlto-emit-index-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,INDEXFILES,NOIMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that imports files are created with --thinlto-emit-imports-files.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --thinlto-emit-imports-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,NOINDEXFILES,IMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that both index and imports files are emitted with both flags.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --thinlto-emit-index-files \
+RUN:   --thinlto-emit-imports-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=NOBACKEND,INDEXFILES,IMPORTSFILES,EMPTYIMPORTS
+
+## Check that backend temp files are retained with --save-temps.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --save-temps
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=BACKEND,NOINDEXFILES,NOIMPORTSFILES,NOEMPTYIMPORTS
+
+## Check that all files are emitted when all options are enabled.
+RUN: rm -f *.imports *.thinlto.bc
+RUN: ld.lld @l.rsp --save-temps --thinlto-emit-index-files \
+RUN:   --thinlto-emit-imports-files
+RUN: ls | sort | FileCheck %s \
+RUN:   --check-prefixes=BACKEND,INDEXFILES,IMPORTSFILES,EMPTYIMPORTS
+
+## JSON jobs description, retained with --save-temps.
+## Note that DTLTO temporary files include a PID component.
+NOBACKEND-NOT:      {{^}}my.[[#]].dist-file.json{{$}}
+BACKEND:            {{^}}my.[[#]].dist-file.json{{$}}
+
+## Index/imports files for t1.bc.
+NOIMPORTSFILES-NOT: {{^}}t1.bc.imports{{$}}
+IMPORTSFILES:       {{^}}t1.bc.imports{{$}}
+NOINDEXFILES-NOT:   {{^}}t1.bc.thinlto.bc{{$}}
+INDEXFILES:         {{^}}t1.bc.thinlto.bc{{$}}
+
+## Index/imports files for t2.bc.
+NOIMPORTSFILES-NOT: {{^}}t2.bc.imports{{$}}
+IMPORTSFILES:       {{^}}t2.bc.imports{{$}}
+NOINDEXFILES-NOT:   {{^}}t2.bc.thinlto.bc{{$}}
+INDEXFILES:         {{^}}t2.bc.thinlto.bc{{$}}
+
+## Empty index/imports files for unused t3.bc.
+NOEMPTYIMPORTS-NOT: {{^}}t3.bc.imports{{$}}
+EMPTYIMPORTS:       {{^}}t3.bc.imports{{$}}
+NOINDEXFILES-NOT:   {{^}}t3.bc.thinlto.bc{{$}}
+INDEXFILES:         {{^}}t3.bc.thinlto.bc{{$}}
+
+#--- t1.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @t1() {
+  ret void
+}

lld/test/ELF/dtlto/options.test

@@ -0,0 +1,40 @@
+# REQUIRES: x86
+
+## Test that DTLTO options are passed correctly to the distributor and
+## remote compiler.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: opt -thinlto-bc foo.ll -o foo.o
+
+## Note: validate.py does not perform any compilation. Instead, it validates the
+## received JSON, pretty-prints the JSON and the supplied arguments, and then
+## exits with an error. This allows FileCheck directives to verify the
+## distributor inputs.
+RUN: not ld.lld foo.o \
+RUN:   -o my.elf \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/validate.py \
+RUN:   --thinlto-distributor-arg=darg1=10 \
+RUN:   --thinlto-distributor-arg=darg2=20 \
+RUN:   --thinlto-remote-compiler=my_clang.exe \
+RUN:   --thinlto-remote-compiler-arg=carg1=20 \
+RUN:   --thinlto-remote-compiler-arg=carg2=30 2>&1 | FileCheck %s
+
+CHECK: distributor_args=['darg1=10', 'darg2=20']
+
+CHECK: "linker_output": "my.elf"
+
+CHECK: "my_clang.exe"
+CHECK: "carg1=20"
+CHECK: "carg2=30"
+
+CHECK: error: DTLTO backend compilation: cannot open native object file:
+
+#--- foo.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @foo() {
+  ret void
+}

lld/test/ELF/dtlto/partitions.test

@@ -0,0 +1,40 @@
+# REQUIRES: x86
+
+## Test that DTLTO works with more than one LTO partition.
+
+RUN: rm -rf %t && split-file %s %t && cd %t
+
+RUN: sed 's/@f/@t1/g' f.ll > t1.ll
+RUN: sed 's/@f/@t2/g' f.ll > t2.ll
+
+## Generate bitcode.
+RUN: opt f.ll -o full.bc
+RUN: opt -thinlto-bc t1.ll -o thin1.bc
+RUN: opt -thinlto-bc t2.ll -o thin2.bc
+
+## Generate object files for mock.py to return.
+RUN: llc t1.ll --filetype=obj -o thin1.o
+RUN: llc t2.ll --filetype=obj -o thin2.o
+
+## Link with 3 LTO partitions.
+RUN: ld.lld full.bc thin1.bc thin2.bc \
+RUN:   --thinlto-distributor=%python \
+RUN:   --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \
+RUN:   --thinlto-distributor-arg=thin1.o \
+RUN:   --thinlto-distributor-arg=thin2.o \
+RUN:   --save-temps \
+RUN:   --lto-partitions=3
+
+## DTLTO temporary object files include the task number and a PID component. The
+## task number should incorporate the LTO partition number.
+RUN: ls | sort | FileCheck %s
+CHECK: {{^}}thin1.3.[[PID:[a-zA-Z0-9_]+]].native.o{{$}}
+CHECK: {{^}}thin2.4.[[PID]].native.o{{$}}
+
+#--- f.ll
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f() {
+  ret void
+}

lld/test/lit.cfg.py

@@ -36,6 +36,7 @@
 
 llvm_config.use_default_substitutions()
 llvm_config.use_lld()
+config.substitutions.append(("%llvm_src_root", config.llvm_src_root))
 
 tool_patterns = [
     "llc",