From 57361b762736c1921c4f52313812ed4ef03f3bdc Mon Sep 17 00:00:00 2001 From: Prem Chintalapudi Date: Wed, 26 Jul 2023 13:40:31 -0400 Subject: [PATCH 1/4] Respect linkage type in jl_merge_module --- src/codegen.cpp | 5 ++- src/jitlayers.cpp | 83 ++++++++++++++++++++++++----------------------- 2 files changed, 47 insertions(+), 41 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 164e59ed75421..bc580a7a0e044 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1796,8 +1796,11 @@ static inline GlobalVariable *prepare_global_in(Module *M, GlobalVariable *G) if (!local) { // Copy the GlobalVariable, but without the initializer, so it becomes a declaration GlobalVariable *proto = new GlobalVariable(*M, G->getValueType(), - G->isConstant(), GlobalVariable::ExternalLinkage, + G->isConstant(), G->getLinkage(), nullptr, G->getName(), nullptr, G->getThreadLocalMode()); + if (proto->hasLocalLinkage()) { + proto->setInitializer(G->getInitializer()); + } proto->copyAttributesFrom(G); // DLLImport only needs to be set for the shadow module // it just gets annoying in the JIT diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 2ba1ad35ec9ec..f9c8de05e4091 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1864,21 +1864,22 @@ void jl_merge_module(orc::ThreadSafeModule &destTSM, orc::ThreadSafeModule srcTS assert(dest.getDataLayout() == src.getDataLayout() && "Cannot merge modules with different data layouts!"); assert(dest.getTargetTriple() == src.getTargetTriple() && "Cannot merge modules with different target triples!"); - for (Module::global_iterator I = src.global_begin(), E = src.global_end(); I != E;) { - GlobalVariable *sG = &*I; - GlobalVariable *dG = cast_or_null(dest.getNamedValue(sG->getName())); - ++I; + for (auto &SG : make_early_inc_range(src.globals())) { + GlobalVariable *dG = cast_or_null(dest.getNamedValue(SG.getName())); + if (SG.hasLocalLinkage()) { + dG = nullptr; + } // Replace a declaration with the definition: - if (dG) { - if (sG->isDeclaration()) { - sG->replaceAllUsesWith(dG); - sG->eraseFromParent(); + if (dG && !dG->hasLocalLinkage()) { + if (SG.isDeclaration()) { + SG.replaceAllUsesWith(dG); + SG.eraseFromParent(); continue; } //// If we start using llvm.used, we need to enable and test this - //else if (!dG->isDeclaration() && dG->hasAppendingLinkage() && sG->hasAppendingLinkage()) { + //else if (!dG->isDeclaration() && dG->hasAppendingLinkage() && SG.hasAppendingLinkage()) { // auto *dCA = cast(dG->getInitializer()); - // auto *sCA = cast(sG->getInitializer()); + // auto *sCA = cast(SG.getInitializer()); // SmallVector Init; // for (auto &Op : dCA->operands()) // Init.push_back(cast_or_null(Op)); @@ -1890,67 +1891,69 @@ void jl_merge_module(orc::ThreadSafeModule &destTSM, orc::ThreadSafeModule srcTS // GlobalValue::AppendingLinkage, ConstantArray::get(ATy, Init), "", // dG->getThreadLocalMode(), dG->getType()->getAddressSpace()); // GV->copyAttributesFrom(dG); - // sG->replaceAllUsesWith(GV); + // SG.replaceAllUsesWith(GV); // dG->replaceAllUsesWith(GV); - // GV->takeName(sG); - // sG->eraseFromParent(); + // GV->takeName(SG); + // SG.eraseFromParent(); // dG->eraseFromParent(); // continue; //} else { - assert(dG->isDeclaration() || dG->getInitializer() == sG->getInitializer()); - dG->replaceAllUsesWith(sG); + assert(dG->isDeclaration() || dG->getInitializer() == SG.getInitializer()); + dG->replaceAllUsesWith(&SG); dG->eraseFromParent(); } } // Reparent the global variable: - sG->removeFromParent(); - dest.getGlobalList().push_back(sG); + SG.removeFromParent(); + dest.getGlobalList().push_back(&SG); // Comdat is owned by the Module - sG->setComdat(nullptr); + SG.setComdat(nullptr); } - for (Module::iterator I = src.begin(), E = src.end(); I != E;) { - Function *sG = &*I; - Function *dG = cast_or_null(dest.getNamedValue(sG->getName())); - ++I; + for (auto &SG : make_early_inc_range(src)) { + Function *dG = cast_or_null(dest.getNamedValue(SG.getName())); + if (SG.hasLocalLinkage()) { + dG = nullptr; + } // Replace a declaration with the definition: - if (dG) { - if (sG->isDeclaration()) { - sG->replaceAllUsesWith(dG); - sG->eraseFromParent(); + if (dG && !dG->hasLocalLinkage()) { + if (SG.isDeclaration()) { + SG.replaceAllUsesWith(dG); + SG.eraseFromParent(); continue; } else { assert(dG->isDeclaration()); - dG->replaceAllUsesWith(sG); + dG->replaceAllUsesWith(&SG); dG->eraseFromParent(); } } // Reparent the global variable: - sG->removeFromParent(); - dest.getFunctionList().push_back(sG); + SG.removeFromParent(); + dest.getFunctionList().push_back(&SG); // Comdat is owned by the Module - sG->setComdat(nullptr); + SG.setComdat(nullptr); } - for (Module::alias_iterator I = src.alias_begin(), E = src.alias_end(); I != E;) { - GlobalAlias *sG = &*I; - GlobalAlias *dG = cast_or_null(dest.getNamedValue(sG->getName())); - ++I; - if (dG) { + for (auto &SG : make_early_inc_range(src.aliases())) { + GlobalAlias *dG = cast_or_null(dest.getNamedValue(SG.getName())); + if (SG.hasLocalLinkage()) { + dG = nullptr; + } + if (dG && !dG->hasLocalLinkage()) { if (!dG->isDeclaration()) { // aliases are always definitions, so this test is reversed from the above two - sG->replaceAllUsesWith(dG); - sG->eraseFromParent(); + SG.replaceAllUsesWith(dG); + SG.eraseFromParent(); continue; } else { - dG->replaceAllUsesWith(sG); + dG->replaceAllUsesWith(&SG); dG->eraseFromParent(); } } - sG->removeFromParent(); - dest.getAliasList().push_back(sG); + SG.removeFromParent(); + dest.getAliasList().push_back(&SG); } // metadata nodes need to be explicitly merged not just copied From 7bffed324bc4790c93d88d80274efd679d367834 Mon Sep 17 00:00:00 2001 From: Prem Chintalapudi Date: Mon, 31 Jul 2023 18:59:43 -0500 Subject: [PATCH 2/4] Delay JIT-ting dlsym lookups until actually committing to entering the JIT --- src/aotcompile.cpp | 3 + src/ccall.cpp | 104 ++++++++++++++-------------- src/codegen.cpp | 2 + src/jitlayers.cpp | 167 ++++++++++++++++++++++++++++++++++++++++++++- src/jitlayers.h | 4 ++ 5 files changed, 225 insertions(+), 55 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index c04f1954ce1e4..7324278557179 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -2144,6 +2144,9 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t* dump, jl_method_instance_t *mi, siz global.second->setVisibility(GlobalValue::DefaultVisibility); } } + if (!jl_options.image_codegen) { + optimizeDLSyms(*m.getModuleUnlocked()); + } assert(!verifyLLVMIR(*m.getModuleUnlocked())); if (optimize) { #ifndef JL_USE_NEW_PM diff --git a/src/ccall.cpp b/src/ccall.cpp index 6e40071f9525a..0354d51ab6d88 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -170,8 +170,15 @@ static Value *runtime_sym_lookup( // f_lib is actually one of the special sentinel values libname = ConstantExpr::getIntToPtr(ConstantInt::get(emission_context.DL.getIntPtrType(irbuilder.getContext()), (uintptr_t)f_lib), getInt8PtrTy(irbuilder.getContext())); } - llvmf = irbuilder.CreateCall(prepare_call_in(jl_builderModule(irbuilder), jldlsym_func), + auto lookup = irbuilder.CreateCall(prepare_call_in(jl_builderModule(irbuilder), jldlsym_func), { libname, nameval, libptrgv }); + if (runtime_lib) { + lookup->addFnAttr(Attribute::get(lookup->getContext(), "julia.libname", f_lib)); + } else { + lookup->addFnAttr(Attribute::get(lookup->getContext(), "julia.libidx", std::to_string((uintptr_t) f_lib))); + } + lookup->addFnAttr(Attribute::get(lookup->getContext(), "julia.fname", f_name)); + llvmf = lookup; } setName(emission_context, llvmf, f_name + StringRef(".found")); StoreInst *store = irbuilder.CreateAlignedStore(llvmf, llvmgv, Align(sizeof(void*))); @@ -187,17 +194,6 @@ static Value *runtime_sym_lookup( return irbuilder.CreateBitCast(p, funcptype); } -static Value *runtime_sym_lookup( - jl_codectx_t &ctx, - PointerType *funcptype, const char *f_lib, jl_value_t *lib_expr, - const char *f_name, Function *f, - GlobalVariable *libptrgv, - GlobalVariable *llvmgv, bool runtime_lib) -{ - return runtime_sym_lookup(ctx.emission_context, ctx.builder, &ctx, funcptype, f_lib, lib_expr, - f_name, f, libptrgv, llvmgv, runtime_lib); -} - static Value *runtime_sym_lookup( jl_codectx_t &ctx, PointerType *funcptype, const char *f_lib, jl_value_t *lib_expr, @@ -225,7 +221,7 @@ static Value *runtime_sym_lookup( libptrgv = prepare_global_in(jl_Module, libptrgv); } llvmgv = prepare_global_in(jl_Module, llvmgv); - return runtime_sym_lookup(ctx, funcptype, f_lib, lib_expr, f_name, f, libptrgv, llvmgv, runtime_lib); + return runtime_sym_lookup(ctx.emission_context, ctx.builder, &ctx, funcptype, f_lib, lib_expr, f_name, f, libptrgv, llvmgv, runtime_lib); } // Emit a "PLT" entry that will be lazily initialized @@ -250,12 +246,17 @@ static GlobalVariable *emit_plt_thunk( plt->setAttributes(attrs); if (cc != CallingConv::C) plt->setCallingConv(cc); - fname += "_got"; auto T_pvoidfunc = JuliaType::get_pvoidfunc_ty(M->getContext()); GlobalVariable *got = new GlobalVariable(*M, T_pvoidfunc, false, GlobalVariable::ExternalLinkage, ConstantExpr::getBitCast(plt, T_pvoidfunc), - fname); + fname + "_got"); + if (runtime_lib) { + got->addAttribute("julia.libname", f_lib); + } else { + got->addAttribute("julia.libidx", std::to_string((uintptr_t) f_lib)); + } + got->addAttribute("julia.fname", f_name); BasicBlock *b0 = BasicBlock::Create(M->getContext(), "top", plt); IRBuilder<> irbuilder(b0); Value *ptr = runtime_sym_lookup(ctx.emission_context, irbuilder, NULL, funcptype, f_lib, NULL, f_name, plt, libptrgv, @@ -263,8 +264,8 @@ static GlobalVariable *emit_plt_thunk( StoreInst *store = irbuilder.CreateAlignedStore(irbuilder.CreateBitCast(ptr, T_pvoidfunc), got, Align(sizeof(void*))); store->setAtomic(AtomicOrdering::Release); SmallVector args; - for (Function::arg_iterator arg = plt->arg_begin(), arg_e = plt->arg_end(); arg != arg_e; ++arg) - args.push_back(&*arg); + for (auto &arg : plt->args()) + args.push_back(&arg); assert(cast(ptr->getType())->isOpaqueOrPointeeTypeMatches(functype)); CallInst *ret = irbuilder.CreateCall( functype, @@ -307,7 +308,6 @@ static Value *emit_plt( CallingConv::ID cc, const char *f_lib, const char *f_name) { ++PLT; - assert(ctx.emission_context.imaging); // Don't do this for vararg functions so that the `musttail` is only // an optimization and is not required to function correctly. assert(!functype->isVarArg()); @@ -724,26 +724,26 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg if (sym.lib_expr) { res = runtime_sym_lookup(ctx, cast(getInt8PtrTy(ctx.builder.getContext())), NULL, sym.lib_expr, sym.f_name, ctx.f); } - else if (ctx.emission_context.imaging) { + else /*if (ctx.emission_context.imaging) */{ res = runtime_sym_lookup(ctx, cast(getInt8PtrTy(ctx.builder.getContext())), sym.f_lib, NULL, sym.f_name, ctx.f); res = ctx.builder.CreatePtrToInt(res, lrt); } - else { - void *symaddr; - - void* libsym = jl_get_library_(sym.f_lib, 0); - int symbol_found = jl_dlsym(libsym, sym.f_name, &symaddr, 0); - if (!libsym || !symbol_found) { - // Error mode, either the library or the symbol couldn't be find during compiletime. - // Fallback to a runtime symbol lookup. - res = runtime_sym_lookup(ctx, cast(getInt8PtrTy(ctx.builder.getContext())), sym.f_lib, NULL, sym.f_name, ctx.f); - res = ctx.builder.CreatePtrToInt(res, lrt); - } else { - // since we aren't saving this code, there's no sense in - // putting anything complicated here: just JIT the address of the cglobal - res = ConstantInt::get(lrt, (uint64_t)symaddr); - } - } + // else { + // void *symaddr; + + // void* libsym = jl_get_library_(sym.f_lib, 0); + // int symbol_found = jl_dlsym(libsym, sym.f_name, &symaddr, 0); + // if (!libsym || !symbol_found) { + // // Error mode, either the library or the symbol couldn't be find during compiletime. + // // Fallback to a runtime symbol lookup. + // res = runtime_sym_lookup(ctx, cast(getInt8PtrTy(ctx.builder.getContext())), sym.f_lib, NULL, sym.f_name, ctx.f); + // res = ctx.builder.CreatePtrToInt(res, lrt); + // } else { + // // since we aren't saving this code, there's no sense in + // // putting anything complicated here: just JIT the address of the cglobal + // res = ConstantInt::get(lrt, (uint64_t)symaddr); + // } + // } } JL_GC_POP(); @@ -2106,7 +2106,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( ++DeferredCCallLookups; llvmf = runtime_sym_lookup(ctx, funcptype, NULL, symarg.lib_expr, symarg.f_name, ctx.f); } - else if (ctx.emission_context.imaging) { + else /*if (ctx.emission_context.imaging) */{ ++DeferredCCallLookups; // vararg requires musttail, // but musttail is incompatible with noreturn. @@ -2115,22 +2115,22 @@ jl_cgval_t function_sig_t::emit_a_ccall( else llvmf = emit_plt(ctx, functype, attributes, cc, symarg.f_lib, symarg.f_name); } - else { - void *symaddr; - void *libsym = jl_get_library_(symarg.f_lib, 0); - int symbol_found = jl_dlsym(libsym, symarg.f_name, &symaddr, 0); - if (!libsym || !symbol_found) { - ++DeferredCCallLookups; - // either the library or the symbol could not be found, place a runtime - // lookup here instead. - llvmf = runtime_sym_lookup(ctx, funcptype, symarg.f_lib, NULL, symarg.f_name, ctx.f); - } else { - ++LiteralCCalls; - // since we aren't saving this code, there's no sense in - // putting anything complicated here: just JIT the function address - llvmf = literal_static_pointer_val(symaddr, funcptype); - } - } + // else { + // void *symaddr; + // void *libsym = jl_get_library_(symarg.f_lib, 0); + // int symbol_found = jl_dlsym(libsym, symarg.f_name, &symaddr, 0); + // if (!libsym || !symbol_found) { + // ++DeferredCCallLookups; + // // either the library or the symbol could not be found, place a runtime + // // lookup here instead. + // llvmf = runtime_sym_lookup(ctx, funcptype, symarg.f_lib, NULL, symarg.f_name, ctx.f); + // } else { + // ++LiteralCCalls; + // // since we aren't saving this code, there's no sense in + // // putting anything complicated here: just JIT the function address + // llvmf = literal_static_pointer_val(symaddr, funcptype); + // } + // } } OperandBundleDef OpBundle("jl_roots", gc_uses); diff --git a/src/codegen.cpp b/src/codegen.cpp index bc580a7a0e044..5a8c9e15936ce 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2366,6 +2366,8 @@ std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &conte if (!m->getModuleFlag("Debug Info Version")) m->addModuleFlag(llvm::Module::Warning, "Debug Info Version", llvm::DEBUG_METADATA_VERSION); + if (imaging_mode) + m->addModuleFlag(llvm::Module::Error, "julia.imaging_mode", 1); m->setDataLayout(DL); m->setTargetTriple(triple.str()); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index f9c8de05e4091..70502336ad2d2 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1404,9 +1404,9 @@ namespace { struct JITPointersT { - JITPointersT(orc::ExecutionSession &ES) : ES(ES) {} + JITPointersT(orc::ExecutionSession &ES) JL_NOTSAFEPOINT : ES(ES) {} - Expected operator()(orc::ThreadSafeModule TSM, orc::MaterializationResponsibility &R) { + Expected operator()(orc::ThreadSafeModule TSM, orc::MaterializationResponsibility &R) JL_NOTSAFEPOINT { TSM.withModuleDo([&](Module &M) { for (auto &GV : make_early_inc_range(M.globals())) { if (auto *Shared = getSharedBytes(GV)) { @@ -1458,6 +1458,166 @@ namespace { orc::ExecutionSession &ES; }; + + struct DLSymOptimizer { + DLSymOptimizer(bool named) JL_NOTSAFEPOINT { + this->named = named; + symbols_mutex = std::make_unique(); +#define INIT_RUNTIME_LIBRARY(libname, handle) \ + do { \ + auto libidx = (uintptr_t) libname; \ + if (libidx >= runtime_symbols.size()) { \ + runtime_symbols.resize(libidx + 1); \ + } \ + runtime_symbols[libidx].first = handle; \ + } while (0) + + INIT_RUNTIME_LIBRARY(NULL, jl_RTLD_DEFAULT_handle); + INIT_RUNTIME_LIBRARY(JL_EXE_LIBNAME, jl_exe_handle); + INIT_RUNTIME_LIBRARY(JL_LIBJULIA_INTERNAL_DL_LIBNAME, jl_libjulia_internal_handle); + INIT_RUNTIME_LIBRARY(JL_LIBJULIA_DL_LIBNAME, jl_libjulia_handle); + +#undef INIT_RUNTIME_LIBRARY + } + + void *lookup_symbol(void *libhandle, const char *fname) JL_NOTSAFEPOINT { + void *addr; + jl_dlsym(libhandle, fname, &addr, 0); + return addr; + } + + void *lookup(const char *libname, const char *fname) JL_NOTSAFEPOINT { + StringRef lib(libname); + StringRef f(fname); + std::lock_guard lock(*symbols_mutex); + auto uit = user_symbols.find(lib); + if (uit == user_symbols.end()) { + void *handle = jl_get_library_(libname, 0); + if (!handle) + return nullptr; + uit = user_symbols.insert(std::make_pair(lib, std::make_pair(handle, StringMap()))).first; + } + auto &symmap = uit->second.second; + auto it = symmap.find(f); + if (it != symmap.end()) { + return it->second; + } + void *handle = lookup_symbol(uit->second.first, fname); + symmap[f] = handle; + return handle; + } + + void *lookup(uintptr_t libidx, const char *fname) JL_NOTSAFEPOINT { + std::lock_guard lock(*symbols_mutex); + runtime_symbols.resize(std::max(runtime_symbols.size(), libidx + 1)); + auto it = runtime_symbols[libidx].second.find(fname); + if (it != runtime_symbols[libidx].second.end()) { + return it->second; + } + auto handle = lookup_symbol(runtime_symbols[libidx].first, fname); + runtime_symbols[libidx].second[fname] = handle; + return handle; + } + + void operator()(Module &M) JL_NOTSAFEPOINT { + for (auto &GV : M.globals()) { + auto Name = GV.getName(); + if (Name.startswith("jlplt") && Name.endswith("got")) { + auto fname = GV.getAttribute("julia.fname").getValueAsString().str(); + void *addr; + if (GV.hasAttribute("julia.libname")) { + auto libname = GV.getAttribute("julia.libname").getValueAsString().str(); + addr = lookup(libname.data(), fname.data()); + } else { + assert(GV.hasAttribute("julia.libidx") && "PLT entry should have either libname or libidx attribute!"); + auto libidx = (uintptr_t)std::stoull(GV.getAttribute("julia.libidx").getValueAsString().str()); + addr = lookup(libidx, fname.data()); + } + if (addr) { + Function *Thunk = nullptr; + if (!GV.isDeclaration()) { + Thunk = cast(GV.getInitializer()->stripPointerCasts()); + assert(++Thunk->uses().begin() == Thunk->uses().end() && "Thunk should only have one use in PLT initializer!"); + assert(Thunk->hasLocalLinkage() && "Thunk should not have non-local linkage!"); + } else { + GV.setLinkage(GlobalValue::PrivateLinkage); + } + auto init = ConstantExpr::getIntToPtr(ConstantInt::get(M.getDataLayout().getIntPtrType(M.getContext()), (uintptr_t)addr), GV.getValueType()); + if (named) { + auto T = GV.getValueType(); + assert(T->isPointerTy()); + if (!T->isOpaquePointerTy()) { + T = T->getNonOpaquePointerElementType(); + } + init = GlobalAlias::create(T, 0, GlobalValue::PrivateLinkage, GV.getName() + ".jit", init, &M); + } + GV.setInitializer(init); + GV.setConstant(true); + GV.setUnnamedAddr(GlobalValue::UnnamedAddr::Global); + if (Thunk) { + Thunk->eraseFromParent(); + } + } + } + } + + for (auto &F : M) { + for (auto &BB : F) { + for (auto &I : make_early_inc_range(BB)) { + auto CI = dyn_cast(&I); + if (!CI) + continue; + auto Callee = CI->getCalledFunction(); + if (!Callee || Callee->getName() != XSTR(jl_load_and_lookup)) + continue; + auto fname = CI->getFnAttr("julia.fname").getValueAsString().str(); + void *addr; + if (CI->hasFnAttr("julia.libname")) { + auto libname = CI->getFnAttr("julia.libname").getValueAsString().str(); + addr = lookup(libname.data(), fname.data()); + } else { + assert(CI->hasFnAttr("julia.libidx") && "PLT entry should have either libname or libidx attribute!"); + auto libidx = (uintptr_t)std::stoull(CI->getFnAttr("julia.libidx").getValueAsString().str()); + addr = lookup(libidx, fname.data()); + } + if (addr) { + auto init = ConstantExpr::getIntToPtr(ConstantInt::get(M.getDataLayout().getIntPtrType(M.getContext()), (uintptr_t)addr), CI->getType()); + if (named) { + auto T = CI->getType(); + assert(T->isPointerTy()); + if (!T->isOpaquePointerTy()) { + T = T->getNonOpaquePointerElementType(); + } + init = GlobalAlias::create(T, 0, GlobalValue::PrivateLinkage, CI->getName() + ".jit", init, &M); + } + CI->replaceAllUsesWith(init); + CI->eraseFromParent(); + } + } + } + } + } + + Expected operator()(orc::ThreadSafeModule TSM, orc::MaterializationResponsibility &R) JL_NOTSAFEPOINT { + TSM.withModuleDo([this](Module &M) JL_NOTSAFEPOINT { + // Don't optimize these out in imaging mode + if (M.getModuleFlag("julia.imaging_mode")) + return; + (*this)(M); + }); + + return std::move(TSM); + } + + std::unique_ptr symbols_mutex; + StringMap>> user_symbols; + SmallVector>> runtime_symbols; + bool named; + }; +} + +void optimizeDLSyms(Module &M) JL_NOTSAFEPOINT { + DLSymOptimizer(true)(M); } llvm::DataLayout jl_create_datalayout(TargetMachine &TM) { @@ -1499,7 +1659,8 @@ JuliaOJIT::JuliaOJIT() CompileLayer(ES, LockLayer, std::make_unique>(orc::irManglingOptionsFromTargetOptions(TM->Options), *TM)), JITPointersLayer(ES, CompileLayer, orc::IRTransformLayer::TransformFunction(JITPointersT(ES))), OptimizeLayer(ES, JITPointersLayer, orc::IRTransformLayer::TransformFunction(OptimizerT(*TM, PrintLLVMTimers))), - OptSelLayer(ES, OptimizeLayer, orc::IRTransformLayer::TransformFunction(selectOptLevel)), + DLSymOptLayer(ES, OptimizeLayer, orc::IRTransformLayer::TransformFunction(DLSymOptimizer(false))), + OptSelLayer(ES, DLSymOptLayer, orc::IRTransformLayer::TransformFunction(selectOptLevel)), DepsVerifyLayer(ES, OptSelLayer, orc::IRTransformLayer::TransformFunction(validateExternRelocations)), ExternalCompileLayer(ES, LockLayer, std::make_unique>(orc::irManglingOptionsFromTargetOptions(TM->Options), *TM)) diff --git a/src/jitlayers.h b/src/jitlayers.h index dfba515dc6fe8..d1814c0ec9e8d 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -334,6 +334,7 @@ class JuliaOJIT { typedef orc::IRCompileLayer CompileLayerT; typedef orc::IRTransformLayer JITPointersLayerT; typedef orc::IRTransformLayer OptimizeLayerT; + typedef orc::IRTransformLayer DLSymOptLayerT; typedef orc::IRTransformLayer OptSelLayerT; typedef orc::IRTransformLayer DepsVerifyLayerT; typedef object::OwningBinary OwningObj; @@ -551,6 +552,7 @@ class JuliaOJIT { CompileLayerT CompileLayer; JITPointersLayerT JITPointersLayer; OptimizeLayerT OptimizeLayer; + DLSymOptLayerT DLSymOptLayer; OptSelLayerT OptSelLayer; DepsVerifyLayerT DepsVerifyLayer; CompileLayerT ExternalCompileLayer; @@ -570,6 +572,8 @@ Module &jl_codegen_params_t::shared_module() JL_NOTSAFEPOINT { return *_shared_module; } +void optimizeDLSyms(Module &M) JL_NOTSAFEPOINT; + Pass *createLowerPTLSPass(bool imaging_mode) JL_NOTSAFEPOINT; Pass *createCombineMulAddPass() JL_NOTSAFEPOINT; Pass *createFinalLowerGCPass() JL_NOTSAFEPOINT; From 2771eca1d16cf1fb09b6bf87b93ad5454e864bee Mon Sep 17 00:00:00 2001 From: Prem Chintalapudi Date: Tue, 1 Aug 2023 16:00:17 -0500 Subject: [PATCH 3/4] Don't annotate direct jl_load_and_lookup calls, steal the names from global variables for those --- src/ccall.cpp | 38 -------------------------------------- src/jitlayers.cpp | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 46 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 0354d51ab6d88..8872683646906 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -172,12 +172,6 @@ static Value *runtime_sym_lookup( } auto lookup = irbuilder.CreateCall(prepare_call_in(jl_builderModule(irbuilder), jldlsym_func), { libname, nameval, libptrgv }); - if (runtime_lib) { - lookup->addFnAttr(Attribute::get(lookup->getContext(), "julia.libname", f_lib)); - } else { - lookup->addFnAttr(Attribute::get(lookup->getContext(), "julia.libidx", std::to_string((uintptr_t) f_lib))); - } - lookup->addFnAttr(Attribute::get(lookup->getContext(), "julia.fname", f_name)); llvmf = lookup; } setName(emission_context, llvmf, f_name + StringRef(".found")); @@ -728,22 +722,6 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg res = runtime_sym_lookup(ctx, cast(getInt8PtrTy(ctx.builder.getContext())), sym.f_lib, NULL, sym.f_name, ctx.f); res = ctx.builder.CreatePtrToInt(res, lrt); } - // else { - // void *symaddr; - - // void* libsym = jl_get_library_(sym.f_lib, 0); - // int symbol_found = jl_dlsym(libsym, sym.f_name, &symaddr, 0); - // if (!libsym || !symbol_found) { - // // Error mode, either the library or the symbol couldn't be find during compiletime. - // // Fallback to a runtime symbol lookup. - // res = runtime_sym_lookup(ctx, cast(getInt8PtrTy(ctx.builder.getContext())), sym.f_lib, NULL, sym.f_name, ctx.f); - // res = ctx.builder.CreatePtrToInt(res, lrt); - // } else { - // // since we aren't saving this code, there's no sense in - // // putting anything complicated here: just JIT the address of the cglobal - // res = ConstantInt::get(lrt, (uint64_t)symaddr); - // } - // } } JL_GC_POP(); @@ -2115,22 +2093,6 @@ jl_cgval_t function_sig_t::emit_a_ccall( else llvmf = emit_plt(ctx, functype, attributes, cc, symarg.f_lib, symarg.f_name); } - // else { - // void *symaddr; - // void *libsym = jl_get_library_(symarg.f_lib, 0); - // int symbol_found = jl_dlsym(libsym, symarg.f_name, &symaddr, 0); - // if (!libsym || !symbol_found) { - // ++DeferredCCallLookups; - // // either the library or the symbol could not be found, place a runtime - // // lookup here instead. - // llvmf = runtime_sym_lookup(ctx, funcptype, symarg.f_lib, NULL, symarg.f_name, ctx.f); - // } else { - // ++LiteralCCalls; - // // since we aren't saving this code, there's no sense in - // // putting anything complicated here: just JIT the function address - // llvmf = literal_static_pointer_val(symaddr, funcptype); - // } - // } } OperandBundleDef OpBundle("jl_roots", gc_uses); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 70502336ad2d2..e6aa69b6e09b3 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1407,7 +1407,7 @@ namespace { JITPointersT(orc::ExecutionSession &ES) JL_NOTSAFEPOINT : ES(ES) {} Expected operator()(orc::ThreadSafeModule TSM, orc::MaterializationResponsibility &R) JL_NOTSAFEPOINT { - TSM.withModuleDo([&](Module &M) { + TSM.withModuleDo([&](Module &M) JL_NOTSAFEPOINT { for (auto &GV : make_early_inc_range(M.globals())) { if (auto *Shared = getSharedBytes(GV)) { ++InternedGlobals; @@ -1563,6 +1563,7 @@ namespace { for (auto &F : M) { for (auto &BB : F) { + SmallVector to_delete; for (auto &I : make_early_inc_range(BB)) { auto CI = dyn_cast(&I); if (!CI) @@ -1570,14 +1571,21 @@ namespace { auto Callee = CI->getCalledFunction(); if (!Callee || Callee->getName() != XSTR(jl_load_and_lookup)) continue; - auto fname = CI->getFnAttr("julia.fname").getValueAsString().str(); + // Long-winded way of extracting fname without needing a second copy in an attribute + auto fname = cast(cast(CI->getArgOperand(1)->stripPointerCasts())->getInitializer())->getAsCString(); + auto libarg = CI->getArgOperand(0)->stripPointerCasts(); + // Should only use in store and phi node + // Note that this uses the raw output of codegen, + // which is why we can assume this + assert(++++CI->use_begin() == CI->use_end()); void *addr; - if (CI->hasFnAttr("julia.libname")) { - auto libname = CI->getFnAttr("julia.libname").getValueAsString().str(); + if (auto GV = dyn_cast(libarg)) { + auto libname = cast(GV->getInitializer())->getAsCString(); addr = lookup(libname.data(), fname.data()); } else { - assert(CI->hasFnAttr("julia.libidx") && "PLT entry should have either libname or libidx attribute!"); - auto libidx = (uintptr_t)std::stoull(CI->getFnAttr("julia.libidx").getValueAsString().str()); + assert(cast(libarg)->getOpcode() == Instruction::IntToPtr && "libarg should be either a global variable or a integer index!"); + libarg = cast(libarg)->getOperand(0); + auto libidx = cast(libarg)->getZExtValue(); addr = lookup(libidx, fname.data()); } if (addr) { @@ -1590,10 +1598,23 @@ namespace { } init = GlobalAlias::create(T, 0, GlobalValue::PrivateLinkage, CI->getName() + ".jit", init, &M); } - CI->replaceAllUsesWith(init); - CI->eraseFromParent(); + // DCE and SimplifyCFG will kill the branching structure around + // the call, so we don't need to worry about removing everything + for (auto user : make_early_inc_range(CI->users())) { + if (auto SI = dyn_cast(user)) { + to_delete.push_back(SI); + } else { + auto PHI = cast(user); + PHI->replaceAllUsesWith(init); + to_delete.push_back(PHI); + } + } + to_delete.push_back(CI); } } + for (auto I : to_delete) { + I->eraseFromParent(); + } } } } From 6e067b5267d90bf900c69ae3960b0025e87c0e5c Mon Sep 17 00:00:00 2001 From: Prem Chintalapudi Date: Wed, 2 Aug 2023 11:29:20 -0500 Subject: [PATCH 4/4] Move dlsym optimization to before addModule, since it is a safepoint --- src/jitlayers.cpp | 315 +++++++++++++++++++++++----------------------- src/jitlayers.h | 12 +- 2 files changed, 167 insertions(+), 160 deletions(-) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index e6aa69b6e09b3..16acd68af61c5 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -226,14 +226,17 @@ static jl_callptr_t _jl_compile_codeinst( jl_compile_workqueue(params, *temp_module, CompilationPolicy::Default); } - if (params._shared_module) + if (params._shared_module) { + jl_ExecutionEngine->optimizeDLSyms(*params._shared_module); jl_ExecutionEngine->addModule(orc::ThreadSafeModule(std::move(params._shared_module), params.tsctx)); + } // In imaging mode, we can't inline global variable initializers in order to preserve // the fiction that we don't know what loads from the global will return. Thus, we // need to emit a separate module for the globals before any functions are compiled, // to ensure that the globals are defined when they are compiled. if (params.imaging) { + // Won't contain any PLT/dlsym calls, so no need to optimize those jl_ExecutionEngine->addModule(jl_get_globals_module(params.tsctx, params.imaging, params.DL, params.TargetTriple, params.global_targets)); } else { StringMap NewGlobals; @@ -262,6 +265,7 @@ static jl_callptr_t _jl_compile_codeinst( orc::ThreadSafeModule &TSM = std::get<0>(def.second); //The underlying context object is still locked because params is not destroyed yet auto M = TSM.getModuleUnlocked(); + jl_ExecutionEngine->optimizeDLSyms(*M); for (auto &F : M->global_objects()) { if (!F.isDeclaration() && F.getLinkage() == GlobalValue::ExternalLinkage) { NewExports[F.getName()] = &TSM; @@ -398,11 +402,17 @@ int jl_compile_extern_c_impl(LLVMOrcThreadSafeModuleRef llvmmod, void *p, void * if (success && p == NULL) { jl_jit_globals(params.global_targets); assert(params.workqueue.empty()); - if (params._shared_module) + if (params._shared_module) { + jl_ExecutionEngine->optimizeDLSyms(*params._shared_module); jl_ExecutionEngine->addModule(orc::ThreadSafeModule(std::move(params._shared_module), params.tsctx)); + } } - if (success && llvmmod == NULL) + if (success && llvmmod == NULL) { + into->withModuleDo([&](Module &M) { + jl_ExecutionEngine->optimizeDLSyms(M); + }); jl_ExecutionEngine->addModule(std::move(*into)); + } } JL_UNLOCK(&jl_codegen_lock); if (timed) { @@ -1427,7 +1437,7 @@ namespace { // making a copy per object file of output. // we memoize them using the ExecutionSession's string pool; // this makes it unsafe to call clearDeadEntries() on the pool. - Constant *getSharedBytes(GlobalVariable &GV) { + Constant *getSharedBytes(GlobalVariable &GV) JL_NOTSAFEPOINT { // We could probably technically get away with // interning even external linkage globals, // as long as they have global unnamedaddr, @@ -1458,187 +1468,176 @@ namespace { orc::ExecutionSession &ES; }; +} + - struct DLSymOptimizer { - DLSymOptimizer(bool named) JL_NOTSAFEPOINT { - this->named = named; - symbols_mutex = std::make_unique(); +struct JuliaOJIT::DLSymOptimizer { + DLSymOptimizer(bool named) JL_NOTSAFEPOINT { + this->named = named; #define INIT_RUNTIME_LIBRARY(libname, handle) \ - do { \ - auto libidx = (uintptr_t) libname; \ - if (libidx >= runtime_symbols.size()) { \ - runtime_symbols.resize(libidx + 1); \ - } \ - runtime_symbols[libidx].first = handle; \ - } while (0) - - INIT_RUNTIME_LIBRARY(NULL, jl_RTLD_DEFAULT_handle); - INIT_RUNTIME_LIBRARY(JL_EXE_LIBNAME, jl_exe_handle); - INIT_RUNTIME_LIBRARY(JL_LIBJULIA_INTERNAL_DL_LIBNAME, jl_libjulia_internal_handle); - INIT_RUNTIME_LIBRARY(JL_LIBJULIA_DL_LIBNAME, jl_libjulia_handle); + do { \ + auto libidx = (uintptr_t) libname; \ + if (libidx >= runtime_symbols.size()) { \ + runtime_symbols.resize(libidx + 1); \ + } \ + runtime_symbols[libidx].first = handle; \ + } while (0) + + INIT_RUNTIME_LIBRARY(NULL, jl_RTLD_DEFAULT_handle); + INIT_RUNTIME_LIBRARY(JL_EXE_LIBNAME, jl_exe_handle); + INIT_RUNTIME_LIBRARY(JL_LIBJULIA_INTERNAL_DL_LIBNAME, jl_libjulia_internal_handle); + INIT_RUNTIME_LIBRARY(JL_LIBJULIA_DL_LIBNAME, jl_libjulia_handle); #undef INIT_RUNTIME_LIBRARY - } + } - void *lookup_symbol(void *libhandle, const char *fname) JL_NOTSAFEPOINT { - void *addr; - jl_dlsym(libhandle, fname, &addr, 0); - return addr; - } + void *lookup_symbol(void *libhandle, const char *fname) { + void *addr; + jl_dlsym(libhandle, fname, &addr, 0); + return addr; + } - void *lookup(const char *libname, const char *fname) JL_NOTSAFEPOINT { - StringRef lib(libname); - StringRef f(fname); - std::lock_guard lock(*symbols_mutex); - auto uit = user_symbols.find(lib); - if (uit == user_symbols.end()) { - void *handle = jl_get_library_(libname, 0); - if (!handle) - return nullptr; - uit = user_symbols.insert(std::make_pair(lib, std::make_pair(handle, StringMap()))).first; - } - auto &symmap = uit->second.second; - auto it = symmap.find(f); - if (it != symmap.end()) { - return it->second; - } - void *handle = lookup_symbol(uit->second.first, fname); - symmap[f] = handle; - return handle; + void *lookup(const char *libname, const char *fname) { + StringRef lib(libname); + StringRef f(fname); + std::lock_guard lock(symbols_mutex); + auto uit = user_symbols.find(lib); + if (uit == user_symbols.end()) { + void *handle = jl_get_library_(libname, 0); + if (!handle) + return nullptr; + uit = user_symbols.insert(std::make_pair(lib, std::make_pair(handle, StringMap()))).first; } + auto &symmap = uit->second.second; + auto it = symmap.find(f); + if (it != symmap.end()) { + return it->second; + } + void *handle = lookup_symbol(uit->second.first, fname); + symmap[f] = handle; + return handle; + } - void *lookup(uintptr_t libidx, const char *fname) JL_NOTSAFEPOINT { - std::lock_guard lock(*symbols_mutex); - runtime_symbols.resize(std::max(runtime_symbols.size(), libidx + 1)); - auto it = runtime_symbols[libidx].second.find(fname); - if (it != runtime_symbols[libidx].second.end()) { - return it->second; + void *lookup(uintptr_t libidx, const char *fname) { + std::lock_guard lock(symbols_mutex); + runtime_symbols.resize(std::max(runtime_symbols.size(), libidx + 1)); + auto it = runtime_symbols[libidx].second.find(fname); + if (it != runtime_symbols[libidx].second.end()) { + return it->second; + } + auto handle = lookup_symbol(runtime_symbols[libidx].first, fname); + runtime_symbols[libidx].second[fname] = handle; + return handle; + } + + void operator()(Module &M) { + for (auto &GV : M.globals()) { + auto Name = GV.getName(); + if (Name.startswith("jlplt") && Name.endswith("got")) { + auto fname = GV.getAttribute("julia.fname").getValueAsString().str(); + void *addr; + if (GV.hasAttribute("julia.libname")) { + auto libname = GV.getAttribute("julia.libname").getValueAsString().str(); + addr = lookup(libname.data(), fname.data()); + } else { + assert(GV.hasAttribute("julia.libidx") && "PLT entry should have either libname or libidx attribute!"); + auto libidx = (uintptr_t)std::stoull(GV.getAttribute("julia.libidx").getValueAsString().str()); + addr = lookup(libidx, fname.data()); + } + if (addr) { + Function *Thunk = nullptr; + if (!GV.isDeclaration()) { + Thunk = cast(GV.getInitializer()->stripPointerCasts()); + assert(++Thunk->uses().begin() == Thunk->uses().end() && "Thunk should only have one use in PLT initializer!"); + assert(Thunk->hasLocalLinkage() && "Thunk should not have non-local linkage!"); + } else { + GV.setLinkage(GlobalValue::PrivateLinkage); + } + auto init = ConstantExpr::getIntToPtr(ConstantInt::get(M.getDataLayout().getIntPtrType(M.getContext()), (uintptr_t)addr), GV.getValueType()); + if (named) { + auto T = GV.getValueType(); + assert(T->isPointerTy()); + if (!T->isOpaquePointerTy()) { + T = T->getNonOpaquePointerElementType(); + } + init = GlobalAlias::create(T, 0, GlobalValue::PrivateLinkage, GV.getName() + ".jit", init, &M); + } + GV.setInitializer(init); + GV.setConstant(true); + GV.setUnnamedAddr(GlobalValue::UnnamedAddr::Global); + if (Thunk) { + Thunk->eraseFromParent(); + } + } } - auto handle = lookup_symbol(runtime_symbols[libidx].first, fname); - runtime_symbols[libidx].second[fname] = handle; - return handle; } - void operator()(Module &M) JL_NOTSAFEPOINT { - for (auto &GV : M.globals()) { - auto Name = GV.getName(); - if (Name.startswith("jlplt") && Name.endswith("got")) { - auto fname = GV.getAttribute("julia.fname").getValueAsString().str(); + for (auto &F : M) { + for (auto &BB : F) { + SmallVector to_delete; + for (auto &I : make_early_inc_range(BB)) { + auto CI = dyn_cast(&I); + if (!CI) + continue; + auto Callee = CI->getCalledFunction(); + if (!Callee || Callee->getName() != XSTR(jl_load_and_lookup)) + continue; + // Long-winded way of extracting fname without needing a second copy in an attribute + auto fname = cast(cast(CI->getArgOperand(1)->stripPointerCasts())->getInitializer())->getAsCString(); + auto libarg = CI->getArgOperand(0)->stripPointerCasts(); + // Should only use in store and phi node + // Note that this uses the raw output of codegen, + // which is why we can assume this + assert(++++CI->use_begin() == CI->use_end()); void *addr; - if (GV.hasAttribute("julia.libname")) { - auto libname = GV.getAttribute("julia.libname").getValueAsString().str(); + if (auto GV = dyn_cast(libarg)) { + auto libname = cast(GV->getInitializer())->getAsCString(); addr = lookup(libname.data(), fname.data()); } else { - assert(GV.hasAttribute("julia.libidx") && "PLT entry should have either libname or libidx attribute!"); - auto libidx = (uintptr_t)std::stoull(GV.getAttribute("julia.libidx").getValueAsString().str()); + assert(cast(libarg)->getOpcode() == Instruction::IntToPtr && "libarg should be either a global variable or a integer index!"); + libarg = cast(libarg)->getOperand(0); + auto libidx = cast(libarg)->getZExtValue(); addr = lookup(libidx, fname.data()); } if (addr) { - Function *Thunk = nullptr; - if (!GV.isDeclaration()) { - Thunk = cast(GV.getInitializer()->stripPointerCasts()); - assert(++Thunk->uses().begin() == Thunk->uses().end() && "Thunk should only have one use in PLT initializer!"); - assert(Thunk->hasLocalLinkage() && "Thunk should not have non-local linkage!"); - } else { - GV.setLinkage(GlobalValue::PrivateLinkage); - } - auto init = ConstantExpr::getIntToPtr(ConstantInt::get(M.getDataLayout().getIntPtrType(M.getContext()), (uintptr_t)addr), GV.getValueType()); + auto init = ConstantExpr::getIntToPtr(ConstantInt::get(M.getDataLayout().getIntPtrType(M.getContext()), (uintptr_t)addr), CI->getType()); if (named) { - auto T = GV.getValueType(); + auto T = CI->getType(); assert(T->isPointerTy()); if (!T->isOpaquePointerTy()) { T = T->getNonOpaquePointerElementType(); } - init = GlobalAlias::create(T, 0, GlobalValue::PrivateLinkage, GV.getName() + ".jit", init, &M); - } - GV.setInitializer(init); - GV.setConstant(true); - GV.setUnnamedAddr(GlobalValue::UnnamedAddr::Global); - if (Thunk) { - Thunk->eraseFromParent(); - } - } - } - } - - for (auto &F : M) { - for (auto &BB : F) { - SmallVector to_delete; - for (auto &I : make_early_inc_range(BB)) { - auto CI = dyn_cast(&I); - if (!CI) - continue; - auto Callee = CI->getCalledFunction(); - if (!Callee || Callee->getName() != XSTR(jl_load_and_lookup)) - continue; - // Long-winded way of extracting fname without needing a second copy in an attribute - auto fname = cast(cast(CI->getArgOperand(1)->stripPointerCasts())->getInitializer())->getAsCString(); - auto libarg = CI->getArgOperand(0)->stripPointerCasts(); - // Should only use in store and phi node - // Note that this uses the raw output of codegen, - // which is why we can assume this - assert(++++CI->use_begin() == CI->use_end()); - void *addr; - if (auto GV = dyn_cast(libarg)) { - auto libname = cast(GV->getInitializer())->getAsCString(); - addr = lookup(libname.data(), fname.data()); - } else { - assert(cast(libarg)->getOpcode() == Instruction::IntToPtr && "libarg should be either a global variable or a integer index!"); - libarg = cast(libarg)->getOperand(0); - auto libidx = cast(libarg)->getZExtValue(); - addr = lookup(libidx, fname.data()); + init = GlobalAlias::create(T, 0, GlobalValue::PrivateLinkage, CI->getName() + ".jit", init, &M); } - if (addr) { - auto init = ConstantExpr::getIntToPtr(ConstantInt::get(M.getDataLayout().getIntPtrType(M.getContext()), (uintptr_t)addr), CI->getType()); - if (named) { - auto T = CI->getType(); - assert(T->isPointerTy()); - if (!T->isOpaquePointerTy()) { - T = T->getNonOpaquePointerElementType(); - } - init = GlobalAlias::create(T, 0, GlobalValue::PrivateLinkage, CI->getName() + ".jit", init, &M); + // DCE and SimplifyCFG will kill the branching structure around + // the call, so we don't need to worry about removing everything + for (auto user : make_early_inc_range(CI->users())) { + if (auto SI = dyn_cast(user)) { + to_delete.push_back(SI); + } else { + auto PHI = cast(user); + PHI->replaceAllUsesWith(init); + to_delete.push_back(PHI); } - // DCE and SimplifyCFG will kill the branching structure around - // the call, so we don't need to worry about removing everything - for (auto user : make_early_inc_range(CI->users())) { - if (auto SI = dyn_cast(user)) { - to_delete.push_back(SI); - } else { - auto PHI = cast(user); - PHI->replaceAllUsesWith(init); - to_delete.push_back(PHI); - } - } - to_delete.push_back(CI); } + to_delete.push_back(CI); } - for (auto I : to_delete) { - I->eraseFromParent(); - } + } + for (auto I : to_delete) { + I->eraseFromParent(); } } } + } - Expected operator()(orc::ThreadSafeModule TSM, orc::MaterializationResponsibility &R) JL_NOTSAFEPOINT { - TSM.withModuleDo([this](Module &M) JL_NOTSAFEPOINT { - // Don't optimize these out in imaging mode - if (M.getModuleFlag("julia.imaging_mode")) - return; - (*this)(M); - }); - - return std::move(TSM); - } - - std::unique_ptr symbols_mutex; - StringMap>> user_symbols; - SmallVector>> runtime_symbols; - bool named; - }; -} + std::mutex symbols_mutex; + StringMap>> user_symbols; + SmallVector>> runtime_symbols; + bool named; +}; -void optimizeDLSyms(Module &M) JL_NOTSAFEPOINT { - DLSymOptimizer(true)(M); +void optimizeDLSyms(Module &M) { + JuliaOJIT::DLSymOptimizer(true)(M); } llvm::DataLayout jl_create_datalayout(TargetMachine &TM) { @@ -1659,6 +1658,7 @@ JuliaOJIT::JuliaOJIT() GlobalJD(ES.createBareJITDylib("JuliaGlobals")), JD(ES.createBareJITDylib("JuliaOJIT")), ExternalJD(ES.createBareJITDylib("JuliaExternal")), + DLSymOpt(std::make_unique(false)), ContextPool([](){ auto ctx = std::make_unique(); return orc::ThreadSafeContext(std::move(ctx)); @@ -1680,8 +1680,7 @@ JuliaOJIT::JuliaOJIT() CompileLayer(ES, LockLayer, std::make_unique>(orc::irManglingOptionsFromTargetOptions(TM->Options), *TM)), JITPointersLayer(ES, CompileLayer, orc::IRTransformLayer::TransformFunction(JITPointersT(ES))), OptimizeLayer(ES, JITPointersLayer, orc::IRTransformLayer::TransformFunction(OptimizerT(*TM, PrintLLVMTimers))), - DLSymOptLayer(ES, OptimizeLayer, orc::IRTransformLayer::TransformFunction(DLSymOptimizer(false))), - OptSelLayer(ES, DLSymOptLayer, orc::IRTransformLayer::TransformFunction(selectOptLevel)), + OptSelLayer(ES, OptimizeLayer, orc::IRTransformLayer::TransformFunction(selectOptLevel)), DepsVerifyLayer(ES, OptSelLayer, orc::IRTransformLayer::TransformFunction(validateExternRelocations)), ExternalCompileLayer(ES, LockLayer, std::make_unique>(orc::irManglingOptionsFromTargetOptions(TM->Options), *TM)) @@ -2029,6 +2028,10 @@ void JuliaOJIT::printTimers() reportAndResetTimings(); } +void JuliaOJIT::optimizeDLSyms(Module &M) { + (*DLSymOpt)(M); +} + JuliaOJIT *jl_ExecutionEngine; // destructively move the contents of src into dest diff --git a/src/jitlayers.h b/src/jitlayers.h index d1814c0ec9e8d..9d938cf1d8b0e 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -334,7 +334,6 @@ class JuliaOJIT { typedef orc::IRCompileLayer CompileLayerT; typedef orc::IRTransformLayer JITPointersLayerT; typedef orc::IRTransformLayer OptimizeLayerT; - typedef orc::IRTransformLayer DLSymOptLayerT; typedef orc::IRTransformLayer OptSelLayerT; typedef orc::IRTransformLayer DepsVerifyLayerT; typedef object::OwningBinary OwningObj; @@ -449,6 +448,8 @@ class JuliaOJIT { std::unique_ptr mutex; }; + struct DLSymOptimizer; + private: // Custom object emission notification handler for the JuliaOJIT template @@ -518,6 +519,9 @@ class JuliaOJIT { } std::string getMangledName(StringRef Name) JL_NOTSAFEPOINT; std::string getMangledName(const GlobalValue *GV) JL_NOTSAFEPOINT; + + // Note that this is a safepoint due to jl_get_library_ and jl_dlsym calls + void optimizeDLSyms(Module &M); private: const std::unique_ptr TM; @@ -532,6 +536,8 @@ class JuliaOJIT { int RLST_inc = 0; DenseMap ReverseLocalSymbolTable; + std::unique_ptr DLSymOpt; + //Compilation streams jl_locked_stream dump_emitted_mi_name_stream; jl_locked_stream dump_compiles_stream; @@ -552,11 +558,9 @@ class JuliaOJIT { CompileLayerT CompileLayer; JITPointersLayerT JITPointersLayer; OptimizeLayerT OptimizeLayer; - DLSymOptLayerT DLSymOptLayer; OptSelLayerT OptSelLayer; DepsVerifyLayerT DepsVerifyLayer; CompileLayerT ExternalCompileLayer; - }; extern JuliaOJIT *jl_ExecutionEngine; std::unique_ptr jl_create_llvm_module(StringRef name, LLVMContext &ctx, bool imaging_mode, const DataLayout &DL = jl_ExecutionEngine->getDataLayout(), const Triple &triple = jl_ExecutionEngine->getTargetTriple()) JL_NOTSAFEPOINT; @@ -572,7 +576,7 @@ Module &jl_codegen_params_t::shared_module() JL_NOTSAFEPOINT { return *_shared_module; } -void optimizeDLSyms(Module &M) JL_NOTSAFEPOINT; +void optimizeDLSyms(Module &M); Pass *createLowerPTLSPass(bool imaging_mode) JL_NOTSAFEPOINT; Pass *createCombineMulAddPass() JL_NOTSAFEPOINT;