diff --git a/base/compiler/validation.jl b/base/compiler/validation.jl index bcde5d894159c..03113a7ce14c1 100644 --- a/base/compiler/validation.jl +++ b/base/compiler/validation.jl @@ -28,6 +28,7 @@ const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange{Int}}( :isdefined => 1:1, :code_coverage_effect => 0:0, :loopinfo => 0:typemax(Int), + :ivdepscope => 1:1, :gc_preserve_begin => 0:typemax(Int), :gc_preserve_end => 0:typemax(Int), :thunk => 1:1, @@ -145,7 +146,7 @@ function validate_code!(errors::Vector{>:InvalidCodeError}, c::CodeInfo, is_top_ head === :inbounds || head === :foreigncall || head === :cfunction || head === :const || head === :enter || head === :leave || head === :pop_exception || head === :method || head === :global || head === :static_parameter || - head === :new || head === :splatnew || head === :thunk || head === :loopinfo || + head === :new || head === :splatnew || head === :thunk || head === :loopinfo || head === :ivdepscope || head === :throw_undef_if_not || head === :code_coverage_effect || head === :inline || head === :noinline validate_val!(x) else diff --git a/base/simdloop.jl b/base/simdloop.jl index 29e2382cf39aa..c3d495f16aff9 100644 --- a/base/simdloop.jl +++ b/base/simdloop.jl @@ -4,7 +4,7 @@ module SimdLoop -export @simd, simd_outer_range, simd_inner_length, simd_index +export @simd, @ivdep, simd_outer_range, simd_inner_length, simd_index # Error thrown from ill-formed uses of @simd struct SimdError <: Exception @@ -58,7 +58,7 @@ function compile(x, ivdep) (isa(x, Expr) && x.head === :for) || throw(SimdError("for loop expected")) length(x.args) == 2 || throw(SimdError("1D for loop expected")) check_body!(x) - + ivdepend = ivdep === nothing ? nothing : Expr(:ivdepscope, :end) var,range = parse_iteration_space(x.args[1]) r = gensym("r") # Range value j = gensym("i") # Iteration variable for outer loop @@ -73,10 +73,12 @@ function compile(x, ivdep) # Lower loop in way that seems to work best for LLVM 3.3 vectorizer. let $i = zero($n) while $i < $n + $ivdep local $var = Base.simd_index($r,$j,$i) $(x.args[2]) # Body of loop $i += 1 - $(Expr(:loopinfo, Symbol("julia.simdloop"), ivdep)) # Mark loop as SIMD loop + $ivdepend + $(Expr(:loopinfo, Symbol("julia.simdloop"))) # Mark loop as SIMD loop end end end @@ -130,10 +132,27 @@ end macro simd(ivdep, forloop) if ivdep === :ivdep - esc(compile(forloop, Symbol("julia.ivdep"))) + esc(compile(forloop, Expr(:ivdepscope, :begin))) else throw(SimdError("Only ivdep is valid as the first argument to @simd")) end end +""" + @ivdep + +Annotate the following scope is free of loop-carried memory dependencies. + +!!! note + @ivdep is valid only within a @simd loop +""" +macro ivdep(ex) + esc(quote + $(Expr(:ivdepscope, :begin)) + temp = $ex + $(Expr(:ivdepscope, :end)) + temp + end) +end + end # module SimdLoop diff --git a/src/ast.c b/src/ast.c index bdc891ebd3e10..43f9ffed3683d 100644 --- a/src/ast.c +++ b/src/ast.c @@ -68,6 +68,7 @@ JL_DLLEXPORT jl_sym_t *jl_copyast_sym; JL_DLLEXPORT jl_sym_t *jl_cfunction_sym; JL_DLLEXPORT jl_sym_t *jl_pure_sym; JL_DLLEXPORT jl_sym_t *jl_loopinfo_sym; +JL_DLLEXPORT jl_sym_t *jl_ivdepscope_sym; JL_DLLEXPORT jl_sym_t *jl_meta_sym; JL_DLLEXPORT jl_sym_t *jl_inert_sym; JL_DLLEXPORT jl_sym_t *jl_polly_sym; @@ -317,6 +318,7 @@ void jl_init_common_symbols(void) jl_newvar_sym = jl_symbol("newvar"); jl_copyast_sym = jl_symbol("copyast"); jl_loopinfo_sym = jl_symbol("loopinfo"); + jl_ivdepscope_sym = jl_symbol("ivdepscope"); jl_pure_sym = jl_symbol("pure"); jl_meta_sym = jl_symbol("meta"); jl_list_sym = jl_symbol("list"); diff --git a/src/ast.scm b/src/ast.scm index a1615cc01e2fe..d12cd0da41ca1 100644 --- a/src/ast.scm +++ b/src/ast.scm @@ -296,7 +296,7 @@ ;; predicates and accessors (define (quoted? e) - (memq (car e) '(quote top core globalref outerref line break inert meta inbounds inline noinline loopinfo))) + (memq (car e) '(quote top core globalref outerref line break inert meta inbounds inline noinline loopinfo ivdepscope))) (define (quotify e) `',e) (define (unquote e) (if (and (pair? e) (memq (car e) '(quote inert))) diff --git a/src/codegen.cpp b/src/codegen.cpp index 2ed36f21fa1e2..3f3b853cf4dcf 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -669,6 +669,14 @@ static const auto jl_loopinfo_marker_func = new JuliaFunction{ AttributeSet(), None); }, }; +static const auto jl_ivdepscope_func = new JuliaFunction{ + "julia.ivdepscope", + [](LLVMContext &C) { return FunctionType::get(T_void, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + Attributes(C, {Attribute::ReadOnly, Attribute::NoRecurse, Attribute::InaccessibleMemOnly}), + AttributeSet(), + None); }, +}; static const auto jl_write_barrier_func = new JuliaFunction{ "julia.write_barrier", [](LLVMContext &C) { return FunctionType::get(T_void, @@ -4694,6 +4702,20 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval) I->setMetadata("julia.loopinfo", MD); return jl_cgval_t(); } + else if (head == jl_ivdepscope_sym) { + // parse Expr(:ivdepscope, :begin/end) + SmallVector MDs; + for (int i = 0, ie = nargs; i < ie; ++i) { + Metadata *MD = to_md_tree(args[i]); + if (MD) + MDs.push_back(MD); + } + + MDNode* MD = MDNode::get(jl_LLVMContext, MDs); + CallInst *I = ctx.builder.CreateCall(prepare_call(jl_ivdepscope_func)); + I->setMetadata("julia.ivdepscope", MD); + return jl_cgval_t(); + } else if (head == jl_leave_sym || head == jl_coverageeffect_sym || head == jl_pop_exception_sym || head == jl_enter_sym || head == jl_inbounds_sym || head == jl_aliasscope_sym || head == jl_popaliasscope_sym || head == jl_inline_sym || head == jl_noinline_sym) { @@ -7897,6 +7919,7 @@ static void init_jit_functions(void) add_named_global(jl_alloc_obj_func, (void*)NULL); add_named_global(jl_newbits_func, (void*)jl_new_bits); add_named_global(jl_loopinfo_marker_func, (void*)NULL); + add_named_global(jl_ivdepscope_func, &jl_ivdepscope_error); add_named_global(jl_typeof_func, (void*)NULL); add_named_global(jl_write_barrier_func, (void*)NULL); add_named_global(jldlsym_func, &jl_load_and_lookup); diff --git a/src/interpreter.c b/src/interpreter.c index 6b07a8f7e5971..6f95995efedbd 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -314,7 +314,7 @@ static jl_value_t *eval_value(jl_value_t *e, interpreter_state *s) else if (head == jl_boundscheck_sym) { return jl_true; } - else if (head == jl_meta_sym || head == jl_coverageeffect_sym || head == jl_inbounds_sym || head == jl_loopinfo_sym || + else if (head == jl_meta_sym || head == jl_coverageeffect_sym || head == jl_inbounds_sym || head == jl_loopinfo_sym || head == jl_ivdepscope_sym || head == jl_aliasscope_sym || head == jl_popaliasscope_sym || head == jl_inline_sym || head == jl_noinline_sym) { return jl_nothing; } diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 6d09309cfe310..2c4b33d10c01a 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -472,6 +472,7 @@ XX(jl_tty_set_mode) \ XX(jl_tupletype_fill) \ XX(jl_typeassert) \ + XX(jl_ivdepscope_error) \ XX(jl_type_equality_is_identity) \ XX(jl_type_error) \ XX(jl_type_error_rt) \ diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index e71cda3f9afee..c4bfb4c6e4db3 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -3516,7 +3516,7 @@ f(x) = yt(x) (define lambda-opt-ignored-exprs (Set '(quote top core line inert local-def unnecessary copyast - meta inbounds boundscheck loopinfo decl aliasscope popaliasscope + meta inbounds boundscheck loopinfo ivdepscope decl aliasscope popaliasscope thunk with-static-parameters toplevel-only global globalref outerref const-if-global thismodule const atomic null true false ssavalue isdefined toplevel module lambda @@ -4625,7 +4625,7 @@ f(x) = yt(x) (cons (car e) args))) ;; metadata expressions - ((line meta inbounds loopinfo gc_preserve_end aliasscope popaliasscope inline noinline) + ((line meta inbounds loopinfo ivdepscope gc_preserve_end aliasscope popaliasscope inline noinline) (let ((have-ret? (and (pair? code) (pair? (car code)) (eq? (caar code) 'return)))) (cond ((eq? (car e) 'line) (set! current-loc e) diff --git a/src/julia_internal.h b/src/julia_internal.h index ee46eb88297af..1c2eda3f7d5ed 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -534,6 +534,7 @@ JL_DLLEXPORT jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type, const JL_DLLEXPORT jl_value_t *jl_get_keyword_sorter(jl_value_t *f); JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t); +JL_DLLEXPORT void jl_ivdepscope_error(void); #define JL_CALLABLE(name) \ JL_DLLEXPORT jl_value_t *name(jl_value_t *F, jl_value_t **args, uint32_t nargs) @@ -1408,6 +1409,7 @@ extern JL_DLLEXPORT jl_sym_t *jl_copyast_sym; extern JL_DLLEXPORT jl_sym_t *jl_cfunction_sym; extern JL_DLLEXPORT jl_sym_t *jl_pure_sym; extern JL_DLLEXPORT jl_sym_t *jl_loopinfo_sym; +extern JL_DLLEXPORT jl_sym_t *jl_ivdepscope_sym; extern JL_DLLEXPORT jl_sym_t *jl_meta_sym; extern JL_DLLEXPORT jl_sym_t *jl_inert_sym; extern JL_DLLEXPORT jl_sym_t *jl_polly_sym; diff --git a/src/llvm-simdloop.cpp b/src/llvm-simdloop.cpp index 8d80a535b2319..18afa0528c58d 100644 --- a/src/llvm-simdloop.cpp +++ b/src/llvm-simdloop.cpp @@ -115,13 +115,11 @@ static bool markLoopInfo(Module &M, Function *marker, function_refgetParent()->getParent()); Loop *L = LI.getLoopFor(I->getParent()); - I->removeFromParent(); if (!L) continue; LLVM_DEBUG(dbgs() << "LSL: loopinfo marker found\n"); bool simd = false; - bool ivdep = false; SmallVector MDs; BasicBlock *Lh = L->getHeader(); @@ -144,8 +142,6 @@ static bool markLoopInfo(Module &M, Function *marker, function_refgetString().startswith("julia")) { if (S->getString().equals("julia.simdloop")) simd = true; - if (S->getString().equals("julia.ivdep")) - ivdep = true; continue; } } @@ -154,7 +150,9 @@ static bool markLoopInfo(Module &M, Function *marker, function_refgetLoopID(); if (n) { @@ -173,40 +171,62 @@ static bool markLoopInfo(Module &M, Function *marker, function_refgetContext(), ArrayRef(LoopID)); - // If ivdep is true we assume that there is no memory dependency between loop iterations + // mark the inner-most loop is free of memory dependency within julia ivdep scope. // This is a fairly strong assumption and does often not hold true for generic code. - if (ivdep) { - // Mark memory references so that Loop::isAnnotatedParallel will return true for this loop. - for (BasicBlock *BB : L->blocks()) { - for (Instruction &I : *BB) { - if (I.mayReadOrWriteMemory()) { - I.setMetadata(LLVMContext::MD_mem_parallel_loop_access, m); - } - } + int ivdep = 0; + for (BasicBlock *BB : L->blocks()) { + for (Instruction &I : *BB) { + if (I.hasMetadataOtherThanDebugLoc()) { + if (MDNode *JLMD= I.getMetadata("julia.ivdepscope")) { + ToDelete.push_back(&I); + LLVM_DEBUG(dbgs() << "LSL: found julia.ivdepscope "); + if (JLMD->getNumOperands() < 1) + continue; + if (MDString *S = dyn_cast(JLMD->getOperand(0))) { + LLVM_DEBUG(dbgs() << S->getString() << "\n"); + if (S->getString().equals("begin")) + ivdep += 1; + else + ivdep -= 1; + } + } + } + if (ivdep > 0 &&I.mayReadOrWriteMemory()) { + I.setMetadata(LLVMContext::MD_mem_parallel_loop_access, m); + } } - assert(L->isAnnotatedParallel()); } - - if (simd) { - // Mark floating-point reductions as okay to reassociate/commute. - for (BasicBlock::iterator I = Lh->begin(), E = Lh->end(); I != E; ++I) { - if (PHINode *Phi = dyn_cast(I)) - enableUnsafeAlgebraIfReduction(Phi, L); - else - break; - } + // if ivdep != 0 + // thrown() ?? + + // Mark floating-point reductions as okay to reassociate/commute. + for (BasicBlock::iterator I = Lh->begin(), E = Lh->end(); I != E; ++I) { + if (PHINode *Phi = dyn_cast(I)) + enableUnsafeAlgebraIfReduction(Phi, L); + else + break; } Changed = true; } for (Instruction *I : ToDelete) - I->deleteValue(); + I->eraseFromParent(); marker->eraseFromParent(); return Changed; } +static void eraseIvdepScope(Module &M, Function *marker) +{ + for (User *U : marker->users()) { + Instruction *I = cast(U); + // remove "ivdepscope" from unreachable branch to make error message clearer. + if (isa(I -> getParent() -> back())) + I -> eraseFromParent(); + } +} + } // end anonymous namespace @@ -223,7 +243,12 @@ PreservedAnalyses LowerSIMDLoop::run(Module &M, ModuleAnalysisManager &AM) Function *loopinfo_marker = M.getFunction("julia.loopinfo_marker"); if (!loopinfo_marker) + { + Function *ivdepscope = M.getFunction("julia.ivdepscope"); + if (ivdepscope) + eraseIvdepScope(M, ivdepscope); return PreservedAnalyses::all(); + } FunctionAnalysisManager &FAM = AM.getResult(M).getManager(); @@ -258,6 +283,11 @@ class LowerSIMDLoopLegacy : public ModulePass { if (loopinfo_marker) Changed |= markLoopInfo(M, loopinfo_marker, GetLI); + else { + Function *ivdepscope = M.getFunction("julia.ivdepscope"); + if (ivdepscope) + eraseIvdepScope(M, ivdepscope); + } return Changed; } diff --git a/src/macroexpand.scm b/src/macroexpand.scm index 34414d24bde75..bbe95b93fed67 100644 --- a/src/macroexpand.scm +++ b/src/macroexpand.scm @@ -354,7 +354,7 @@ ,(resolve-expansion-vars-with-new-env (caddr arg) env m parent-scope inarg)))) (else `(global ,(resolve-expansion-vars-with-new-env arg env m parent-scope inarg)))))) - ((using import export meta line inbounds boundscheck loopinfo inline noinline) (map unescape e)) + ((using import export meta line inbounds boundscheck loopinfo ivdepscope inline noinline) (map unescape e)) ((macrocall) e) ; invalid syntax anyways, so just act like it's quoted. ((symboliclabel) e) ((symbolicgoto) e) diff --git a/src/method.c b/src/method.c index d8cd1c30a94e1..40d2ce1a07ab1 100644 --- a/src/method.c +++ b/src/method.c @@ -84,6 +84,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve e->head == jl_quote_sym || e->head == jl_inert_sym || e->head == jl_meta_sym || e->head == jl_inbounds_sym || e->head == jl_boundscheck_sym || e->head == jl_loopinfo_sym || + e->head == jl_ivdepscope_sym || e->head == jl_aliasscope_sym || e->head == jl_popaliasscope_sym || e->head == jl_inline_sym || e->head == jl_noinline_sym) { // ignore these diff --git a/src/rtutils.c b/src/rtutils.c index b4432d8af3d0c..af65bb54f96fc 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -214,6 +214,11 @@ JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t) jl_type_error("typeassert", t, x); } +JL_DLLEXPORT void jl_ivdepscope_error(void) +{ + jl_errorf("Found ivdepscope outside @simd."); +} + #ifndef HAVE_SSP JL_DLLEXPORT uintptr_t __stack_chk_guard = (uintptr_t)0xBAD57ACCBAD67ACC; // 0xBADSTACKBADSTACK diff --git a/test/simdloop.jl b/test/simdloop.jl index 88e41364ef222..d2bfba1875a8a 100644 --- a/test/simdloop.jl +++ b/test/simdloop.jl @@ -161,3 +161,10 @@ Base.SimdLoop.simd_index(v::iter31113, j, i) = j Base.SimdLoop.simd_inner_length(v::iter31113, j) = 1 Base.SimdLoop.simd_outer_range(v::iter31113) = v @test 2001000 == simd_sum_over_array(iter31113(Vector(1:2000))) + +#@ivdep thrown +ivdep_out_simd(x) = @inbounds for i in eachindex(x) + Base.@ivdep x[i] += 1 +end +@test_throws "Found ivdepscope outside @simd." ivdep_out_simd([1,2,3,4]) +@test_throws MethodError ivdep_out_simd((1,2,3,4))