Add Expr(:ivdepscope) to support not marking the entire loop body as ivdep

base/compiler/validation.jl

@@ -28,6 +28,7 @@ const VALID_EXPR_HEADS = IdDict{Symbol,UnitRange{Int}}(
     :isdefined => 1:1,
     :code_coverage_effect => 0:0,
     :loopinfo => 0:typemax(Int),
+    :ivdepscope => 1:1,
     :gc_preserve_begin => 0:typemax(Int),
     :gc_preserve_end => 0:typemax(Int),
     :thunk => 1:1,
@@ -145,7 +146,7 @@ function validate_code!(errors::Vector{>:InvalidCodeError}, c::CodeInfo, is_top_
                 head === :inbounds || head === :foreigncall || head === :cfunction ||
                 head === :const || head === :enter || head === :leave || head === :pop_exception ||
                 head === :method || head === :global || head === :static_parameter ||
-                head === :new || head === :splatnew || head === :thunk || head === :loopinfo ||
+                head === :new || head === :splatnew || head === :thunk || head === :loopinfo || head === :ivdepscope ||
                 head === :throw_undef_if_not || head === :code_coverage_effect || head === :inline || head === :noinline
                 validate_val!(x)
             else

base/simdloop.jl

@@ -4,7 +4,7 @@
 
 module SimdLoop
 
-export @simd, simd_outer_range, simd_inner_length, simd_index
+export @simd, @ivdep, simd_outer_range, simd_inner_length, simd_index
 
 # Error thrown from ill-formed uses of @simd
 struct SimdError <: Exception
@@ -58,7 +58,7 @@ function compile(x, ivdep)
     (isa(x, Expr) && x.head === :for) || throw(SimdError("for loop expected"))
     length(x.args) == 2 || throw(SimdError("1D for loop expected"))
     check_body!(x)
-
+    ivdepend = ivdep === nothing ? nothing : Expr(:ivdepscope, :end)
     var,range = parse_iteration_space(x.args[1])
     r = gensym("r") # Range value
     j = gensym("i") # Iteration variable for outer loop
@@ -73,10 +73,12 @@ function compile(x, ivdep)
                         # Lower loop in way that seems to work best for LLVM 3.3 vectorizer.
                         let $i = zero($n)
                             while $i < $n
+                                $ivdep
                                 local $var = Base.simd_index($r,$j,$i)
                                 $(x.args[2])        # Body of loop
                                 $i += 1
-                                $(Expr(:loopinfo, Symbol("julia.simdloop"), ivdep))  # Mark loop as SIMD loop
+                                $ivdepend
+                                $(Expr(:loopinfo, Symbol("julia.simdloop")))  # Mark loop as SIMD loop
                             end
                         end
                     end
@@ -130,10 +132,27 @@ end
 
 macro simd(ivdep, forloop)
     if ivdep === :ivdep
-        esc(compile(forloop, Symbol("julia.ivdep")))
+        esc(compile(forloop, Expr(:ivdepscope, :begin)))
     else
         throw(SimdError("Only ivdep is valid as the first argument to @simd"))
     end
 end
 
+"""
+    @ivdep
+
+Annotate the following scope is free of loop-carried memory dependencies.
+
+!!! note
+    @ivdep is valid only within a @simd loop
+"""
+macro ivdep(ex)
+    esc(quote
+        $(Expr(:ivdepscope, :begin))
+        temp = $ex
+        $(Expr(:ivdepscope, :end))
+        temp
+    end)
+end
+
 end # module SimdLoop

src/ast.c

@@ -68,6 +68,7 @@ JL_DLLEXPORT jl_sym_t *jl_copyast_sym;
 JL_DLLEXPORT jl_sym_t *jl_cfunction_sym;
 JL_DLLEXPORT jl_sym_t *jl_pure_sym;
 JL_DLLEXPORT jl_sym_t *jl_loopinfo_sym;
+JL_DLLEXPORT jl_sym_t *jl_ivdepscope_sym;
 JL_DLLEXPORT jl_sym_t *jl_meta_sym;
 JL_DLLEXPORT jl_sym_t *jl_inert_sym;
 JL_DLLEXPORT jl_sym_t *jl_polly_sym;
@@ -317,6 +318,7 @@ void jl_init_common_symbols(void)
     jl_newvar_sym = jl_symbol("newvar");
     jl_copyast_sym = jl_symbol("copyast");
     jl_loopinfo_sym = jl_symbol("loopinfo");
+    jl_ivdepscope_sym = jl_symbol("ivdepscope");
     jl_pure_sym = jl_symbol("pure");
     jl_meta_sym = jl_symbol("meta");
     jl_list_sym = jl_symbol("list");

src/ast.scm

@@ -296,7 +296,7 @@
 ;; predicates and accessors
 
 (define (quoted? e)
-  (memq (car e) '(quote top core globalref outerref line break inert meta inbounds inline noinline loopinfo)))
+  (memq (car e) '(quote top core globalref outerref line break inert meta inbounds inline noinline loopinfo ivdepscope)))
 (define (quotify e) `',e)
 (define (unquote e)
   (if (and (pair? e) (memq (car e) '(quote inert)))

src/codegen.cpp

@@ -669,6 +669,14 @@ static const auto jl_loopinfo_marker_func = new JuliaFunction{
             AttributeSet(),
             None); },
 };
+static const auto jl_ivdepscope_func = new JuliaFunction{
+    "julia.ivdepscope",
+    [](LLVMContext &C) { return FunctionType::get(T_void, false); },
+    [](LLVMContext &C) { return AttributeList::get(C,
+            Attributes(C, {Attribute::ReadOnly, Attribute::NoRecurse, Attribute::InaccessibleMemOnly}),
+            AttributeSet(),
+            None); },
+};
 static const auto jl_write_barrier_func = new JuliaFunction{
     "julia.write_barrier",
     [](LLVMContext &C) { return FunctionType::get(T_void,
@@ -4694,6 +4702,20 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval)
         I->setMetadata("julia.loopinfo", MD);
         return jl_cgval_t();
     }
+    else if (head == jl_ivdepscope_sym) {
+        // parse Expr(:ivdepscope, :begin/end)
+        SmallVector<Metadata *, 8> MDs;
+        for (int i = 0, ie = nargs; i < ie; ++i) {
+            Metadata *MD = to_md_tree(args[i]);
+            if (MD)
+                MDs.push_back(MD);
+        }
+
+        MDNode* MD = MDNode::get(jl_LLVMContext, MDs);
+        CallInst *I = ctx.builder.CreateCall(prepare_call(jl_ivdepscope_func));
+        I->setMetadata("julia.ivdepscope", MD);
+        return jl_cgval_t();
+    }
     else if (head == jl_leave_sym || head == jl_coverageeffect_sym
             || head == jl_pop_exception_sym || head == jl_enter_sym || head == jl_inbounds_sym
             || head == jl_aliasscope_sym || head == jl_popaliasscope_sym || head == jl_inline_sym || head == jl_noinline_sym) {
@@ -7897,6 +7919,7 @@ static void init_jit_functions(void)
     add_named_global(jl_alloc_obj_func, (void*)NULL);
     add_named_global(jl_newbits_func, (void*)jl_new_bits);
     add_named_global(jl_loopinfo_marker_func, (void*)NULL);
+    add_named_global(jl_ivdepscope_func, &jl_ivdepscope_error);
     add_named_global(jl_typeof_func, (void*)NULL);
     add_named_global(jl_write_barrier_func, (void*)NULL);
     add_named_global(jldlsym_func, &jl_load_and_lookup);

src/interpreter.c

@@ -314,7 +314,7 @@ static jl_value_t *eval_value(jl_value_t *e, interpreter_state *s)
     else if (head == jl_boundscheck_sym) {
         return jl_true;
     }
-    else if (head == jl_meta_sym || head == jl_coverageeffect_sym || head == jl_inbounds_sym || head == jl_loopinfo_sym ||
+    else if (head == jl_meta_sym || head == jl_coverageeffect_sym || head == jl_inbounds_sym || head == jl_loopinfo_sym || head == jl_ivdepscope_sym ||
              head == jl_aliasscope_sym || head == jl_popaliasscope_sym || head == jl_inline_sym || head == jl_noinline_sym) {
         return jl_nothing;
     }

src/jl_exported_funcs.inc

@@ -472,6 +472,7 @@
     XX(jl_tty_set_mode) \
     XX(jl_tupletype_fill) \
     XX(jl_typeassert) \
+    XX(jl_ivdepscope_error) \
     XX(jl_type_equality_is_identity) \
     XX(jl_type_error) \
     XX(jl_type_error_rt) \

src/julia-syntax.scm

@@ -3516,7 +3516,7 @@ f(x) = yt(x)
 
 (define lambda-opt-ignored-exprs
   (Set '(quote top core line inert local-def unnecessary copyast
-         meta inbounds boundscheck loopinfo decl aliasscope popaliasscope
+         meta inbounds boundscheck loopinfo ivdepscope decl aliasscope popaliasscope
          thunk with-static-parameters toplevel-only
          global globalref outerref const-if-global thismodule
          const atomic null true false ssavalue isdefined toplevel module lambda
@@ -4625,7 +4625,7 @@ f(x) = yt(x)
                (cons (car e) args)))
 
             ;; metadata expressions
-            ((line meta inbounds loopinfo gc_preserve_end aliasscope popaliasscope inline noinline)
+            ((line meta inbounds loopinfo ivdepscope gc_preserve_end aliasscope popaliasscope inline noinline)
              (let ((have-ret? (and (pair? code) (pair? (car code)) (eq? (caar code) 'return))))
                (cond ((eq? (car e) 'line)
                       (set! current-loc e)

src/julia_internal.h

@@ -534,6 +534,7 @@ JL_DLLEXPORT jl_value_t *jl_get_exceptionf(jl_datatype_t *exception_type, const
 
 JL_DLLEXPORT jl_value_t *jl_get_keyword_sorter(jl_value_t *f);
 JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t);
+JL_DLLEXPORT void jl_ivdepscope_error(void);
 
 #define JL_CALLABLE(name)                                               \
     JL_DLLEXPORT jl_value_t *name(jl_value_t *F, jl_value_t **args, uint32_t nargs)
@@ -1408,6 +1409,7 @@ extern JL_DLLEXPORT jl_sym_t *jl_copyast_sym;
 extern JL_DLLEXPORT jl_sym_t *jl_cfunction_sym;
 extern JL_DLLEXPORT jl_sym_t *jl_pure_sym;
 extern JL_DLLEXPORT jl_sym_t *jl_loopinfo_sym;
+extern JL_DLLEXPORT jl_sym_t *jl_ivdepscope_sym;
 extern JL_DLLEXPORT jl_sym_t *jl_meta_sym;
 extern JL_DLLEXPORT jl_sym_t *jl_inert_sym;
 extern JL_DLLEXPORT jl_sym_t *jl_polly_sym;

src/llvm-simdloop.cpp

@@ -115,13 +115,11 @@ static bool markLoopInfo(Module &M, Function *marker, function_ref<LoopInfo &(Fu
 
         LoopInfo &LI = GetLI(*I->getParent()->getParent());
         Loop *L = LI.getLoopFor(I->getParent());
-        I->removeFromParent();
         if (!L)
             continue;
 
         LLVM_DEBUG(dbgs() << "LSL: loopinfo marker found\n");
         bool simd = false;
-        bool ivdep = false;
         SmallVector<Metadata *, 8> MDs;
 
         BasicBlock *Lh = L->getHeader();
@@ -144,8 +142,6 @@ static bool markLoopInfo(Module &M, Function *marker, function_ref<LoopInfo &(Fu
                         if (S->getString().startswith("julia")) {
                             if (S->getString().equals("julia.simdloop"))
                                 simd = true;
-                            if (S->getString().equals("julia.ivdep"))
-                                ivdep = true;
                             continue;
                         }
                     }
@@ -154,7 +150,9 @@ static bool markLoopInfo(Module &M, Function *marker, function_ref<LoopInfo &(Fu
             }
         }
 
-        LLVM_DEBUG(dbgs() << "LSL: simd: " << simd << " ivdep: " << ivdep << "\n");
+        LLVM_DEBUG(dbgs() << "LSL: simd: " << simd << "\n");
+        if (!simd)
+            continue;
 
         MDNode *n = L->getLoopID();
         if (n) {
@@ -173,40 +171,62 @@ static bool markLoopInfo(Module &M, Function *marker, function_ref<LoopInfo &(Fu
 
         MDNode *m = MDNode::get(Lh->getContext(), ArrayRef<Metadata *>(LoopID));
 
-        // If ivdep is true we assume that there is no memory dependency between loop iterations
+        // mark the inner-most loop is free of memory dependency within julia ivdep scope.
         // This is a fairly strong assumption and does often not hold true for generic code.
-        if (ivdep) {
-            // Mark memory references so that Loop::isAnnotatedParallel will return true for this loop.
-            for (BasicBlock *BB : L->blocks()) {
-               for (Instruction &I : *BB) {
-                   if (I.mayReadOrWriteMemory()) {
-                       I.setMetadata(LLVMContext::MD_mem_parallel_loop_access, m);
-                   }
-               }
+        int ivdep = 0;
+        for (BasicBlock *BB : L->blocks()) {
+            for (Instruction &I : *BB) {
+                if (I.hasMetadataOtherThanDebugLoc()) {
+                    if (MDNode *JLMD= I.getMetadata("julia.ivdepscope")) {
+                        ToDelete.push_back(&I);
+                        LLVM_DEBUG(dbgs() << "LSL: found julia.ivdepscope ");
+                        if (JLMD->getNumOperands() < 1)
+                            continue;
+                        if (MDString *S = dyn_cast<MDString>(JLMD->getOperand(0))) {
+                            LLVM_DEBUG(dbgs() << S->getString() << "\n");
+                            if (S->getString().equals("begin"))
+                                ivdep += 1;
+                            else
+                                ivdep -= 1;
+                        }
+                    }
+                }
+                if (ivdep > 0 &&I.mayReadOrWriteMemory()) {
+                    I.setMetadata(LLVMContext::MD_mem_parallel_loop_access, m);
+                }
             }
-            assert(L->isAnnotatedParallel());
         }
-
-        if (simd) {
-            // Mark floating-point reductions as okay to reassociate/commute.
-            for (BasicBlock::iterator I = Lh->begin(), E = Lh->end(); I != E; ++I) {
-                if (PHINode *Phi = dyn_cast<PHINode>(I))
-                    enableUnsafeAlgebraIfReduction(Phi, L);
-                else
-                    break;
-            }
+        // if ivdep != 0
+            // thrown() ??
+
+        // Mark floating-point reductions as okay to reassociate/commute.
+        for (BasicBlock::iterator I = Lh->begin(), E = Lh->end(); I != E; ++I) {
+            if (PHINode *Phi = dyn_cast<PHINode>(I))
+                enableUnsafeAlgebraIfReduction(Phi, L);
+            else
+                break;
         }
 
         Changed = true;
     }
 
     for (Instruction *I : ToDelete)
-        I->deleteValue();
+        I->eraseFromParent();
     marker->eraseFromParent();
 
     return Changed;
 }
 
+static void eraseIvdepScope(Module &M, Function *marker)
+{
+    for (User *U : marker->users()) {
+        Instruction *I = cast<Instruction>(U);
+        // remove "ivdepscope" from unreachable branch to make error message clearer.
+        if (isa<UnreachableInst>(I -> getParent() -> back()))
+            I -> eraseFromParent();
+    }
+}
+
 } // end anonymous namespace
 
 
@@ -223,7 +243,12 @@ PreservedAnalyses LowerSIMDLoop::run(Module &M, ModuleAnalysisManager &AM)
     Function *loopinfo_marker = M.getFunction("julia.loopinfo_marker");
 
     if (!loopinfo_marker)
+    {
+        Function *ivdepscope = M.getFunction("julia.ivdepscope");
+        if (ivdepscope)
+            eraseIvdepScope(M, ivdepscope);
         return PreservedAnalyses::all();
+    }
 
     FunctionAnalysisManager &FAM =
       AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
@@ -258,6 +283,11 @@ class LowerSIMDLoopLegacy : public ModulePass {
 
     if (loopinfo_marker)
         Changed |= markLoopInfo(M, loopinfo_marker, GetLI);
+    else {
+        Function *ivdepscope = M.getFunction("julia.ivdepscope");
+        if (ivdepscope)
+            eraseIvdepScope(M, ivdepscope);
+    }
 
     return Changed;
   }

src/macroexpand.scm

@@ -354,7 +354,7 @@
                                    ,(resolve-expansion-vars-with-new-env (caddr arg) env m parent-scope inarg))))
                              (else
                               `(global ,(resolve-expansion-vars-with-new-env arg env m parent-scope inarg))))))
-           ((using import export meta line inbounds boundscheck loopinfo inline noinline) (map unescape e))
+           ((using import export meta line inbounds boundscheck loopinfo ivdepscope inline noinline) (map unescape e))
            ((macrocall) e) ; invalid syntax anyways, so just act like it's quoted.
            ((symboliclabel) e)
            ((symbolicgoto) e)

src/method.c

@@ -84,6 +84,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve
             e->head == jl_quote_sym || e->head == jl_inert_sym ||
             e->head == jl_meta_sym || e->head == jl_inbounds_sym ||
             e->head == jl_boundscheck_sym || e->head == jl_loopinfo_sym ||
+            e->head == jl_ivdepscope_sym ||
             e->head == jl_aliasscope_sym || e->head == jl_popaliasscope_sym ||
             e->head == jl_inline_sym || e->head == jl_noinline_sym) {
             // ignore these

src/rtutils.c

@@ -214,6 +214,11 @@ JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t)
         jl_type_error("typeassert", t, x);
 }
 
+JL_DLLEXPORT void jl_ivdepscope_error(void)
+{
+    jl_errorf("Found ivdepscope outside @simd.");
+}
+
 #ifndef HAVE_SSP
 JL_DLLEXPORT uintptr_t __stack_chk_guard = (uintptr_t)0xBAD57ACCBAD67ACC; // 0xBADSTACKBADSTACK
 

test/simdloop.jl

@@ -161,3 +161,10 @@ Base.SimdLoop.simd_index(v::iter31113, j, i) = j
 Base.SimdLoop.simd_inner_length(v::iter31113, j) = 1
 Base.SimdLoop.simd_outer_range(v::iter31113) = v
 @test 2001000 == simd_sum_over_array(iter31113(Vector(1:2000)))
+
+#@ivdep thrown
+ivdep_out_simd(x) = @inbounds for i in eachindex(x)
+    Base.@ivdep x[i] += 1
+end
+@test_throws "Found ivdepscope outside @simd." ivdep_out_simd([1,2,3,4])
+@test_throws MethodError ivdep_out_simd((1,2,3,4))