!fixup address latest comments, thanks

Author: fhahn

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -1067,7 +1067,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
     /// Extracts all lanes from its (non-scalable) vector operand. This is an
     /// abstract VPInstruction whose single defined VPValue represents VF
     /// scalars extracted from a vector, to be replaced by VF ExtractElement
-    /// VPInstructions?
+    /// VPInstructions.
     Unpack,
   };
 

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -1285,6 +1285,9 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
     return getNumOperands() > 1;
   case VPInstruction::PtrAdd:
     return Op == getOperand(0) || vputils::onlyFirstLaneUsed(this);
+  case VPInstruction::WidePtrAdd:
+    // WidePtrAdd supports scalar and vector base addresses.
+    return false;
   case VPInstruction::ComputeAnyOfResult:
   case VPInstruction::ComputeFindIVResult:
     return Op == getOperand(1);

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -3800,35 +3800,41 @@ void VPlanTransforms::materializePacksAndUnpacks(VPlan &Plan) {
   // Create explicit VPInstructions to convert vectors to scalars.
   for (VPBasicBlock *VPBB : VPBBsInsideLoopRegion) {
     for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
-      if (isa<VPReplicateRecipe, VPInstruction, VPScalarIVStepsRecipe>(&R))
+      if (isa<VPReplicateRecipe, VPInstruction, VPScalarIVStepsRecipe,
+              VPDerivedIVRecipe, VPCanonicalIVPHIRecipe>(&R))
         continue;
       for (VPValue *Def : R.definedValues()) {
+        // Skip recipes that are single-scalar or only have their first lane
+        // used.
+        // TODO: The Defs skipped here may or may not be vector values.
+        // Introduce Unpacks, and remove them later, if they are guaranteed to
+        // produce scalar values.
         if (vputils::isSingleScalar(Def) || vputils::onlyFirstLaneUsed(Def))
           continue;
 
-        auto IsInsideReplicateRegion = [LoopRegion](VPUser *U) {
+        // At the moment, we only create unpacks for scalar users outside
+        // replicate regions. Recipes inside replicate regions still manually
+        // extract the required lanes.
+        // TODO: Remove once replicate regions are
+        // unrolled completely.
+        auto IsCandidateUnpackUser = [Def](VPUser *U) {
           VPRegionBlock *ParentRegion =
               cast<VPRecipeBase>(U)->getParent()->getParent();
-          return ParentRegion && ParentRegion != LoopRegion;
+          return U->usesScalars(Def) &&
+                 (!ParentRegion || !ParentRegion->isReplicator());
         };
-        // At the moment, we only create unpacks for scalar users outside
-        // replicate regions. Recipes inside replicate regions still manually
-        // extract the required lanes. TODO: Remove once replicate regions are
-        // unrolled explicitly.
-        if (none_of(Def->users(), [Def, &IsInsideReplicateRegion](VPUser *U) {
-              return !IsInsideReplicateRegion(U) && U->usesScalars(Def);
-            }))
+        if (none_of(Def->users(), IsCandidateUnpackUser))
           continue;
 
         auto *Unpack = new VPInstruction(VPInstruction::Unpack, {Def});
         if (R.isPhi())
           Unpack->insertBefore(*VPBB, VPBB->getFirstNonPhi());
         else
           Unpack->insertAfter(&R);
-        Def->replaceUsesWithIf(
-            Unpack, [Def, &IsInsideReplicateRegion](VPUser &U, unsigned) {
-              return !IsInsideReplicateRegion(&U) && U.usesScalars(Def);
-            });
+        Def->replaceUsesWithIf(Unpack,
+                               [&IsCandidateUnpackUser](VPUser &U, unsigned) {
+                                 return IsCandidateUnpackUser(&U);
+                               });
       }
     }
   }

llvm/lib/Transforms/Vectorize/VPlanTransforms.h

@@ -326,7 +326,7 @@ struct VPlanTransforms {
                                             VPBasicBlock *VectorPH);
 
   /// Add explicit Build[Struct]Vector recipes that combine multiple scalar
-  /// values into single vectors and UnpackVector to extract scalars from a
+  /// values into single vectors and Unpack recipes to extract scalars from a
   /// vector as needed.
   static void materializePacksAndUnpacks(VPlan &Plan);