Sync to upstream/release/596 (luau-lang#1050)

- Cleaned up `FFlag::FixFindBindingAtFunctionName`,
`FFlag::LuauNormalizeBlockedTypes`, `FFlag::LuauPCallDebuggerFix`
- Added support for break and continue into control flow analysis
- The old type unification engine will now report a more fine-grained
error at times, indicating that type normalization in particular failed

# New Type Solver

- Refactor of Unifier2, the new unification implementation for Luau
- Completed MVP of new unification implementation
- Dramatically simplified overload selection logic
- Type family reduction can now apply sooner to free types that have
been solved
- Subtyping now supports table indexers
- Generalization now replaces bad generics with unknown

# Native Code Generation

- Reduce stack spills caused by FINDUPVAL and STORE_TAG
- Improve Generate SHL/SHR/SAR/rotates with immediate operands in X64
- Removed redundant case re-check in table lookup fallback

---------

Co-authored-by: Arseny Kapoulkine <[email protected]>
Co-authored-by: Vyacheslav Egorov <[email protected]>
Co-authored-by: Andy Friesen <[email protected]>
Co-authored-by: Lily Brown <[email protected]>

Author: 5 people

Analysis/include/Luau/ConstraintSolver.h

@@ -203,8 +203,9 @@ struct ConstraintSolver
      * the result.
      * @param subType the sub-type to unify.
      * @param superType the super-type to unify.
+     * @returns optionally a unification too complex error if unification failed
      */
-    ErrorVec unify(NotNull<Scope> scope, Location location, TypeId subType, TypeId superType);
+    std::optional<TypeError> unify(NotNull<Scope> scope, Location location, TypeId subType, TypeId superType);
 
     /**
      * Creates a new Unifier and performs a single unification operation. Commits
@@ -233,6 +234,15 @@ struct ConstraintSolver
     void reportError(TypeErrorData&& data, const Location& location);
     void reportError(TypeError e);
 
+    /**
+     * Checks the existing set of constraints to see if there exist any that contain
+     * the provided free type, indicating that it is not yet ready to be replaced by
+     * one of its bounds.
+     * @param ty the free type that to check for related constraints
+     * @returns whether or not it is unsafe to replace the free type by one of its bounds
+     */
+    bool hasUnresolvedConstraints(TypeId ty);
+
 private:
 
     /** Helper used by tryDispatch(SubtypeConstraint) and

Analysis/include/Luau/Subtyping.h

@@ -124,6 +124,8 @@ struct Subtyping
     SubtypingResult isCovariantWith(const PrimitiveType* subPrim, const TableType* superTable);
     SubtypingResult isCovariantWith(const SingletonType* subSingleton, const TableType* superTable);
 
+    SubtypingResult isCovariantWith(const TableIndexer& subIndexer, const TableIndexer& superIndexer);
+
     SubtypingResult isCovariantWith(const NormalizedType* subNorm, const NormalizedType* superNorm);
     SubtypingResult isCovariantWith(const NormalizedClassType& subClass, const NormalizedClassType& superClass);
     SubtypingResult isCovariantWith(const NormalizedClassType& subClass, const TypeIds& superTables);

Analysis/include/Luau/Type.h

@@ -141,8 +141,6 @@ struct BlockedType
 {
     BlockedType();
     int index;
-
-    static int DEPRECATED_nextIndex;
 };
 
 struct PrimitiveType

Analysis/include/Luau/TypeFamily.h

@@ -54,7 +54,7 @@ struct TypeFamily
 
     /// The reducer function for the type family.
     std::function<TypeFamilyReductionResult<TypeId>(std::vector<TypeId>, std::vector<TypePackId>, NotNull<TypeArena>, NotNull<BuiltinTypes>,
-        NotNull<TxnLog>, NotNull<Scope>, NotNull<Normalizer>)>
+        NotNull<TxnLog>, NotNull<Scope>, NotNull<Normalizer>, ConstraintSolver*)>
         reducer;
 };
 
@@ -68,7 +68,7 @@ struct TypePackFamily
 
     /// The reducer function for the type pack family.
     std::function<TypeFamilyReductionResult<TypePackId>(std::vector<TypeId>, std::vector<TypePackId>, NotNull<TypeArena>, NotNull<BuiltinTypes>,
-        NotNull<TxnLog>, NotNull<Scope>, NotNull<Normalizer>)>
+        NotNull<TxnLog>, NotNull<Scope>, NotNull<Normalizer>, ConstraintSolver*)>
         reducer;
 };
 
@@ -81,37 +81,69 @@ struct FamilyGraphReductionResult
     DenseHashSet<TypePackId> reducedPacks{nullptr};
 };
 
+/**
+  * Attempt to reduce all instances of any type or type pack family in the type
+  * graph provided.
+  *
+  * @param entrypoint the entry point to the type graph.
+  * @param location the location the reduction is occurring at; used to populate
+  * type errors.
+  * @param arena an arena to allocate types into.
+  * @param builtins the built-in types.
+  * @param normalizer the normalizer to use when normalizing types
+  * @param log a TxnLog to use. If one is provided, substitution will take place
+  * against the TxnLog, otherwise substitutions will directly mutate the type
+  * graph. Do not provide the empty TxnLog, as a result.
+  */
+FamilyGraphReductionResult reduceFamilies(TypeId entrypoint, Location location, NotNull<TypeArena> arena, NotNull<BuiltinTypes> builtins,
+    NotNull<Scope> scope, NotNull<Normalizer> normalizer, TxnLog* log = nullptr, bool force = false);
+
+ /**
+  * Attempt to reduce all instances of any type or type pack family in the type
+  * graph provided.
+  *
+  * @param entrypoint the entry point to the type graph.
+  * @param location the location the reduction is occurring at; used to populate
+  * type errors.
+  * @param arena an arena to allocate types into.
+  * @param builtins the built-in types.
+  * @param normalizer the normalizer to use when normalizing types
+  * @param log a TxnLog to use. If one is provided, substitution will take place
+  * against the TxnLog, otherwise substitutions will directly mutate the type
+  * graph. Do not provide the empty TxnLog, as a result.
+  */
+FamilyGraphReductionResult reduceFamilies(TypePackId entrypoint, Location location, NotNull<TypeArena> arena, NotNull<BuiltinTypes> builtins,
+    NotNull<Scope> scope, NotNull<Normalizer> normalizer, TxnLog* log = nullptr, bool force = false);
+
 /**
  * Attempt to reduce all instances of any type or type pack family in the type
  * graph provided.
  *
+ * @param solver the constraint solver this reduction is being performed in.
  * @param entrypoint the entry point to the type graph.
  * @param location the location the reduction is occurring at; used to populate
  * type errors.
- * @param arena an arena to allocate types into.
- * @param builtins the built-in types.
  * @param log a TxnLog to use. If one is provided, substitution will take place
  * against the TxnLog, otherwise substitutions will directly mutate the type
  * graph. Do not provide the empty TxnLog, as a result.
  */
-FamilyGraphReductionResult reduceFamilies(TypeId entrypoint, Location location, NotNull<TypeArena> arena, NotNull<BuiltinTypes> builtins,
-    NotNull<Scope> scope, NotNull<Normalizer> normalizer, TxnLog* log = nullptr, bool force = false);
+FamilyGraphReductionResult reduceFamilies(
+    NotNull<ConstraintSolver> solver, TypeId entrypoint, Location location, NotNull<Scope> scope, TxnLog* log = nullptr, bool force = false);
 
 /**
  * Attempt to reduce all instances of any type or type pack family in the type
  * graph provided.
  *
+ * @param solver the constraint solver this reduction is being performed in.
  * @param entrypoint the entry point to the type graph.
  * @param location the location the reduction is occurring at; used to populate
  * type errors.
- * @param arena an arena to allocate types into.
- * @param builtins the built-in types.
  * @param log a TxnLog to use. If one is provided, substitution will take place
  * against the TxnLog, otherwise substitutions will directly mutate the type
  * graph. Do not provide the empty TxnLog, as a result.
  */
-FamilyGraphReductionResult reduceFamilies(TypePackId entrypoint, Location location, NotNull<TypeArena> arena, NotNull<BuiltinTypes> builtins,
-    NotNull<Scope> scope, NotNull<Normalizer> normalizer, TxnLog* log = nullptr, bool force = false);
+FamilyGraphReductionResult reduceFamilies(
+    NotNull<ConstraintSolver> solver, TypePackId entrypoint, Location location, NotNull<Scope> scope, TxnLog* log = nullptr, bool force = false);
 
 struct BuiltinTypeFamilies
 {

Analysis/include/Luau/Unifier2.h

@@ -4,6 +4,9 @@
 
 #include "Luau/DenseHash.h"
 #include "Luau/NotNull.h"
+#include "Type.h"
+#include "TypeCheckLimits.h"
+#include "TypeChecker2.h"
 
 #include <optional>
 #include <vector>
@@ -26,16 +29,44 @@ enum class OccursCheckResult
     Fail
 };
 
+struct TypePairHash
+{
+    size_t hashOne(Luau::TypeId key) const
+    {
+        return (uintptr_t(key) >> 4) ^ (uintptr_t(key) >> 9);
+    }
+
+    size_t hashOne(Luau::TypePackId key) const
+    {
+        return (uintptr_t(key) >> 4) ^ (uintptr_t(key) >> 9);
+    }
+
+    size_t operator()(const std::pair<Luau::TypeId, Luau::TypeId>& x) const
+    {
+        return hashOne(x.first) ^ (hashOne(x.second) << 1);
+    }
+
+    size_t operator()(const std::pair<Luau::TypePackId, Luau::TypePackId>& x) const
+    {
+        return hashOne(x.first) ^ (hashOne(x.second) << 1);
+    }
+};
+
 struct Unifier2
 {
     NotNull<TypeArena> arena;
     NotNull<BuiltinTypes> builtinTypes;
+    NotNull<Scope> scope;
     NotNull<InternalErrorReporter> ice;
+    TypeCheckLimits limits;
+
+    DenseHashSet<std::pair<TypeId, TypeId>, TypePairHash> seenTypePairings{{nullptr, nullptr}};
+    DenseHashSet<std::pair<TypePackId, TypePackId>, TypePairHash> seenTypePackPairings{{nullptr, nullptr}};
 
     int recursionCount = 0;
     int recursionLimit = 0;
 
-    Unifier2(NotNull<TypeArena> arena, NotNull<BuiltinTypes> builtinTypes, NotNull<InternalErrorReporter> ice);
+    Unifier2(NotNull<TypeArena> arena, NotNull<BuiltinTypes> builtinTypes, NotNull<Scope> scope, NotNull<InternalErrorReporter> ice);
 
     /** Attempt to commit the subtype relation subTy <: superTy to the type
      * graph.
@@ -50,11 +81,18 @@ struct Unifier2
      * free TypePack to another and encounter an occurs check violation.
      */
     bool unify(TypeId subTy, TypeId superTy);
+    bool unify(TypeId subTy, const FunctionType* superFn);
+    bool unify(const UnionType* subUnion, TypeId superTy);
+    bool unify(TypeId subTy, const UnionType* superUnion);
+    bool unify(const IntersectionType* subIntersection, TypeId superTy);
+    bool unify(TypeId subTy, const IntersectionType* superIntersection);
+    bool unify(const TableType* subTable, const TableType* superTable);
+    bool unify(const MetatableType* subMetatable, const MetatableType* superMetatable);
 
     // TODO think about this one carefully.  We don't do unions or intersections of type packs
     bool unify(TypePackId subTp, TypePackId superTp);
 
-    std::optional<TypeId> generalize(NotNull<Scope> scope, TypeId ty);
+    std::optional<TypeId> generalize(TypeId ty);
 
 private:
     /**

Analysis/include/Luau/VisitType.h

@@ -226,11 +226,16 @@ struct GenericTypeVisitor
             {
                 if (visit(ty, *ftv))
                 {
-                    LUAU_ASSERT(ftv->lowerBound);
-                    traverse(ftv->lowerBound);
-
-                    LUAU_ASSERT(ftv->upperBound);
-                    traverse(ftv->upperBound);
+                    // TODO: Replace these if statements with assert()s when we
+                    // delete FFlag::DebugLuauDeferredConstraintResolution.
+                    //
+                    // When the old solver is used, these pointers are always
+                    // unused. When the new solver is used, they are never null.
+                    if (ftv->lowerBound)
+                        traverse(ftv->lowerBound);
+
+                    if (ftv->upperBound)
+                        traverse(ftv->upperBound);
                 }
             }
             else

Analysis/src/AstQuery.cpp

@@ -12,7 +12,6 @@
 #include <algorithm>
 
 LUAU_FASTFLAG(DebugLuauReadWriteProperties)
-LUAU_FASTFLAGVARIABLE(FixFindBindingAtFunctionName, false);
 
 namespace Luau
 {
@@ -151,19 +150,12 @@ struct FindNode : public AstVisitor
 
     bool visit(AstStatFunction* node) override
     {
-        if (FFlag::FixFindBindingAtFunctionName)
-        {
-            visit(static_cast<AstNode*>(node));
-            if (node->name->location.contains(pos))
-                node->name->visit(this);
-            else if (node->func->location.contains(pos))
-                node->func->visit(this);
-            return false;
-        }
-        else
-        {
-            return AstVisitor::visit(node);
-        }
+        visit(static_cast<AstNode*>(node));
+        if (node->name->location.contains(pos))
+            node->name->visit(this);
+        else if (node->func->location.contains(pos))
+            node->func->visit(this);
+        return false;
     }
 
     bool visit(AstStatBlock* block) override
@@ -208,19 +200,12 @@ struct FindFullAncestry final : public AstVisitor
 
     bool visit(AstStatFunction* node) override
     {
-        if (FFlag::FixFindBindingAtFunctionName)
-        {
-            visit(static_cast<AstNode*>(node));
-            if (node->name->location.contains(pos))
-                node->name->visit(this);
-            else if (node->func->location.contains(pos))
-                node->func->visit(this);
-            return false;
-        }
-        else
-        {
-            return AstVisitor::visit(node);
-        }
+        visit(static_cast<AstNode*>(node));
+        if (node->name->location.contains(pos))
+            node->name->visit(this);
+        else if (node->func->location.contains(pos))
+            node->func->visit(this);
+        return false;
     }
 
     bool visit(AstNode* node) override