[cfe] Account for extension type self-dependency via raw types

Closes #54241

Change-Id: I7afcaab225f93763f0dd2852bb51227c05abe4bb
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/340403
Reviewed-by: Johnni Winther <[email protected]>
Commit-Queue: Chloe Stefantsova <[email protected]>

pkg/front_end/lib/src/fasta/source/source_extension_type_declaration_builder.dart

@@ -2,8 +2,6 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'package:front_end/src/fasta/builder/record_type_builder.dart';
-import 'package:front_end/src/fasta/kernel/body_builder_context.dart';
 import 'package:kernel/ast.dart';
 import 'package:kernel/class_hierarchy.dart';
 import 'package:kernel/core_types.dart';
@@ -20,10 +18,16 @@ import '../builder/library_builder.dart';
 import '../builder/member_builder.dart';
 import '../builder/metadata_builder.dart';
 import '../builder/name_iterator.dart';
+import '../builder/record_type_builder.dart';
 import '../builder/type_builder.dart';
+import '../dill/dill_class_builder.dart';
+import '../dill/dill_extension_type_declaration_builder.dart';
+import '../dill/dill_type_alias_builder.dart';
+import '../kernel/body_builder_context.dart';
 import '../kernel/hierarchy/hierarchy_builder.dart';
 import '../kernel/kernel_helper.dart';
 import '../kernel/type_algorithms.dart';
+import '../kernel/type_builder_computer.dart';
 import '../messages.dart';
 import '../problems.dart';
 import '../scope.dart';
@@ -373,6 +377,65 @@ class SourceExtensionTypeDeclarationBuilder
               return true;
             }
           }
+        } else if (declaration != null && declaration.typeVariablesCount > 0) {
+          List<TypeVariableBuilderBase>? typeParameters;
+          List<TypeParameter>? typeParametersFromKernel;
+          bool isFromKernel = false;
+          switch (declaration) {
+            case ClassBuilder():
+              typeParameters = declaration.typeVariables;
+              if (declaration is DillClassBuilder) {
+                isFromKernel = true;
+                typeParametersFromKernel = declaration.cls.typeParameters;
+              }
+            case TypeAliasBuilder():
+              typeParameters = declaration.typeVariables;
+              if (declaration is DillTypeAliasBuilder) {
+                isFromKernel = true;
+                typeParametersFromKernel = declaration.typedef.typeParameters;
+              }
+            case ExtensionTypeDeclarationBuilder():
+              typeParameters = declaration.typeParameters;
+              if (declaration is DillExtensionTypeDeclarationBuilder) {
+                isFromKernel = true;
+                typeParametersFromKernel =
+                    declaration.extensionTypeDeclaration.typeParameters;
+              }
+            case BuiltinTypeDeclarationBuilder():
+            case InvalidTypeDeclarationBuilder():
+            case OmittedTypeDeclarationBuilder():
+            case ExtensionBuilder():
+            case TypeVariableBuilderBase():
+          }
+          if (typeParameters != null) {
+            TypeBuilderComputer? typeBuilderComputer;
+            for (int i = 0; i < typeParameters.length; i++) {
+              TypeVariableBuilderBase typeParameter = typeParameters[i];
+              if (!isFromKernel) {
+                if (_checkRepresentationDependency(
+                    typeParameter.defaultType!,
+                    rootExtensionTypeDeclaration,
+                    seenExtensionTypeDeclarations.toSet(),
+                    usedTypeAliasBuilders.toSet())) {
+                  return true;
+                }
+              } else if (typeParametersFromKernel != null) {
+                assert(
+                    typeParameters.length == typeParametersFromKernel.length);
+                typeBuilderComputer ??=
+                    new TypeBuilderComputer(libraryBuilder.loader);
+                if (_checkRepresentationDependency(
+                    typeParametersFromKernel[i]
+                        .defaultType
+                        .accept(typeBuilderComputer),
+                    rootExtensionTypeDeclaration,
+                    seenExtensionTypeDeclarations.toSet(),
+                    usedTypeAliasBuilders.toSet())) {
+                  return true;
+                }
+              }
+            }
+          }
         }
       case FunctionTypeBuilder(
           :List<StructuralVariableBuilder>? typeVariables,

pkg/front_end/testcases/extension_types/issue54241.dart

@@ -0,0 +1,6 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+class A<X extends E> {}
+extension type E(Function<Y extends A>() it) {}

pkg/front_end/testcases/extension_types/issue54241.dart.strong.expect

@@ -0,0 +1,27 @@
+library;
+//
+// Problems in library:
+//
+// pkg/front_end/testcases/extension_types/issue54241.dart:6:18: Error: An extension type can't depend on itself through its representation type.
+// extension type E(Function<Y extends A>() it) {}
+//                  ^
+//
+import self as self;
+import "dart:core" as core;
+
+class A<X extends self::E /* = invalid-type */> extends core::Object {
+  synthetic constructor •() → self::A<self::A::X%>
+    : super core::Object::•()
+    ;
+}
+extension type E(invalid-type it) {
+  abstract extension-type-member representation-field get it() → <Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic;
+  constructor • = self::E|constructor#;
+  constructor tearoff • = self::E|constructor#_#new#tearOff;
+}
+static extension-type-member method E|constructor#(<Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic it) → self::E /* = invalid-type */ {
+  lowered final self::E /* = invalid-type */ #this = it;
+  return #this;
+}
+static extension-type-member method E|constructor#_#new#tearOff(<Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic it) → self::E /* = invalid-type */
+  return self::E|constructor#(it);

pkg/front_end/testcases/extension_types/issue54241.dart.strong.transformed.expect

@@ -0,0 +1,27 @@
+library;
+//
+// Problems in library:
+//
+// pkg/front_end/testcases/extension_types/issue54241.dart:6:18: Error: An extension type can't depend on itself through its representation type.
+// extension type E(Function<Y extends A>() it) {}
+//                  ^
+//
+import self as self;
+import "dart:core" as core;
+
+class A<X extends self::E /* = invalid-type */> extends core::Object {
+  synthetic constructor •() → self::A<self::A::X%>
+    : super core::Object::•()
+    ;
+}
+extension type E(invalid-type it) {
+  abstract extension-type-member representation-field get it() → <Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic;
+  constructor • = self::E|constructor#;
+  constructor tearoff • = self::E|constructor#_#new#tearOff;
+}
+static extension-type-member method E|constructor#(<Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic it) → self::E /* = invalid-type */ {
+  lowered final self::E /* = invalid-type */ #this = it;
+  return #this;
+}
+static extension-type-member method E|constructor#_#new#tearOff(<Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic it) → self::E /* = invalid-type */
+  return self::E|constructor#(it);

pkg/front_end/testcases/extension_types/issue54241.dart.textual_outline.expect

@@ -0,0 +1,3 @@
+class A<X extends E> {}
+
+extension type E(Function<Y extends A>() it) {}

pkg/front_end/testcases/extension_types/issue54241.dart.textual_outline_modelled.expect

@@ -0,0 +1,3 @@
+class A<X extends E> {}
+
+extension type E(Function<Y extends A>() it) {}

pkg/front_end/testcases/extension_types/issue54241.dart.weak.expect

@@ -0,0 +1,27 @@
+library;
+//
+// Problems in library:
+//
+// pkg/front_end/testcases/extension_types/issue54241.dart:6:18: Error: An extension type can't depend on itself through its representation type.
+// extension type E(Function<Y extends A>() it) {}
+//                  ^
+//
+import self as self;
+import "dart:core" as core;
+
+class A<X extends self::E /* = invalid-type */> extends core::Object {
+  synthetic constructor •() → self::A<self::A::X%>
+    : super core::Object::•()
+    ;
+}
+extension type E(invalid-type it) {
+  abstract extension-type-member representation-field get it() → <Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic;
+  constructor • = self::E|constructor#;
+  constructor tearoff • = self::E|constructor#_#new#tearOff;
+}
+static extension-type-member method E|constructor#(<Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic it) → self::E /* = invalid-type */ {
+  lowered final self::E /* = invalid-type */ #this = it;
+  return #this;
+}
+static extension-type-member method E|constructor#_#new#tearOff(<Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic it) → self::E /* = invalid-type */
+  return self::E|constructor#(it);

pkg/front_end/testcases/extension_types/issue54241.dart.weak.modular.expect

@@ -0,0 +1,27 @@
+library;
+//
+// Problems in library:
+//
+// pkg/front_end/testcases/extension_types/issue54241.dart:6:18: Error: An extension type can't depend on itself through its representation type.
+// extension type E(Function<Y extends A>() it) {}
+//                  ^
+//
+import self as self;
+import "dart:core" as core;
+
+class A<X extends self::E /* = invalid-type */> extends core::Object {
+  synthetic constructor •() → self::A<self::A::X%>
+    : super core::Object::•()
+    ;
+}
+extension type E(invalid-type it) {
+  abstract extension-type-member representation-field get it() → <Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic;
+  constructor • = self::E|constructor#;
+  constructor tearoff • = self::E|constructor#_#new#tearOff;
+}
+static extension-type-member method E|constructor#(<Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic it) → self::E /* = invalid-type */ {
+  lowered final self::E /* = invalid-type */ #this = it;
+  return #this;
+}
+static extension-type-member method E|constructor#_#new#tearOff(<Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic it) → self::E /* = invalid-type */
+  return self::E|constructor#(it);

pkg/front_end/testcases/extension_types/issue54241.dart.weak.outline.expect

@@ -0,0 +1,24 @@
+library;
+//
+// Problems in library:
+//
+// pkg/front_end/testcases/extension_types/issue54241.dart:6:18: Error: An extension type can't depend on itself through its representation type.
+// extension type E(Function<Y extends A>() it) {}
+//                  ^
+//
+import self as self;
+import "dart:core" as core;
+
+class A<X extends self::E /* = invalid-type */> extends core::Object {
+  synthetic constructor •() → self::A<self::A::X%>
+    ;
+}
+extension type E(invalid-type it) {
+  abstract extension-type-member representation-field get it() → <Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic;
+  constructor • = self::E|constructor#;
+  constructor tearoff • = self::E|constructor#_#new#tearOff;
+}
+static extension-type-member method E|constructor#(<Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic it) → self::E /* = invalid-type */
+  ;
+static extension-type-member method E|constructor#_#new#tearOff(<Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic it) → self::E /* = invalid-type */
+  return self::E|constructor#(it);

pkg/front_end/testcases/extension_types/issue54241.dart.weak.transformed.expect

@@ -0,0 +1,27 @@
+library;
+//
+// Problems in library:
+//
+// pkg/front_end/testcases/extension_types/issue54241.dart:6:18: Error: An extension type can't depend on itself through its representation type.
+// extension type E(Function<Y extends A>() it) {}
+//                  ^
+//
+import self as self;
+import "dart:core" as core;
+
+class A<X extends self::E /* = invalid-type */> extends core::Object {
+  synthetic constructor •() → self::A<self::A::X%>
+    : super core::Object::•()
+    ;
+}
+extension type E(invalid-type it) {
+  abstract extension-type-member representation-field get it() → <Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic;
+  constructor • = self::E|constructor#;
+  constructor tearoff • = self::E|constructor#_#new#tearOff;
+}
+static extension-type-member method E|constructor#(<Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic it) → self::E /* = invalid-type */ {
+  lowered final self::E /* = invalid-type */ #this = it;
+  return #this;
+}
+static extension-type-member method E|constructor#_#new#tearOff(<Y extends self::A<self::E /* = invalid-type */> = dynamic>() → dynamic it) → self::E /* = invalid-type */
+  return self::E|constructor#(it);