Scheduler ci yaml merge (#4006)

CiYaml now tracks one or more ci.yaml files in a fusion tree

CiYamlInner is the old CiYaml. Instead of merging or updating protobufs to know which target are engine targets and which are framework targets; keep them apart. We'll have to use this information later when scheduling engine builds first and waiting before we can schedule the test runs.

The fusion-engine-ciyaml will need to be updated to include paths; that needs to be added to the merge script.

- Downloading a ci.yaml path should respect the path requested
- Only adding the ability to find targets; scheduling to follow this PR.

The `settings.json` file has only the dart line length specified; which should be a shared setting between all developers as it is enforced by CI.

app_dart/.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "dart.lineLength": 120
+}

app_dart/bin/gae_server.dart

@@ -39,12 +39,15 @@ Future<void> main() async {
     // Gerrit service class to communicate with GoB.
     final GerritService gerritService = GerritService(config: config);
 
+    final fusionTester = FusionTester();
+
     /// Cocoon scheduler service to manage validating commits in presubmit and postsubmit.
     final Scheduler scheduler = Scheduler(
       cache: cache,
       config: config,
       githubChecksService: githubChecksService,
       luciBuildService: luciBuildService,
+      fusionTester: fusionTester,
     );
 
     final BranchService branchService = BranchService(

app_dart/bin/local_server.dart

@@ -46,12 +46,15 @@ Future<void> main() async {
   // Gerrit service class to communicate with GoB.
   final GerritService gerritService = GerritService(config: config);
 
+  final fusionTester = FusionTester();
+
   /// Cocoon scheduler service to manage validating commits in presubmit and postsubmit.
   final Scheduler scheduler = Scheduler(
     cache: cache,
     config: config,
     githubChecksService: githubChecksService,
     luciBuildService: luciBuildService,
+    fusionTester: fusionTester,
   );
 
   final BranchService branchService = BranchService(

app_dart/bin/validate_scheduler_config.dart

@@ -25,6 +25,7 @@ void main(List<String> args) async {
   final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(configYaml);
   print(
     CiYaml(
+      type: CiType.any,
       slug: Config.flutterSlug,
       branch: Config.defaultBranch(Config.flutterSlug),
       config: unCheckedSchedulerConfig,

app_dart/integration_test/validate_all_ci_configs_test.dart

@@ -37,6 +37,7 @@ Future<void> main() async {
       final pb.SchedulerConfig currentSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(configYaml);
       try {
         CiYaml(
+          type: CiType.any,
           slug: config.slug,
           branch: Config.defaultBranch(config.slug),
           config: currentSchedulerConfig,
@@ -59,6 +60,7 @@ Future<void> main() async {
         final pb.SchedulerConfig schedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(configYaml);
         // Validate using the existing CiYaml logic.
         CiYaml(
+          type: CiType.any,
           slug: config.slug,
           branch: config.branch,
           config: schedulerConfig,

app_dart/lib/server.dart

@@ -23,7 +23,9 @@ Server createServer({
   required CommitService commitService,
   required GerritService gerritService,
   required Scheduler scheduler,
+  FusionTester? fusionTester,
 }) {
+  fusionTester ??= FusionTester();
   final Map<String, RequestHandler<dynamic>> handlers = <String, RequestHandler<dynamic>>{
     '/api/check_flaky_builders': CheckFlakyBuilders(
       config: config,
@@ -67,7 +69,7 @@ Server createServer({
       gerritService: gerritService,
       scheduler: scheduler,
       commitService: commitService,
-      fusionTester: FusionTester(),
+      fusionTester: fusionTester,
     ),
     '/api/v2/presubmit-luci-subscription': PresubmitLuciSubscription(
       cache: cache,

app_dart/lib/src/foundation/utils.dart

@@ -22,6 +22,7 @@ import '../request_handling/exceptions.dart';
 import '../service/logging.dart';
 
 const String kCiYamlPath = '.ci.yaml';
+const String kCiYamlFusionEnginePath = 'engine/src/flutter/$kCiYamlPath';
 const String kTestOwnerPath = 'TESTOWNERS';
 
 /// Signature for a function that calculates the backoff duration to wait in
@@ -293,13 +294,13 @@ List<String> validateOwnership(
   final YamlMap? ciYaml = loadYaml(ciYamlContent) as YamlMap?;
   final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(ciYaml);
 
-  final CiYaml ciYamlFromProto = CiYaml(
+  final CiYamlSet ciYamlFromProto = CiYamlSet(
     slug: Config.flutterSlug,
     branch: Config.defaultBranch(Config.flutterSlug),
-    config: unCheckedSchedulerConfig,
+    yamls: {CiType.any: unCheckedSchedulerConfig},
   );
 
-  final pb.SchedulerConfig schedulerConfig = ciYamlFromProto.config;
+  final pb.SchedulerConfig schedulerConfig = ciYamlFromProto.configFor(CiType.any);
 
   for (pb.Target target in schedulerConfig.targets) {
     final String builder = target.name;

app_dart/lib/src/model/ci_yaml/ci_yaml.dart

@@ -11,19 +11,136 @@ import 'package:github/github.dart';
 import '../proto/internal/scheduler.pb.dart' as pb;
 import 'target.dart';
 
+/// Key used for selecting which .ci.yaml to use
+enum CiType {
+  /// "Default" associated with this slug / ci.yaml
+  ///
+  /// Pre-fusion: engine, framework, etc
+  /// Post-fusion: framework, etc
+  ///
+  /// NOTE: Required to be present
+  any,
+
+  /// Engine's .ci.yaml file located in the monorepo
+  fusionEngine,
+}
+
+/// Wrapper around one or more [CiYamlSet] files contained in a single repository.
+///
+/// Single sourced repositories will have exactly one `.ci.yaml` file at the
+/// root of the tree. This will have the type [CiType.any].
+///
+/// Merged repositories can have multiple `.ci.yaml` files located throughout
+/// the tree. Today we only support the root type as [CiType.any] and the second
+/// as [CiType.fusionEngine].
+class CiYamlSet {
+  CiYamlSet({
+    required this.slug,
+    required this.branch,
+    required Map<CiType, pb.SchedulerConfig> yamls,
+    CiYamlSet? totConfig,
+    bool validate = false,
+    this.isFusion = false,
+  }) {
+    for (final MapEntry(key: type, value: config) in yamls.entries) {
+      configs[type] = CiYaml(
+        slug: slug,
+        branch: branch,
+        config: config,
+        type: type,
+        validate: validate,
+        isFusion: isFusion,
+        totConfig: totConfig?.configs[type],
+      );
+    }
+  }
+
+  final bool isFusion;
+
+  final configs = <CiType, CiYaml>{};
+
+  /// Get's the [pb.SchedulerConfig] for the requested [type].
+  ///
+  /// The type is expected to exist and will fail otherwise.
+  pb.SchedulerConfig configFor(CiType type) => configs[type]!.config;
+
+  /// Get's the [CiYaml] for the requested [type].
+  ///
+  /// The type is expected to exist and will fail otherwise.
+  CiYaml ciYamlFor(CiType type) => configs[type]!;
+
+  /// The [RepositorySlug] that [config] is from.
+  final RepositorySlug slug;
+
+  /// The git branch currently being scheduled against.
+  final String branch;
+
+  /// Gets all [Target] that run on presubmit for this config.
+  List<Target> presubmitTargets({CiType type = CiType.any}) => configs[type]!.presubmitTargets;
+
+  /// Gets all [Target] that run on postsubmit for this config.
+  List<Target> postsubmitTargets({CiType type = CiType.any}) => configs[type]!.postsubmitTargets;
+
+  /// Gets the first [Target] matching [builderName] or null.
+  Target? getFirstPostsubmitTarget(
+    String builderName, {
+    CiType type = CiType.any,
+  }) =>
+      configs[type]!.getFirstPostsubmitTarget(builderName);
+
+  /// List of target names used to filter target from release candidate branches
+  /// that were already removed from main.
+  List<String>? totTargetNames({CiType type = CiType.any}) => configs[type]!.totTargetNames;
+
+  /// List of postsubmit target names used to filter target from release candidate branches
+  /// that were already removed from main.
+  List<String>? totPostsubmitTargetNames({CiType type = CiType.any}) => configs[type]!.totPostsubmitTargetNames;
+
+  /// Filters post submit targets to remove targets we do not want backfilled.
+  List<Target> backfillTargets({CiType type = CiType.any}) => configs[type]!.backfillTargets;
+
+  /// Filters targets that were removed from main. [slug] is the gihub
+  /// slug for branch under test, [targets] is the list of targets from
+  /// the branch under test and [totTargetNames] is the list of target
+  /// names enabled on the default branch.
+  List<Target> filterOutdatedTargets(
+    slug,
+    targets,
+    totTargetNames, {
+    CiType type = CiType.any,
+  }) =>
+      configs[type]!.filterOutdatedTargets(slug, targets, totTargetNames);
+
+  /// Filters [targets] to those that should be started immediately.
+  ///
+  /// Targets with a dependency are triggered when there dependency pushes a notification that it has finished.
+  /// This shouldn't be confused for targets that have the property named dependency, which is used by the
+  /// flutter_deps recipe module on LUCI.
+  List<Target> getInitialTargets(
+    List<Target> targets, {
+    CiType type = CiType.any,
+  }) =>
+      configs[type]!.getInitialTargets(targets);
+
+  /// Get an unfiltered list of all [targets] that are found in the ci.yaml file.
+  List<Target> targets({CiType type = CiType.any}) => configs[type]!.targets;
+}
+
 /// This is a wrapper class around S[pb.SchedulerConfig].
 ///
 /// See //CI_YAML.md for high level documentation.
 class CiYaml {
-  /// Creates [CiYaml] from a [RepositorySlug], [branch], [pb.SchedulerConfig] and an optional [CiYaml] of tip of tree CiYaml.
+  /// Creates [CiYamlSet] from a [RepositorySlug], [branch], [pb.SchedulerConfig] and an optional [CiYamlSet] of tip of tree CiYaml.
   ///
   /// If [totConfig] is passed, the validation will verify no new targets have been added that may temporarily break the LUCI infrastructure (such as new prod or presubmit targets).
   CiYaml({
     required this.slug,
     required this.branch,
     required this.config,
+    required this.type,
     CiYaml? totConfig,
     bool validate = false,
+    this.isFusion = false,
   }) {
     if (validate) {
       _validate(config, branch, totSchedulerConfig: totConfig?.config);
@@ -37,6 +154,10 @@ class CiYaml {
         totConfig?.postsubmitTargets.map((Target target) => target.value.name).toList() ?? <String>[];
   }
 
+  final CiType type;
+
+  final bool isFusion;
+
   /// List of target names used to filter target from release candidate branches
   /// that were already removed from main.
   List<String>? totTargetNames;
@@ -196,7 +317,7 @@ class CiYaml {
     return regexp.hasMatch(branch);
   }
 
-  /// Validates [pb.SchedulerConfig] extracted from [CiYaml] files.
+  /// Validates [pb.SchedulerConfig] extracted from [CiYamlSet] files.
   ///
   /// A [pb.SchedulerConfig] file is considered good if:
   ///   1. It has at least one [pb.Target] in [pb.SchedulerConfig.targets]
@@ -258,14 +379,27 @@ class CiYaml {
             }
           }
         }
-        // TODO(flutter#157358) - Handle merged repository yaml files.
+
         // Verify runIf includes foundational files.
         if (target.runIf.isNotEmpty) {
-          if (!target.runIf.contains('.ci.yaml')) {
-            exceptions.add('ERROR: ${target.name} is missing `.ci.yaml` in runIf');
-          }
-          if (slug == Config.engineSlug && !target.runIf.contains('DEPS')) {
-            exceptions.add('ERROR: ${target.name} is missing `DEPS` in runIf');
+          if (isFusion && type == CiType.fusionEngine) {
+            // Look in different locations if fusion && engine ci.yaml
+            if (!target.runIf.contains('engine/src/flutter/.ci.yaml')) {
+              exceptions.add(
+                'ERROR: ${target.name} is missing `engine/src/flutter/.ci.yaml` in runIf',
+              );
+            }
+            if (!target.runIf.contains('DEPS')) {
+              exceptions.add('ERROR: ${target.name} is missing `DEPS` in runIf');
+            }
+          } else {
+            // not fusion or not engine in fusion.
+            if (!target.runIf.contains('.ci.yaml')) {
+              exceptions.add('ERROR: ${target.name} is missing `.ci.yaml` in runIf');
+            }
+            if (slug == Config.engineSlug && !target.runIf.contains('DEPS')) {
+              exceptions.add('ERROR: ${target.name} is missing `DEPS` in runIf');
+            }
           }
         }
       }

app_dart/lib/src/request_handlers/check_flaky_builders.dart

@@ -60,13 +60,13 @@ class CheckFlakyBuilders extends ApiRequestHandler<Body> {
     );
     final YamlMap? ci = loadYaml(ciContent) as YamlMap?;
     final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(ci);
-    final CiYaml ciYaml = CiYaml(
+    final CiYamlSet ciYaml = CiYamlSet(
       slug: slug,
       branch: Config.defaultBranch(slug),
-      config: unCheckedSchedulerConfig,
+      yamls: {CiType.any: unCheckedSchedulerConfig},
     );
 
-    final pb.SchedulerConfig schedulerConfig = ciYaml.config;
+    final pb.SchedulerConfig schedulerConfig = ciYaml.configFor(CiType.any);
     final List<pb.Target> targets = schedulerConfig.targets;
 
     final List<_BuilderInfo> eligibleBuilders =
@@ -116,7 +116,7 @@ class CheckFlakyBuilders extends ApiRequestHandler<Body> {
     GithubService gitHub,
     RepositorySlug slug, {
     required String content,
-    required CiYaml ciYaml,
+    required CiYamlSet ciYaml,
   }) async {
     final YamlMap ci = loadYaml(content) as YamlMap;
     final YamlList targets = ci[kCiYamlTargetsKey] as YamlList;
@@ -164,7 +164,7 @@ class CheckFlakyBuilders extends ApiRequestHandler<Body> {
   }
 
   @visibleForTesting
-  static bool getIgnoreFlakiness(String? builderName, CiYaml ciYaml) {
+  static bool getIgnoreFlakiness(String? builderName, CiYamlSet ciYaml) {
     if (builderName == null) {
       return false;
     }

app_dart/lib/src/request_handlers/file_flaky_issue_and_pr.dart

@@ -41,13 +41,13 @@ class FileFlakyIssueAndPR extends ApiRequestHandler<Body> {
     final List<BuilderStatistic> builderStatisticList = await bigquery.listBuilderStatistic(kBigQueryProjectId);
     final YamlMap? ci = loadYaml(await gitHub.getFileContent(slug, kCiYamlPath)) as YamlMap?;
     final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(ci);
-    final CiYaml ciYaml = CiYaml(
+    final CiYamlSet ciYaml = CiYamlSet(
       slug: slug,
       branch: Config.defaultBranch(slug),
-      config: unCheckedSchedulerConfig,
+      yamls: {CiType.any: unCheckedSchedulerConfig},
     );
 
-    final pb.SchedulerConfig schedulerConfig = ciYaml.config;
+    final pb.SchedulerConfig schedulerConfig = ciYaml.configFor(CiType.any);
     final List<pb.Target> targets = schedulerConfig.targets;
 
     final String testOwnerContent = await gitHub.getFileContent(slug, kTestOwnerPath);
@@ -89,7 +89,7 @@ class FileFlakyIssueAndPR extends ApiRequestHandler<Body> {
     });
   }
 
-  bool shouldSkip(BuilderStatistic statistic, CiYaml ciYaml, List<pb.Target> targets) {
+  bool shouldSkip(BuilderStatistic statistic, CiYamlSet ciYaml, List<pb.Target> targets) {
     // Skips if the target has been removed from .ci.yaml.
     if (!targets.map((e) => e.name).toList().contains(statistic.name)) {
       return true;
@@ -191,9 +191,9 @@ class FileFlakyIssueAndPR extends ApiRequestHandler<Body> {
   }
 
   @visibleForTesting
-  static bool getIgnoreFlakiness(String builderName, CiYaml ciYaml) {
+  static bool getIgnoreFlakiness(String builderName, CiYamlSet ciYaml) {
     final Target? target =
-        ciYaml.postsubmitTargets.singleWhereOrNull((Target target) => target.value.name == builderName);
+        ciYaml.postsubmitTargets().singleWhereOrNull((Target target) => target.value.name == builderName);
     return target == null ? false : target.getIgnoreFlakiness();
   }
 

app_dart/lib/src/request_handlers/flaky_handler_utils.dart

@@ -303,7 +303,7 @@ TestOwnership getTestOwnership(pb.Target target, BuilderType type, String testOw
 
 /// Gets the [BuilderType] of the builder by looking up the information in the
 /// ci.yaml.
-BuilderType getTypeForBuilder(String? targetName, CiYaml ciYaml, {bool unfilteredTargets = false}) {
+BuilderType getTypeForBuilder(String? targetName, CiYamlSet ciYaml, {bool unfilteredTargets = false}) {
   final List<String>? tags = _getTags(targetName, ciYaml, unfilteredTargets: unfilteredTargets);
   if (tags == null || tags.isEmpty) {
     return BuilderType.unknown;
@@ -332,13 +332,13 @@ BuilderType getTypeForBuilder(String? targetName, CiYaml ciYaml, {bool unfiltere
   return hasFrameworkTag && hasHostOnlyTag ? BuilderType.frameworkHostOnly : BuilderType.unknown;
 }
 
-List<String>? _getTags(String? targetName, CiYaml ciYaml, {bool unfilteredTargets = false}) {
+List<String>? _getTags(String? targetName, CiYamlSet ciYaml, {bool unfilteredTargets = false}) {
   final Set<Target> allUniqueTargets = {};
   if (!unfilteredTargets) {
-    allUniqueTargets.addAll(ciYaml.presubmitTargets);
-    allUniqueTargets.addAll(ciYaml.postsubmitTargets);
+    allUniqueTargets.addAll(ciYaml.presubmitTargets());
+    allUniqueTargets.addAll(ciYaml.postsubmitTargets());
   } else {
-    allUniqueTargets.addAll(ciYaml.targets);
+    allUniqueTargets.addAll(ciYaml.targets(type: CiType.any));
   }
 
   final Target? target = allUniqueTargets.firstWhereOrNull((element) => element.value.name == targetName);