diff --git a/app_dart/.vscode/settings.json b/app_dart/.vscode/settings.json new file mode 100644 index 000000000..6ea5cb24a --- /dev/null +++ b/app_dart/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "dart.lineLength": 120 +} \ No newline at end of file diff --git a/app_dart/bin/gae_server.dart b/app_dart/bin/gae_server.dart index f4ff63bb5..5a8b4a606 100644 --- a/app_dart/bin/gae_server.dart +++ b/app_dart/bin/gae_server.dart @@ -39,12 +39,15 @@ Future main() async { // Gerrit service class to communicate with GoB. final GerritService gerritService = GerritService(config: config); + final fusionTester = FusionTester(); + /// Cocoon scheduler service to manage validating commits in presubmit and postsubmit. final Scheduler scheduler = Scheduler( cache: cache, config: config, githubChecksService: githubChecksService, luciBuildService: luciBuildService, + fusionTester: fusionTester, ); final BranchService branchService = BranchService( diff --git a/app_dart/bin/local_server.dart b/app_dart/bin/local_server.dart index 3fe049bc0..381c9284a 100644 --- a/app_dart/bin/local_server.dart +++ b/app_dart/bin/local_server.dart @@ -46,12 +46,15 @@ Future main() async { // Gerrit service class to communicate with GoB. final GerritService gerritService = GerritService(config: config); + final fusionTester = FusionTester(); + /// Cocoon scheduler service to manage validating commits in presubmit and postsubmit. final Scheduler scheduler = Scheduler( cache: cache, config: config, githubChecksService: githubChecksService, luciBuildService: luciBuildService, + fusionTester: fusionTester, ); final BranchService branchService = BranchService( diff --git a/app_dart/bin/validate_scheduler_config.dart b/app_dart/bin/validate_scheduler_config.dart index 3b3fb1168..8a3442035 100644 --- a/app_dart/bin/validate_scheduler_config.dart +++ b/app_dart/bin/validate_scheduler_config.dart @@ -25,6 +25,7 @@ void main(List args) async { final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(configYaml); print( CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, diff --git a/app_dart/integration_test/validate_all_ci_configs_test.dart b/app_dart/integration_test/validate_all_ci_configs_test.dart index a5f79d683..d20208246 100644 --- a/app_dart/integration_test/validate_all_ci_configs_test.dart +++ b/app_dart/integration_test/validate_all_ci_configs_test.dart @@ -37,6 +37,7 @@ Future main() async { final pb.SchedulerConfig currentSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(configYaml); try { CiYaml( + type: CiType.any, slug: config.slug, branch: Config.defaultBranch(config.slug), config: currentSchedulerConfig, @@ -59,6 +60,7 @@ Future main() async { final pb.SchedulerConfig schedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(configYaml); // Validate using the existing CiYaml logic. CiYaml( + type: CiType.any, slug: config.slug, branch: config.branch, config: schedulerConfig, diff --git a/app_dart/lib/server.dart b/app_dart/lib/server.dart index af3f52010..003626931 100644 --- a/app_dart/lib/server.dart +++ b/app_dart/lib/server.dart @@ -23,7 +23,9 @@ Server createServer({ required CommitService commitService, required GerritService gerritService, required Scheduler scheduler, + FusionTester? fusionTester, }) { + fusionTester ??= FusionTester(); final Map> handlers = >{ '/api/check_flaky_builders': CheckFlakyBuilders( config: config, @@ -67,7 +69,7 @@ Server createServer({ gerritService: gerritService, scheduler: scheduler, commitService: commitService, - fusionTester: FusionTester(), + fusionTester: fusionTester, ), '/api/v2/presubmit-luci-subscription': PresubmitLuciSubscription( cache: cache, diff --git a/app_dart/lib/src/foundation/utils.dart b/app_dart/lib/src/foundation/utils.dart index 454da30e5..543295fff 100644 --- a/app_dart/lib/src/foundation/utils.dart +++ b/app_dart/lib/src/foundation/utils.dart @@ -22,6 +22,7 @@ import '../request_handling/exceptions.dart'; import '../service/logging.dart'; const String kCiYamlPath = '.ci.yaml'; +const String kCiYamlFusionEnginePath = 'engine/src/flutter/$kCiYamlPath'; const String kTestOwnerPath = 'TESTOWNERS'; /// Signature for a function that calculates the backoff duration to wait in @@ -293,13 +294,13 @@ List validateOwnership( final YamlMap? ciYaml = loadYaml(ciYamlContent) as YamlMap?; final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(ciYaml); - final CiYaml ciYamlFromProto = CiYaml( + final CiYamlSet ciYamlFromProto = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: unCheckedSchedulerConfig, + yamls: {CiType.any: unCheckedSchedulerConfig}, ); - final pb.SchedulerConfig schedulerConfig = ciYamlFromProto.config; + final pb.SchedulerConfig schedulerConfig = ciYamlFromProto.configFor(CiType.any); for (pb.Target target in schedulerConfig.targets) { final String builder = target.name; diff --git a/app_dart/lib/src/model/ci_yaml/ci_yaml.dart b/app_dart/lib/src/model/ci_yaml/ci_yaml.dart index b5f671094..ce1d95ae9 100644 --- a/app_dart/lib/src/model/ci_yaml/ci_yaml.dart +++ b/app_dart/lib/src/model/ci_yaml/ci_yaml.dart @@ -11,19 +11,136 @@ import 'package:github/github.dart'; import '../proto/internal/scheduler.pb.dart' as pb; import 'target.dart'; +/// Key used for selecting which .ci.yaml to use +enum CiType { + /// "Default" associated with this slug / ci.yaml + /// + /// Pre-fusion: engine, framework, etc + /// Post-fusion: framework, etc + /// + /// NOTE: Required to be present + any, + + /// Engine's .ci.yaml file located in the monorepo + fusionEngine, +} + +/// Wrapper around one or more [CiYamlSet] files contained in a single repository. +/// +/// Single sourced repositories will have exactly one `.ci.yaml` file at the +/// root of the tree. This will have the type [CiType.any]. +/// +/// Merged repositories can have multiple `.ci.yaml` files located throughout +/// the tree. Today we only support the root type as [CiType.any] and the second +/// as [CiType.fusionEngine]. +class CiYamlSet { + CiYamlSet({ + required this.slug, + required this.branch, + required Map yamls, + CiYamlSet? totConfig, + bool validate = false, + this.isFusion = false, + }) { + for (final MapEntry(key: type, value: config) in yamls.entries) { + configs[type] = CiYaml( + slug: slug, + branch: branch, + config: config, + type: type, + validate: validate, + isFusion: isFusion, + totConfig: totConfig?.configs[type], + ); + } + } + + final bool isFusion; + + final configs = {}; + + /// Get's the [pb.SchedulerConfig] for the requested [type]. + /// + /// The type is expected to exist and will fail otherwise. + pb.SchedulerConfig configFor(CiType type) => configs[type]!.config; + + /// Get's the [CiYaml] for the requested [type]. + /// + /// The type is expected to exist and will fail otherwise. + CiYaml ciYamlFor(CiType type) => configs[type]!; + + /// The [RepositorySlug] that [config] is from. + final RepositorySlug slug; + + /// The git branch currently being scheduled against. + final String branch; + + /// Gets all [Target] that run on presubmit for this config. + List presubmitTargets({CiType type = CiType.any}) => configs[type]!.presubmitTargets; + + /// Gets all [Target] that run on postsubmit for this config. + List postsubmitTargets({CiType type = CiType.any}) => configs[type]!.postsubmitTargets; + + /// Gets the first [Target] matching [builderName] or null. + Target? getFirstPostsubmitTarget( + String builderName, { + CiType type = CiType.any, + }) => + configs[type]!.getFirstPostsubmitTarget(builderName); + + /// List of target names used to filter target from release candidate branches + /// that were already removed from main. + List? totTargetNames({CiType type = CiType.any}) => configs[type]!.totTargetNames; + + /// List of postsubmit target names used to filter target from release candidate branches + /// that were already removed from main. + List? totPostsubmitTargetNames({CiType type = CiType.any}) => configs[type]!.totPostsubmitTargetNames; + + /// Filters post submit targets to remove targets we do not want backfilled. + List backfillTargets({CiType type = CiType.any}) => configs[type]!.backfillTargets; + + /// Filters targets that were removed from main. [slug] is the gihub + /// slug for branch under test, [targets] is the list of targets from + /// the branch under test and [totTargetNames] is the list of target + /// names enabled on the default branch. + List filterOutdatedTargets( + slug, + targets, + totTargetNames, { + CiType type = CiType.any, + }) => + configs[type]!.filterOutdatedTargets(slug, targets, totTargetNames); + + /// Filters [targets] to those that should be started immediately. + /// + /// Targets with a dependency are triggered when there dependency pushes a notification that it has finished. + /// This shouldn't be confused for targets that have the property named dependency, which is used by the + /// flutter_deps recipe module on LUCI. + List getInitialTargets( + List targets, { + CiType type = CiType.any, + }) => + configs[type]!.getInitialTargets(targets); + + /// Get an unfiltered list of all [targets] that are found in the ci.yaml file. + List targets({CiType type = CiType.any}) => configs[type]!.targets; +} + /// This is a wrapper class around S[pb.SchedulerConfig]. /// /// See //CI_YAML.md for high level documentation. class CiYaml { - /// Creates [CiYaml] from a [RepositorySlug], [branch], [pb.SchedulerConfig] and an optional [CiYaml] of tip of tree CiYaml. + /// Creates [CiYamlSet] from a [RepositorySlug], [branch], [pb.SchedulerConfig] and an optional [CiYamlSet] of tip of tree CiYaml. /// /// If [totConfig] is passed, the validation will verify no new targets have been added that may temporarily break the LUCI infrastructure (such as new prod or presubmit targets). CiYaml({ required this.slug, required this.branch, required this.config, + required this.type, CiYaml? totConfig, bool validate = false, + this.isFusion = false, }) { if (validate) { _validate(config, branch, totSchedulerConfig: totConfig?.config); @@ -37,6 +154,10 @@ class CiYaml { totConfig?.postsubmitTargets.map((Target target) => target.value.name).toList() ?? []; } + final CiType type; + + final bool isFusion; + /// List of target names used to filter target from release candidate branches /// that were already removed from main. List? totTargetNames; @@ -196,7 +317,7 @@ class CiYaml { return regexp.hasMatch(branch); } - /// Validates [pb.SchedulerConfig] extracted from [CiYaml] files. + /// Validates [pb.SchedulerConfig] extracted from [CiYamlSet] files. /// /// A [pb.SchedulerConfig] file is considered good if: /// 1. It has at least one [pb.Target] in [pb.SchedulerConfig.targets] @@ -258,14 +379,27 @@ class CiYaml { } } } - // TODO(flutter#157358) - Handle merged repository yaml files. + // Verify runIf includes foundational files. if (target.runIf.isNotEmpty) { - if (!target.runIf.contains('.ci.yaml')) { - exceptions.add('ERROR: ${target.name} is missing `.ci.yaml` in runIf'); - } - if (slug == Config.engineSlug && !target.runIf.contains('DEPS')) { - exceptions.add('ERROR: ${target.name} is missing `DEPS` in runIf'); + if (isFusion && type == CiType.fusionEngine) { + // Look in different locations if fusion && engine ci.yaml + if (!target.runIf.contains('engine/src/flutter/.ci.yaml')) { + exceptions.add( + 'ERROR: ${target.name} is missing `engine/src/flutter/.ci.yaml` in runIf', + ); + } + if (!target.runIf.contains('DEPS')) { + exceptions.add('ERROR: ${target.name} is missing `DEPS` in runIf'); + } + } else { + // not fusion or not engine in fusion. + if (!target.runIf.contains('.ci.yaml')) { + exceptions.add('ERROR: ${target.name} is missing `.ci.yaml` in runIf'); + } + if (slug == Config.engineSlug && !target.runIf.contains('DEPS')) { + exceptions.add('ERROR: ${target.name} is missing `DEPS` in runIf'); + } } } } diff --git a/app_dart/lib/src/request_handlers/check_flaky_builders.dart b/app_dart/lib/src/request_handlers/check_flaky_builders.dart index 74f363a41..2d44430ab 100644 --- a/app_dart/lib/src/request_handlers/check_flaky_builders.dart +++ b/app_dart/lib/src/request_handlers/check_flaky_builders.dart @@ -60,13 +60,13 @@ class CheckFlakyBuilders extends ApiRequestHandler { ); final YamlMap? ci = loadYaml(ciContent) as YamlMap?; final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(ci); - final CiYaml ciYaml = CiYaml( + final CiYamlSet ciYaml = CiYamlSet( slug: slug, branch: Config.defaultBranch(slug), - config: unCheckedSchedulerConfig, + yamls: {CiType.any: unCheckedSchedulerConfig}, ); - final pb.SchedulerConfig schedulerConfig = ciYaml.config; + final pb.SchedulerConfig schedulerConfig = ciYaml.configFor(CiType.any); final List targets = schedulerConfig.targets; final List<_BuilderInfo> eligibleBuilders = @@ -116,7 +116,7 @@ class CheckFlakyBuilders extends ApiRequestHandler { GithubService gitHub, RepositorySlug slug, { required String content, - required CiYaml ciYaml, + required CiYamlSet ciYaml, }) async { final YamlMap ci = loadYaml(content) as YamlMap; final YamlList targets = ci[kCiYamlTargetsKey] as YamlList; @@ -164,7 +164,7 @@ class CheckFlakyBuilders extends ApiRequestHandler { } @visibleForTesting - static bool getIgnoreFlakiness(String? builderName, CiYaml ciYaml) { + static bool getIgnoreFlakiness(String? builderName, CiYamlSet ciYaml) { if (builderName == null) { return false; } diff --git a/app_dart/lib/src/request_handlers/file_flaky_issue_and_pr.dart b/app_dart/lib/src/request_handlers/file_flaky_issue_and_pr.dart index e3adefa80..721597010 100644 --- a/app_dart/lib/src/request_handlers/file_flaky_issue_and_pr.dart +++ b/app_dart/lib/src/request_handlers/file_flaky_issue_and_pr.dart @@ -41,13 +41,13 @@ class FileFlakyIssueAndPR extends ApiRequestHandler { final List builderStatisticList = await bigquery.listBuilderStatistic(kBigQueryProjectId); final YamlMap? ci = loadYaml(await gitHub.getFileContent(slug, kCiYamlPath)) as YamlMap?; final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(ci); - final CiYaml ciYaml = CiYaml( + final CiYamlSet ciYaml = CiYamlSet( slug: slug, branch: Config.defaultBranch(slug), - config: unCheckedSchedulerConfig, + yamls: {CiType.any: unCheckedSchedulerConfig}, ); - final pb.SchedulerConfig schedulerConfig = ciYaml.config; + final pb.SchedulerConfig schedulerConfig = ciYaml.configFor(CiType.any); final List targets = schedulerConfig.targets; final String testOwnerContent = await gitHub.getFileContent(slug, kTestOwnerPath); @@ -89,7 +89,7 @@ class FileFlakyIssueAndPR extends ApiRequestHandler { }); } - bool shouldSkip(BuilderStatistic statistic, CiYaml ciYaml, List targets) { + bool shouldSkip(BuilderStatistic statistic, CiYamlSet ciYaml, List targets) { // Skips if the target has been removed from .ci.yaml. if (!targets.map((e) => e.name).toList().contains(statistic.name)) { return true; @@ -191,9 +191,9 @@ class FileFlakyIssueAndPR extends ApiRequestHandler { } @visibleForTesting - static bool getIgnoreFlakiness(String builderName, CiYaml ciYaml) { + static bool getIgnoreFlakiness(String builderName, CiYamlSet ciYaml) { final Target? target = - ciYaml.postsubmitTargets.singleWhereOrNull((Target target) => target.value.name == builderName); + ciYaml.postsubmitTargets().singleWhereOrNull((Target target) => target.value.name == builderName); return target == null ? false : target.getIgnoreFlakiness(); } diff --git a/app_dart/lib/src/request_handlers/flaky_handler_utils.dart b/app_dart/lib/src/request_handlers/flaky_handler_utils.dart index 795b389e4..30f47208c 100644 --- a/app_dart/lib/src/request_handlers/flaky_handler_utils.dart +++ b/app_dart/lib/src/request_handlers/flaky_handler_utils.dart @@ -303,7 +303,7 @@ TestOwnership getTestOwnership(pb.Target target, BuilderType type, String testOw /// Gets the [BuilderType] of the builder by looking up the information in the /// ci.yaml. -BuilderType getTypeForBuilder(String? targetName, CiYaml ciYaml, {bool unfilteredTargets = false}) { +BuilderType getTypeForBuilder(String? targetName, CiYamlSet ciYaml, {bool unfilteredTargets = false}) { final List? tags = _getTags(targetName, ciYaml, unfilteredTargets: unfilteredTargets); if (tags == null || tags.isEmpty) { return BuilderType.unknown; @@ -332,13 +332,13 @@ BuilderType getTypeForBuilder(String? targetName, CiYaml ciYaml, {bool unfiltere return hasFrameworkTag && hasHostOnlyTag ? BuilderType.frameworkHostOnly : BuilderType.unknown; } -List? _getTags(String? targetName, CiYaml ciYaml, {bool unfilteredTargets = false}) { +List? _getTags(String? targetName, CiYamlSet ciYaml, {bool unfilteredTargets = false}) { final Set allUniqueTargets = {}; if (!unfilteredTargets) { - allUniqueTargets.addAll(ciYaml.presubmitTargets); - allUniqueTargets.addAll(ciYaml.postsubmitTargets); + allUniqueTargets.addAll(ciYaml.presubmitTargets()); + allUniqueTargets.addAll(ciYaml.postsubmitTargets()); } else { - allUniqueTargets.addAll(ciYaml.targets); + allUniqueTargets.addAll(ciYaml.targets(type: CiType.any)); } final Target? target = allUniqueTargets.firstWhereOrNull((element) => element.value.name == targetName); diff --git a/app_dart/lib/src/request_handlers/postsubmit_luci_subscription.dart b/app_dart/lib/src/request_handlers/postsubmit_luci_subscription.dart index 54ba49d79..646023ac9 100644 --- a/app_dart/lib/src/request_handlers/postsubmit_luci_subscription.dart +++ b/app_dart/lib/src/request_handlers/postsubmit_luci_subscription.dart @@ -128,8 +128,10 @@ class PostsubmitLuciSubscription extends SubscriptionHandler { } final Commit commit = await datastore.lookupByValue(commitKey); - final CiYaml ciYaml = await scheduler.getCiYaml(commit); - final List postsubmitTargets = ciYaml.postsubmitTargets; + + // TODO(codefu): handle fusion + final CiYamlSet ciYaml = await scheduler.getCiYaml(commit); + final List postsubmitTargets = ciYaml.postsubmitTargets(); if (!postsubmitTargets.any((element) => element.value.name == firestoreTask!.taskName)) { log.warning('Target ${firestoreTask.taskName} has been deleted from TOT. Skip updating.'); return Body.empty; diff --git a/app_dart/lib/src/request_handlers/presubmit_luci_subscription.dart b/app_dart/lib/src/request_handlers/presubmit_luci_subscription.dart index f20e0c638..7ad7f9e03 100644 --- a/app_dart/lib/src/request_handlers/presubmit_luci_subscription.dart +++ b/app_dart/lib/src/request_handlers/presubmit_luci_subscription.dart @@ -146,7 +146,7 @@ class PresubmitLuciSubscription extends SubscriptionHandler { repository: slug.fullName, sha: userData['commit_sha'] as String, ); - late CiYaml ciYaml; + late CiYamlSet ciYaml; try { if (commit.branch == Config.defaultBranch(commit.slug)) { ciYaml = await scheduler.getCiYaml(commit, validate: true); @@ -159,16 +159,18 @@ class PresubmitLuciSubscription extends SubscriptionHandler { return 0; } + // TODO(codefu): support fusion + // Do not block on the target not found. - if (!ciYaml.presubmitTargets.any((element) => element.value.name == builderName)) { + if (!ciYaml.presubmitTargets().any((element) => element.value.name == builderName)) { // do not reschedule log.warning('Did not find builder with name: $builderName in ciYaml for ${commit.sha}'); - final List availableBuilderList = ciYaml.presubmitTargets.map((Target e) => e.value.name).toList(); + final List availableBuilderList = ciYaml.presubmitTargets().map((Target e) => e.value.name).toList(); log.warning('ciYaml presubmit targets found: $availableBuilderList'); return 1; } - final Target target = ciYaml.presubmitTargets.where((element) => element.value.name == builderName).single; + final Target target = ciYaml.presubmitTargets().where((element) => element.value.name == builderName).single; final Map properties = target.getProperties(); if (!properties.containsKey('presubmit_max_attempts')) { return 1; diff --git a/app_dart/lib/src/request_handlers/reset_prod_task.dart b/app_dart/lib/src/request_handlers/reset_prod_task.dart index 8965e0ae2..3cf376df0 100644 --- a/app_dart/lib/src/request_handlers/reset_prod_task.dart +++ b/app_dart/lib/src/request_handlers/reset_prod_task.dart @@ -153,8 +153,9 @@ class ResetProdTask extends ApiRequestHandler { sha ??= commit.id!.split('/').last; taskName ??= task.name; - final CiYaml ciYaml = await scheduler.getCiYaml(commit); - final Target target = ciYaml.postsubmitTargets.singleWhere((Target target) => target.value.name == task.name); + // TODO(codefu): handle fusion + final CiYamlSet ciYaml = await scheduler.getCiYaml(commit); + final Target target = ciYaml.postsubmitTargets().singleWhere((Target target) => target.value.name == task.name); // Prepares Firestore task. firestore.Task? taskDocument; diff --git a/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart b/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart index f4ece8658..6c4aad098 100644 --- a/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart +++ b/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart @@ -66,8 +66,11 @@ class BatchBackfiller extends RequestHandler { List> backfill = >[]; for (List taskColumn in taskMap.values) { final FullTask task = taskColumn.first; - final CiYaml ciYaml = await scheduler.getCiYaml(task.commit); - final List ciYamlTargets = ciYaml.backfillTargets; + + // TODO(codefu): handle fusion + final CiYamlSet ciYaml = await scheduler.getCiYaml(task.commit); + final List ciYamlTargets = ciYaml.backfillTargets(); + // Skips scheduling if the task is not in TOT commit anymore. final bool taskInToT = ciYamlTargets.map((Target target) => target.value.name).toList().contains(task.task.name); if (!taskInToT) { diff --git a/app_dart/lib/src/request_handlers/update_existing_flaky_issues.dart b/app_dart/lib/src/request_handlers/update_existing_flaky_issues.dart index 266abaa92..c8ea9c83c 100644 --- a/app_dart/lib/src/request_handlers/update_existing_flaky_issues.dart +++ b/app_dart/lib/src/request_handlers/update_existing_flaky_issues.dart @@ -34,7 +34,7 @@ class UpdateExistingFlakyIssue extends ApiRequestHandler { static const String kThresholdKey = 'threshold'; static const int kFreshPeriodForOpenFlake = 7; // days - final CiYaml? ciYaml; + final CiYamlSet? ciYaml; @override Future get() async { @@ -42,7 +42,7 @@ class UpdateExistingFlakyIssue extends ApiRequestHandler { final GithubService gitHub = config.createGithubServiceWithToken(await config.githubOAuthToken); final BigqueryService bigquery = await config.createBigQueryService(); - CiYaml? localCiYaml = ciYaml; + CiYamlSet? localCiYaml = ciYaml; if (localCiYaml == null) { final YamlMap? ci = loadYaml( await gitHub.getFileContent( @@ -51,10 +51,10 @@ class UpdateExistingFlakyIssue extends ApiRequestHandler { ), ) as YamlMap?; final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(ci); - localCiYaml = CiYaml( + localCiYaml = CiYamlSet( slug: slug, branch: Config.defaultBranch(slug), - config: unCheckedSchedulerConfig, + yamls: {CiType.any: unCheckedSchedulerConfig}, ); } @@ -90,7 +90,7 @@ class UpdateExistingFlakyIssue extends ApiRequestHandler { required bool bringup, required BuilderStatistic statistic, required Issue existingIssue, - required CiYaml ciYaml, + required CiYamlSet ciYaml, }) async { if (DateTime.now().difference(existingIssue.createdAt!) < const Duration(days: kFreshPeriodForOpenFlake)) { return; @@ -110,7 +110,7 @@ class UpdateExistingFlakyIssue extends ApiRequestHandler { kTestOwnerPath, ); - final pb.SchedulerConfig schedulerConfig = ciYaml.config; + final pb.SchedulerConfig schedulerConfig = ciYaml.configFor(CiType.any); final List targets = schedulerConfig.targets; final String? testOwner = getTestOwnership( @@ -128,14 +128,14 @@ class UpdateExistingFlakyIssue extends ApiRequestHandler { Future _updateExistingFlakyIssue( GithubService gitHub, RepositorySlug slug, - CiYaml ciYaml, { + CiYamlSet ciYaml, { required List prodBuilderStatisticList, required List stagingBuilderStatisticList, required Map nameToExistingIssue, }) async { final Map builderFlakyMap = {}; final Map ignoreFlakyMap = {}; - for (Target target in ciYaml.postsubmitTargets) { + for (Target target in ciYaml.postsubmitTargets()) { builderFlakyMap[target.value.name] = target.value.bringup; if (target.getIgnoreFlakiness()) { ignoreFlakyMap[target.value.name] = true; diff --git a/app_dart/lib/src/service/scheduler.dart b/app_dart/lib/src/service/scheduler.dart index f52361939..7c05b09c6 100644 --- a/app_dart/lib/src/service/scheduler.dart +++ b/app_dart/lib/src/service/scheduler.dart @@ -14,6 +14,7 @@ import 'package:github/github.dart'; import 'package:github/hooks.dart'; import 'package:googleapis/bigquery/v2.dart'; import 'package:googleapis/firestore/v1.dart'; +import 'package:meta/meta.dart'; import 'package:retry/retry.dart'; import 'package:truncate/truncate.dart'; import 'package:yaml/yaml.dart'; @@ -50,6 +51,7 @@ class Scheduler { required this.config, required this.githubChecksService, required this.luciBuildService, + required this.fusionTester, this.datastoreProvider = DatastoreService.defaultProvider, this.httpClientProvider = Providers.freshHttpClient, this.buildStatusProvider = BuildStatusService.defaultProvider, @@ -61,7 +63,7 @@ class Scheduler { final DatastoreServiceProvider datastoreProvider; final GithubChecksService githubChecksService; final HttpClientProvider httpClientProvider; - + final FusionTester fusionTester; late DatastoreService datastore; late FirestoreService firestoreService; LuciBuildService luciBuildService; @@ -144,9 +146,10 @@ class Scheduler { return; } - final CiYaml ciYaml = await getCiYaml(commit); + final CiYamlSet ciYaml = await getCiYaml(commit); - final List initialTargets = ciYaml.getInitialTargets(ciYaml.postsubmitTargets); + // TODO(codefu): support fusion engine + final List initialTargets = ciYaml.getInitialTargets(ciYaml.postsubmitTargets()); final List tasks = targetsToTask(commit, initialTargets).toList(); final List> toBeScheduled = >[]; @@ -240,65 +243,84 @@ class Scheduler { } /// Process and filters ciyaml. - Future getCiYaml( + Future getCiYaml( Commit commit, { bool validate = false, }) async { + final isFusion = await fusionTester.isFusionBasedRef(commit.slug, commit.sha!); final Commit totCommit = await generateTotCommit(slug: commit.slug, branch: Config.defaultBranch(commit.slug)); - final CiYaml totYaml = await _getCiYaml(totCommit); - return _getCiYaml(commit, totCiYaml: totYaml, validate: validate); + final CiYamlSet totYaml = await _getCiYaml(totCommit, isFusionCommit: isFusion); + return _getCiYaml(commit, totCiYaml: totYaml, validate: validate, isFusionCommit: isFusion); } /// Load in memory the `.ci.yaml`. - Future _getCiYaml( + Future _getCiYaml( Commit commit, { - CiYaml? totCiYaml, + CiYamlSet? totCiYaml, bool validate = false, RetryOptions retryOptions = const RetryOptions(delayFactor: Duration(seconds: 2), maxAttempts: 4), + bool isFusionCommit = false, }) async { - String ciPath; - ciPath = '${commit.repository}/${commit.sha!}/$kCiYamlPath'; - final Uint8List ciYamlBytes = (await cache.getOrCreate( - subcacheName, - ciPath, - createFn: () => _downloadCiYaml( - commit, - ciPath, - retryOptions: retryOptions, - ), - ttl: const Duration(hours: 1), - ))!; - final pb.SchedulerConfig schedulerConfig = pb.SchedulerConfig.fromBuffer(ciYamlBytes); - log.fine('Retrieved .ci.yaml for $ciPath'); + Future getSchedulerConfig(String ciPath) async { + final Uint8List ciYamlBytes = (await cache.getOrCreate( + subcacheName, + // This is a key for a cache; not a path - so its needs to be 'unique' + '${commit.repository}/${commit.sha!}/$ciPath', + createFn: () async => (await _downloadCiYaml( + commit, + // actual path to go and fetch + ciPath, + retryOptions: retryOptions, + )) + .writeToBuffer(), + ttl: const Duration(hours: 1), + ))!; + final pb.SchedulerConfig schedulerConfig = pb.SchedulerConfig.fromBuffer(ciYamlBytes); + log.fine('Retrieved .ci.yaml for $ciPath'); + return schedulerConfig; + } + + // First, whatever was asked of us. + final schedulerConfig = await getSchedulerConfig(kCiYamlPath); + + // Second - maybe the engine CI + pb.SchedulerConfig? engineFusionConfig; + if (isFusionCommit) { + // Fetch the engine yaml and mark it up. + engineFusionConfig = await getSchedulerConfig(kCiYamlFusionEnginePath); + log.fine('fusion engine .ci.yaml file fetched'); + } + // If totCiYaml is not null, we assume upper level function has verified that current branch is not a release branch. - return CiYaml( - config: schedulerConfig, + return CiYamlSet( + yamls: { + CiType.any: schedulerConfig, + if (engineFusionConfig != null) CiType.fusionEngine: engineFusionConfig, + }, slug: commit.slug, branch: commit.branch!, totConfig: totCiYaml, validate: validate, + isFusion: isFusionCommit, ); } - /// Get `.ci.yaml` from GitHub, and store the bytes in redis for future retrieval. - /// - /// If GitHub returns [HttpStatus.notFound], an empty config will be inserted assuming - /// that commit does not support the scheduler config file. - Future _downloadCiYaml( + /// Get `.ci.yaml` from GitHub + Future _downloadCiYaml( Commit commit, String ciPath, { RetryOptions retryOptions = const RetryOptions(maxAttempts: 3), }) async { final String configContent = await githubFileContent( commit.slug, - '.ci.yaml', + ciPath, httpClientProvider: httpClientProvider, ref: commit.sha!, retryOptions: retryOptions, ); final YamlMap configYaml = loadYaml(configContent) as YamlMap; final pb.SchedulerConfig schedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(configYaml); - return schedulerConfig.writeToBuffer(); + return schedulerConfig; } /// Cancel all incomplete targets against a pull request. @@ -317,7 +339,7 @@ class Scheduler { /// /// Cancels all existing targets then schedules the targets. /// - /// Schedules a [kCiYamlCheckName] to validate [CiYaml] is valid and all builds were able to be triggered. + /// Schedules a [kCiYamlCheckName] to validate [CiYamlSet] is valid and all builds were able to be triggered. /// If [builderTriggerList] is specified, then trigger only those targets. Future triggerPresubmitTargets({ required PullRequest pullRequest, @@ -585,13 +607,14 @@ class Scheduler { /// Filters targets with runIf, matching them to the diff of [pullRequest]. /// /// In the case there is an issue getting the diff from GitHub, all targets are returned. - Future> getPresubmitTargets(PullRequest pullRequest) async { + @visibleForTesting + Future> getPresubmitTargets(PullRequest pullRequest, {CiType type = CiType.any}) async { final Commit commit = Commit( branch: pullRequest.base!.ref, repository: pullRequest.base!.repo!.fullName, sha: pullRequest.head!.sha, ); - late CiYaml ciYaml; + late CiYamlSet ciYaml; log.info('Attempting to read presubmit targets from ci.yaml for ${pullRequest.number}'); if (commit.branch == Config.defaultBranch(commit.slug)) { ciYaml = await getCiYaml(commit, validate: true); @@ -601,8 +624,10 @@ class Scheduler { log.info('ci.yaml loaded successfully.'); log.info('Collecting presubmit targets for ${pullRequest.number}'); + final inner = ciYaml.ciYamlFor(type); + // Filter out schedulers targets with schedulers different than luci or cocoon. - final List presubmitTargets = ciYaml.presubmitTargets + final List presubmitTargets = inner.presubmitTargets .where( (Target target) => target.value.scheduler == pb.SchedulerSystem.luci || target.value.scheduler == pb.SchedulerSystem.cocoon, @@ -610,11 +635,11 @@ class Scheduler { .toList(); // See https://github.com/flutter/flutter/issues/138430. - final includePostsubmitAsPresubmit = _includePostsubmitAsPresubmit(ciYaml, pullRequest); + final includePostsubmitAsPresubmit = _includePostsubmitAsPresubmit(inner, pullRequest); if (includePostsubmitAsPresubmit) { log.info('Including postsubmit targets as presubmit for ${pullRequest.number}'); - for (Target target in ciYaml.postsubmitTargets) { + for (Target target in inner.postsubmitTargets) { // We don't want to include a presubmit twice // We don't want to run the builder_cache target as a presubmit if (!target.value.presubmit && !target.value.properties.containsKey('cache_name')) { @@ -674,6 +699,7 @@ class Scheduler { /// Relevant APIs: /// https://developer.github.com/v3/checks/runs/#check-runs-and-requested-actions Future processCheckRun(cocoon_checks.CheckRunEvent checkRunEvent) async { + // TODO(codefu): Figure out if we're in fusion or not. switch (checkRunEvent.action) { case 'rerequested': log.fine('Rerun requested by GitHub user: ${checkRunEvent.sender?.login}'); @@ -726,9 +752,9 @@ class Scheduler { final firestore.Task taskDocument = taskDocuments.where((taskDocument) => taskDocument.taskName == checkName).toList().first; log.fine('Latest firestore task is $taskDocument'); - final CiYaml ciYaml = await getCiYaml(commit); + final CiYamlSet ciYaml = await getCiYaml(commit); final Target target = - ciYaml.postsubmitTargets.singleWhere((Target target) => target.value.name == task.name); + ciYaml.postsubmitTargets().singleWhere((Target target) => target.value.name == task.name); await luciBuildService.reschedulePostsubmitBuildUsingCheckRunEvent( checkRunEvent, commit: commit, @@ -799,8 +825,8 @@ class Scheduler { /// Returns the tip of tree [Commit] using specified [branch] and [RepositorySlug]. /// - /// A tip of tree [Commit] is used to help generate the tip of tree [CiYaml]. - /// The generated tip of tree [CiYaml] will be compared against Presubmit Targets in current [CiYaml], + /// A tip of tree [Commit] is used to help generate the tip of tree [CiYamlSet]. + /// The generated tip of tree [CiYamlSet] will be compared against Presubmit Targets in current [CiYamlSet], /// to ensure new targets without `bringup: true` label are not added into the build. Future generateTotCommit({required String branch, required RepositorySlug slug}) async { datastore = datastoreProvider(config.db); diff --git a/app_dart/test/model/ci_yaml/ci_yaml_test.dart b/app_dart/test/model/ci_yaml/ci_yaml_test.dart index 7ed24bad5..58e4f68b1 100644 --- a/app_dart/test/model/ci_yaml/ci_yaml_test.dart +++ b/app_dart/test/model/ci_yaml/ci_yaml_test.dart @@ -62,8 +62,8 @@ void main() { group('initialTargets', () { test('targets without deps', () { - final CiYaml ciYaml = exampleConfig; - final List initialTargets = ciYaml.getInitialTargets(ciYaml.postsubmitTargets); + final ciYaml = exampleConfig; + final List initialTargets = ciYaml.getInitialTargets(ciYaml.postsubmitTargets()); final List initialTargetNames = initialTargets.map((Target target) => target.value.name).toList(); expect( initialTargetNames, @@ -78,25 +78,27 @@ void main() { }); test('filter bringup targets on release branches', () { - final CiYaml ciYaml = CiYaml( + final CiYamlSet ciYaml = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: pb.SchedulerConfig( - enabledBranches: [ - Config.defaultBranch(Config.flutterSlug), - ], - targets: [ - pb.Target( - name: 'Linux A', - ), - pb.Target( - name: 'Mac A', // Should be ignored on release branches - bringup: true, - ), - ], - ), + yamls: { + CiType.any: pb.SchedulerConfig( + enabledBranches: [ + Config.defaultBranch(Config.flutterSlug), + ], + targets: [ + pb.Target( + name: 'Linux A', + ), + pb.Target( + name: 'Mac A', // Should be ignored on release branches + bringup: true, + ), + ], + ), + }, ); - final List initialTargets = ciYaml.getInitialTargets(ciYaml.postsubmitTargets); + final List initialTargets = ciYaml.getInitialTargets(ciYaml.postsubmitTargets()); final List initialTargetNames = initialTargets.map((Target target) => target.value.name).toList(); expect( initialTargetNames, @@ -110,6 +112,7 @@ void main() { group('validations and filters.', () { final CiYaml totCIYaml = CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: pb.SchedulerConfig( @@ -128,6 +131,7 @@ void main() { ), ); final CiYaml ciYaml = CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: 'flutter-2.4-candidate.3', config: pb.SchedulerConfig( @@ -177,8 +181,8 @@ void main() { }); test('Get backfill targets from postsubmit', () { - final CiYaml ciYaml = exampleBackfillConfig; - final List backfillTargets = ciYaml.backfillTargets; + final ciYaml = exampleBackfillConfig; + final List backfillTargets = ciYaml.backfillTargets(); final List backfillTargetNames = backfillTargets.map((Target target) => target.value.name).toList(); expect( backfillTargetNames, @@ -193,6 +197,7 @@ void main() { test('filter release_build targets from release candidate branches', () { final CiYaml releaseYaml = CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: 'flutter-2.4-candidate.3', config: pb.SchedulerConfig( @@ -222,6 +227,7 @@ void main() { test('release_build targets for main are not filtered', () { final CiYaml releaseYaml = CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: 'main', config: pb.SchedulerConfig( @@ -256,6 +262,7 @@ void main() { test('validates yaml config', () { expect( () => CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: pb.SchedulerConfig( @@ -280,6 +287,7 @@ void main() { }); group('Presubmit validation', () { final CiYaml totCIYaml = CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: pb.SchedulerConfig( @@ -299,6 +307,7 @@ void main() { ), ); final CiYaml ciYaml = CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: 'flutter-2.4-candidate.3', config: pb.SchedulerConfig( @@ -336,14 +345,14 @@ void main() { group('flakiness_threshold', () { test('is set', () { - final CiYaml ciYaml = exampleFlakyConfig; + final ciYaml = exampleFlakyConfig; final flaky1 = ciYaml.getFirstPostsubmitTarget('Flaky 1'); expect(flaky1, isNotNull); expect(flaky1?.flakinessThreshold, 0.04); }); test('is missing', () { - final CiYaml ciYaml = exampleFlakyConfig; + final ciYaml = exampleFlakyConfig; final flaky1 = ciYaml.getFirstPostsubmitTarget('Flaky Skip'); expect(flaky1, isNotNull); expect(flaky1?.flakinessThreshold, isNull); diff --git a/app_dart/test/request_handlers/check_flaky_builders_test.dart b/app_dart/test/request_handlers/check_flaky_builders_test.dart index 2d86a6d15..2c64e7e0a 100644 --- a/app_dart/test/request_handlers/check_flaky_builders_test.dart +++ b/app_dart/test/request_handlers/check_flaky_builders_test.dart @@ -29,8 +29,6 @@ const String kCurrentUserName = 'Name'; const String kCurrentUserLogin = 'login'; const String kCurrentUserEmail = 'login@email.com'; -class MockYaml extends Mock implements CiYaml {} - void main() { group('Deflake', () { late CheckFlakyBuilders handler; @@ -659,10 +657,10 @@ void main() { test('getIgnoreFlakiness handles non-existing builderame', () async { final YamlMap? ci = loadYaml(ciYamlContent) as YamlMap?; final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(ci); - final CiYaml ciYaml = CiYaml( + final CiYamlSet ciYaml = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: unCheckedSchedulerConfig, + yamls: {CiType.any: unCheckedSchedulerConfig}, ); CheckFlakyBuilders.getIgnoreFlakiness('Non_existing', ciYaml); }); diff --git a/app_dart/test/request_handlers/file_flaky_issue_and_pr_test.dart b/app_dart/test/request_handlers/file_flaky_issue_and_pr_test.dart index 9fd1d165a..0274626cc 100644 --- a/app_dart/test/request_handlers/file_flaky_issue_and_pr_test.dart +++ b/app_dart/test/request_handlers/file_flaky_issue_and_pr_test.dart @@ -676,10 +676,10 @@ void main() { test('skips when the target doesn not exist', () { final YamlMap? ci = loadYaml(ciYamlContent) as YamlMap?; final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(ci); - final CiYaml ciYaml = CiYaml( + final CiYamlSet ciYaml = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: unCheckedSchedulerConfig, + yamls: {CiType.any: unCheckedSchedulerConfig}, ); final BuilderStatistic builderStatistic = BuilderStatistic( name: 'Mac_android test', @@ -698,10 +698,10 @@ void main() { test('skips if the flakiness_threshold is not met', () { final YamlMap? ci = loadYaml(ciYamlContent) as YamlMap?; final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(ci); - final CiYaml ciYaml = CiYaml( + final CiYamlSet ciYaml = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: unCheckedSchedulerConfig, + yamls: {CiType.any: unCheckedSchedulerConfig}, ); final BuilderStatistic builderStatistic = BuilderStatistic( name: 'Mac_android higher_myflakiness', @@ -724,10 +724,10 @@ void main() { test('honors the flakiness_threshold', () { final YamlMap? ci = loadYaml(ciYamlContent) as YamlMap?; final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(ci); - final CiYaml ciYaml = CiYaml( + final CiYamlSet ciYaml = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: unCheckedSchedulerConfig, + yamls: {CiType.any: unCheckedSchedulerConfig}, ); final BuilderStatistic builderStatistic = BuilderStatistic( name: 'Mac_android higher_myflakiness', @@ -758,10 +758,10 @@ void main() { test('getIgnoreFlakiness handles non-existing builderame', () async { final YamlMap? ci = loadYaml(ciYamlContent) as YamlMap?; final pb.SchedulerConfig unCheckedSchedulerConfig = pb.SchedulerConfig()..mergeFromProto3Json(ci); - final CiYaml ciYaml = CiYaml( + final CiYamlSet ciYaml = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: unCheckedSchedulerConfig, + yamls: {CiType.any: unCheckedSchedulerConfig}, ); expect(FileFlakyIssueAndPR.getIgnoreFlakiness('Non_existing', ciYaml), false); }); diff --git a/app_dart/test/request_handlers/update_existing_flaky_issues_test_data.dart b/app_dart/test/request_handlers/update_existing_flaky_issues_test_data.dart index d4658014c..76d4d0016 100644 --- a/app_dart/test/request_handlers/update_existing_flaky_issues_test_data.dart +++ b/app_dart/test/request_handlers/update_existing_flaky_issues_test_data.dart @@ -223,84 +223,86 @@ https://flutter-dashboard.appspot.com/#/build?taskFilter=Linux%20ci_yaml_flutter Please follow https://github.com/flutter/flutter/blob/master/docs/infra/Reducing-Test-Flakiness.md#fixing-flaky-tests to fix the flakiness and enable the test back after validating the fix (internal dashboard to validate: go/flutter_test_flakiness). '''; -final CiYaml testCiYaml = CiYaml( +final CiYamlSet testCiYaml = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: pb.SchedulerConfig( - enabledBranches: [ - Config.defaultBranch(Config.flutterSlug), - ], - targets: [ - pb.Target( - name: 'Mac_android android_semantics_integration_test', - scheduler: pb.SchedulerSystem.luci, - presubmit: false, - properties: { - 'tags': jsonEncode(['devicelab']), - 'task_name': 'android_semantics_integration_test', - }, - ), - pb.Target( - name: 'Mac_android ignore_myflakiness', - scheduler: pb.SchedulerSystem.luci, - presubmit: false, - properties: { - 'ignore_flakiness': 'true', - 'tags': jsonEncode(['devicelab']), - 'task_name': 'ignore_myflakiness', - }, - ), - pb.Target( - name: 'Linux ci_yaml flutter roller', - scheduler: pb.SchedulerSystem.luci, - bringup: true, - timeout: 30, - runIf: ['.ci.yaml'], - recipe: 'infra/ci_yaml', - properties: { - 'tags': jsonEncode(['framework', 'hostonly', 'shard']), - }, - ), - pb.Target( - name: 'Mac build_tests_1_4', - scheduler: pb.SchedulerSystem.luci, - recipe: 'flutter/flutter_drone', - timeout: 60, - properties: { - 'add_recipes_cq': 'true', - 'shard': 'build_tests', - 'subshard': '1_4', - 'tags': jsonEncode(['framework', 'hostonly', 'shard']), - 'dependencies': jsonEncode([ - { - 'dependency': 'android_sdk', - 'version': 'version:29.0', - }, - { - 'dependency': 'chrome_and_driver', - 'version': 'version:84', - }, - { - 'dependency': 'xcode', - 'version': '13a233', - }, - { - 'dependency': 'open_jdk', - 'version': '11', - }, - { - 'dependency': 'gems', - 'version': 'v3.3.14', - }, - { - 'dependency': 'goldctl', - 'version': 'git_revision:3a77d0b12c697a840ca0c7705208e8622dc94603', - }, - ]), - }, - ), - ], - ), + yamls: { + CiType.any: pb.SchedulerConfig( + enabledBranches: [ + Config.defaultBranch(Config.flutterSlug), + ], + targets: [ + pb.Target( + name: 'Mac_android android_semantics_integration_test', + scheduler: pb.SchedulerSystem.luci, + presubmit: false, + properties: { + 'tags': jsonEncode(['devicelab']), + 'task_name': 'android_semantics_integration_test', + }, + ), + pb.Target( + name: 'Mac_android ignore_myflakiness', + scheduler: pb.SchedulerSystem.luci, + presubmit: false, + properties: { + 'ignore_flakiness': 'true', + 'tags': jsonEncode(['devicelab']), + 'task_name': 'ignore_myflakiness', + }, + ), + pb.Target( + name: 'Linux ci_yaml flutter roller', + scheduler: pb.SchedulerSystem.luci, + bringup: true, + timeout: 30, + runIf: ['.ci.yaml'], + recipe: 'infra/ci_yaml', + properties: { + 'tags': jsonEncode(['framework', 'hostonly', 'shard']), + }, + ), + pb.Target( + name: 'Mac build_tests_1_4', + scheduler: pb.SchedulerSystem.luci, + recipe: 'flutter/flutter_drone', + timeout: 60, + properties: { + 'add_recipes_cq': 'true', + 'shard': 'build_tests', + 'subshard': '1_4', + 'tags': jsonEncode(['framework', 'hostonly', 'shard']), + 'dependencies': jsonEncode([ + { + 'dependency': 'android_sdk', + 'version': 'version:29.0', + }, + { + 'dependency': 'chrome_and_driver', + 'version': 'version:84', + }, + { + 'dependency': 'xcode', + 'version': '13a233', + }, + { + 'dependency': 'open_jdk', + 'version': '11', + }, + { + 'dependency': 'gems', + 'version': 'v3.3.14', + }, + { + 'dependency': 'goldctl', + 'version': 'git_revision:3a77d0b12c697a840ca0c7705208e8622dc94603', + }, + ]), + }, + ), + ], + ), + }, ); const String testOwnersContent = ''' diff --git a/app_dart/test/server_test.dart b/app_dart/test/server_test.dart index 9684e8022..262333a21 100644 --- a/app_dart/test/server_test.dart +++ b/app_dart/test/server_test.dart @@ -10,6 +10,7 @@ import 'package:test/test.dart'; import 'src/datastore/fake_config.dart'; import 'src/request_handling/fake_authentication.dart'; import 'src/service/fake_build_bucket_client.dart'; +import 'src/service/fake_fusion_tester.dart'; import 'src/service/fake_gerrit_service.dart'; import 'src/service/fake_luci_build_service.dart'; import 'src/service/fake_scheduler.dart'; @@ -30,6 +31,7 @@ void main() { commitService: CommitService(config: FakeConfig()), gerritService: FakeGerritService(), scheduler: FakeScheduler(config: FakeConfig()), + fusionTester: FakeFusionTester(), ); }); } diff --git a/app_dart/test/service/scheduler/graph_test.dart b/app_dart/test/service/scheduler/graph_test.dart index 2c3b96ad3..cf457347e 100644 --- a/app_dart/test/service/scheduler/graph_test.dart +++ b/app_dart/test/service/scheduler/graph_test.dart @@ -25,6 +25,7 @@ targets: ''') as YamlMap?; final SchedulerConfig unCheckedSchedulerConfig = SchedulerConfig()..mergeFromProto3Json(singleTargetConfig); final SchedulerConfig schedulerConfig = CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, @@ -57,6 +58,7 @@ targets: final SchedulerConfig unCheckedSchedulerConfig = SchedulerConfig() ..mergeFromProto3Json(targetWithNonexistentScheduler); CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, @@ -90,6 +92,7 @@ targets: ''') as YamlMap?; final SchedulerConfig unCheckedSchedulerConfig = SchedulerConfig()..mergeFromProto3Json(dependentTargetConfig); final SchedulerConfig schedulerConfig = CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, @@ -121,6 +124,7 @@ targets: ''') as YamlMap?; final SchedulerConfig unCheckedSchedulerConfig = SchedulerConfig()..mergeFromProto3Json(twoDependentTargetConfig); final SchedulerConfig schedulerConfig = CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, @@ -152,6 +156,7 @@ targets: final SchedulerConfig unCheckedSchedulerConfig = SchedulerConfig()..mergeFromProto3Json(configWithCycle); expect( () => CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, @@ -179,6 +184,7 @@ targets: ..mergeFromProto3Json(configWithDuplicateTargets); expect( () => CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, @@ -210,6 +216,7 @@ targets: ..mergeFromProto3Json(configWithMultipleDependencies); expect( () => CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, @@ -237,6 +244,7 @@ targets: final SchedulerConfig unCheckedSchedulerConfig = SchedulerConfig()..mergeFromProto3Json(configWithMissingTarget); expect( () => CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, @@ -265,6 +273,7 @@ targets: ''') as YamlMap?; final SchedulerConfig unCheckedSchedulerConfig = SchedulerConfig()..mergeFromProto3Json(totYaml); totConfig = CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, @@ -282,6 +291,7 @@ targets: final SchedulerConfig unCheckedSchedulerConfig = SchedulerConfig()..mergeFromProto3Json(currentYaml); expect( () => CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, @@ -304,6 +314,7 @@ targets: final SchedulerConfig unCheckedSchedulerConfig = SchedulerConfig()..mergeFromProto3Json(currentYaml); expect( () => CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, @@ -325,6 +336,7 @@ targets: final SchedulerConfig unCheckedSchedulerConfig = SchedulerConfig()..mergeFromProto3Json(currentYaml); expect( () => CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, @@ -353,6 +365,7 @@ targets: final SchedulerConfig unCheckedSchedulerConfig = SchedulerConfig()..mergeFromProto3Json(currentYaml); expect( () => CiYaml( + type: CiType.any, slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), config: unCheckedSchedulerConfig, diff --git a/app_dart/test/service/scheduler_test.dart b/app_dart/test/service/scheduler_test.dart index be924b01d..7bb1b9da1 100644 --- a/app_dart/test/service/scheduler_test.dart +++ b/app_dart/test/service/scheduler_test.dart @@ -10,6 +10,7 @@ import 'package:cocoon_service/cocoon_service.dart'; import 'package:cocoon_service/src/model/appengine/commit.dart'; import 'package:cocoon_service/src/model/appengine/stage.dart'; import 'package:cocoon_service/src/model/appengine/task.dart'; +import 'package:cocoon_service/src/model/ci_yaml/ci_yaml.dart'; import 'package:cocoon_service/src/model/firestore/task.dart' as firestore; import 'package:cocoon_service/src/model/ci_yaml/target.dart'; import 'package:cocoon_service/src/model/github/checks.dart' as cocoon_checks; @@ -33,6 +34,7 @@ import '../src/datastore/fake_datastore.dart'; import '../src/service/fake_build_bucket_client.dart'; import '../src/service/fake_build_status_provider.dart'; import '../src/request_handling/fake_pubsub.dart'; +import '../src/service/fake_fusion_tester.dart'; import '../src/service/fake_gerrit_service.dart'; import '../src/service/fake_github_service.dart'; import '../src/service/fake_luci_build_service.dart'; @@ -62,6 +64,27 @@ targets: scheduler: google_internal '''; +const String fusionCiYaml = r''' +enabled_branches: + - master + - main + - codefu + - flutter-\d+\.\d+-candidate\.\d+ +targets: + - name: Linux Z + properties: + custom: abc + - name: Linux Y + enabled_branches: + - stable + scheduler: luci + - name: Linux runIf engine + runIf: + - DEPS + - engine/src/flutter/.ci.yaml + - engine/src/flutter/dev/** +'''; + void main() { late CacheService cache; late FakeConfig config; @@ -71,6 +94,7 @@ void main() { late MockFirestoreService mockFirestoreService; late MockGithubChecksUtil mockGithubChecksUtil; late Scheduler scheduler; + late FakeFusionTester fakeFusion; final PullRequest pullRequest = generatePullRequest(id: 42); @@ -122,6 +146,8 @@ void main() { // Generate check runs based on the name hash code when(mockGithubChecksUtil.createCheckRun(any, any, any, any, output: anyNamed('output'))) .thenAnswer((Invocation invocation) async => generateCheckRun(invocation.positionalArguments[2].hashCode)); + + fakeFusion = FakeFusionTester(); scheduler = Scheduler( cache: cache, config: config, @@ -136,6 +162,7 @@ void main() { branchesValue: ['master', 'main'], ), ), + fusionTester: fakeFusion, ); when(mockGithubChecksUtil.createCheckRun(any, any, any, any)).thenAnswer((_) async { @@ -147,6 +174,38 @@ void main() { }); }); + test('fusion, getPresubmitTargets supports two ci.yamls', () async { + httpClient = MockClient((http.Request request) async { + if (request.url.path.endsWith('engine/src/flutter/.ci.yaml')) { + return http.Response(fusionCiYaml, 200); + } else if (request.url.path.endsWith('.ci.yaml')) { + return http.Response(singleCiYaml, 200); + } + throw Exception('Failed to find ${request.url.path}'); + }); + + fakeFusion.isFusion = (_, __) => true; + + final List presubmitTargets = await scheduler.getPresubmitTargets(pullRequest); + + expect( + [...presubmitTargets.map((Target target) => target.value.name)], + containsAll(['Linux A']), + ); + presubmitTargets + ..clear() + ..addAll( + await scheduler.getPresubmitTargets( + pullRequest, + type: CiType.fusionEngine, + ), + ); + expect( + [...presubmitTargets.map((Target target) => target.value.name)], + containsAll(['Linux Z']), + ); + }); + group('add commits', () { final FakePubSub pubsub = FakePubSub(); List createCommitList( @@ -288,6 +347,7 @@ void main() { githubChecksService: GithubChecksService(config, githubChecksUtil: mockGithubChecksUtil), httpClientProvider: () => httpClient, luciBuildService: luciBuildService, + fusionTester: fakeFusion, ); await scheduler.addCommits(createCommitList(['1'], repo: 'engine', branch: 'main')); @@ -350,6 +410,7 @@ void main() { githubChecksService: GithubChecksService(config, githubChecksUtil: mockGithubChecksUtil), httpClientProvider: () => httpClient, luciBuildService: luciBuildService, + fusionTester: fakeFusion, ); await scheduler.addCommits(createCommitList(['1'], repo: 'engine', branch: 'main')); @@ -478,6 +539,7 @@ targets: branchesValue: ['master', 'main'], ), ), + fusionTester: fakeFusion, ); final PullRequest mergedPr = generatePullRequest( @@ -558,6 +620,7 @@ targets: config: config, githubChecksUtil: mockGithubChecksUtil, ), + fusionTester: fakeFusion, ); when(mockGithubService.github).thenReturn(mockGithubClient); when(mockGithubService.searchIssuesAndPRs(any, any, sort: anyNamed('sort'), pages: anyNamed('pages'))) @@ -634,6 +697,7 @@ targets: githubChecksUtil: mockGithubChecksUtil, buildBucketClient: mockBuildbucket, ), + fusionTester: fakeFusion, ); final cocoon_checks.CheckRunEvent checkRunEvent = cocoon_checks.CheckRunEvent.fromJson( @@ -736,6 +800,7 @@ targets: githubChecksService: GithubChecksService(config, githubChecksUtil: mockGithubChecksUtil), httpClientProvider: () => httpClient, luciBuildService: luciBuildService, + fusionTester: fakeFusion, ); final cocoon_checks.CheckRunEvent checkRunEvent = cocoon_checks.CheckRunEvent.fromJson( jsonDecode(checkRunString) as Map, @@ -1048,6 +1113,7 @@ targets: branchesValue: ['master', 'main'], ), ), + fusionTester: fakeFusion, ); final PullRequest pr = generatePullRequest( repo: Config.engineSlug.name, @@ -1084,6 +1150,7 @@ targets: githubChecksUtil: mockGithubChecksUtil, gerritService: FakeGerritService(branchesValue: ['master']), ), + fusionTester: fakeFusion, ); await scheduler.triggerPresubmitTargets(pullRequest: pullRequest); expect( @@ -1239,6 +1306,7 @@ targets: gerritService: FakeGerritService(branchesValue: ['master']), pubsub: pubsub, ), + fusionTester: fakeFusion, ); when(mockBuildbucket.batch(any)).thenAnswer( (_) async => bbv2.BatchResponse( @@ -1310,6 +1378,7 @@ targets: gerritService: FakeGerritService(branchesValue: ['master']), pubsub: pubsub, ), + fusionTester: fakeFusion, ); when(mockBuildbucket.batch(any)).thenAnswer( (_) async => bbv2.BatchResponse( diff --git a/app_dart/test/src/service/fake_scheduler.dart b/app_dart/test/src/service/fake_scheduler.dart index 37aa5744c..8764d1a06 100644 --- a/app_dart/test/src/service/fake_scheduler.dart +++ b/app_dart/test/src/service/fake_scheduler.dart @@ -3,6 +3,7 @@ // found in the LICENSE file. import 'package:cocoon_service/src/foundation/github_checks_util.dart'; +import 'package:cocoon_service/src/foundation/utils.dart'; import 'package:cocoon_service/src/model/appengine/commit.dart'; import 'package:cocoon_service/src/model/ci_yaml/ci_yaml.dart'; import 'package:cocoon_service/src/model/proto/protos.dart' as pb; @@ -16,6 +17,7 @@ import 'package:github/github.dart'; import 'package:retry/retry.dart'; import '../utilities/entity_generators.dart'; +import 'fake_fusion_tester.dart'; import 'fake_luci_build_service.dart'; /// Fake for [Scheduler] to use for tests that rely on it. @@ -26,6 +28,7 @@ class FakeScheduler extends Scheduler { BuildBucketClient? buildbucket, required super.config, GithubChecksUtil? githubChecksUtil, + FusionTester? fusionTester, }) : super( cache: CacheService(inMemory: true), githubChecksService: GithubChecksService( @@ -38,27 +41,29 @@ class FakeScheduler extends Scheduler { buildBucketClient: buildbucket, githubChecksUtil: githubChecksUtil, ), + fusionTester: fusionTester ?? FakeFusionTester(), ); - final CiYaml _defaultConfig = emptyConfig; + final CiYamlSet _defaultConfig = emptyConfig; - /// [CiYaml] value to be injected on [getCiYaml]. - CiYaml? ciYaml; + /// [CiYamlSet] value to be injected on [getCiYaml]. + CiYamlSet? ciYaml; /// If true, getCiYaml will throw a [FormatException] when validation is /// enforced, simulating failing validation. bool failCiYamlValidation = false; @override - Future getCiYaml( + Future getCiYaml( Commit commit, { - CiYaml? totCiYaml, + CiYamlSet? totCiYaml, RetryOptions? retryOptions, bool validate = false, }) async { if (validate && failCiYamlValidation) { throw const FormatException('Failed validation!'); } + return ciYaml ?? _defaultConfig; } @@ -117,224 +122,246 @@ class FakeScheduler extends Scheduler { } } -final CiYaml emptyConfig = CiYaml( +final emptyConfig = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: pb.SchedulerConfig( - enabledBranches: [ - Config.defaultBranch(Config.flutterSlug), - ], - targets: [ - pb.Target( - name: 'Linux A', - scheduler: pb.SchedulerSystem.luci, - ), - ], - ), + yamls: { + CiType.any: pb.SchedulerConfig( + enabledBranches: [ + Config.defaultBranch(Config.flutterSlug), + ], + targets: [ + pb.Target( + name: 'Linux A', + scheduler: pb.SchedulerSystem.luci, + ), + ], + ), + }, ); -CiYaml exampleConfig = CiYaml( +final exampleConfig = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: pb.SchedulerConfig( - enabledBranches: [ - Config.defaultBranch(Config.flutterSlug), - ], - targets: [ - pb.Target( - name: 'Linux A', - scheduler: pb.SchedulerSystem.luci, - ), - pb.Target( - name: 'Mac A', - scheduler: pb.SchedulerSystem.luci, - ), - pb.Target( - name: 'Windows A', - scheduler: pb.SchedulerSystem.luci, - ), - pb.Target( - name: 'Google Internal Roll', - presubmit: false, - postsubmit: true, - scheduler: pb.SchedulerSystem.google_internal, - ), - ], - ), + yamls: { + CiType.any: pb.SchedulerConfig( + enabledBranches: [ + Config.defaultBranch(Config.flutterSlug), + ], + targets: [ + pb.Target( + name: 'Linux A', + scheduler: pb.SchedulerSystem.luci, + ), + pb.Target( + name: 'Mac A', + scheduler: pb.SchedulerSystem.luci, + ), + pb.Target( + name: 'Windows A', + scheduler: pb.SchedulerSystem.luci, + ), + pb.Target( + name: 'Google Internal Roll', + presubmit: false, + postsubmit: true, + scheduler: pb.SchedulerSystem.google_internal, + ), + ], + ), + }, ); -CiYaml exampleFlakyConfig = CiYaml( +final exampleFlakyConfig = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: pb.SchedulerConfig( - enabledBranches: [ - Config.defaultBranch(Config.flutterSlug), - ], - targets: [ - pb.Target( - name: 'Flaky 1', - scheduler: pb.SchedulerSystem.luci, - properties: { - 'flakiness_threshold': '0.04', - }, - ), - pb.Target( - name: 'Flaky Skip', - scheduler: pb.SchedulerSystem.luci, - properties: { - 'ignore_flakiness': 'true', - }, - ), - ], - ), + yamls: { + CiType.any: pb.SchedulerConfig( + enabledBranches: [ + Config.defaultBranch(Config.flutterSlug), + ], + targets: [ + pb.Target( + name: 'Flaky 1', + scheduler: pb.SchedulerSystem.luci, + properties: { + 'flakiness_threshold': '0.04', + }, + ), + pb.Target( + name: 'Flaky Skip', + scheduler: pb.SchedulerSystem.luci, + properties: { + 'ignore_flakiness': 'true', + }, + ), + ], + ), + }, ); -CiYaml exampleBackfillConfig = CiYaml( +final exampleBackfillConfig = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: pb.SchedulerConfig( - enabledBranches: [ - Config.defaultBranch(Config.flutterSlug), - ], - targets: [ - pb.Target( - name: 'Linux A', - scheduler: pb.SchedulerSystem.luci, - postsubmit: true, - properties: {'backfill': 'true'}, - ), - pb.Target( - name: 'Mac A', - scheduler: pb.SchedulerSystem.luci, - postsubmit: true, - ), - pb.Target( - name: 'Windows A', - scheduler: pb.SchedulerSystem.luci, - postsubmit: true, - properties: {'backfill': 'false'}, - ), - ], - ), + yamls: { + CiType.any: pb.SchedulerConfig( + enabledBranches: [ + Config.defaultBranch(Config.flutterSlug), + ], + targets: [ + pb.Target( + name: 'Linux A', + scheduler: pb.SchedulerSystem.luci, + postsubmit: true, + properties: {'backfill': 'true'}, + ), + pb.Target( + name: 'Mac A', + scheduler: pb.SchedulerSystem.luci, + postsubmit: true, + ), + pb.Target( + name: 'Windows A', + scheduler: pb.SchedulerSystem.luci, + postsubmit: true, + properties: {'backfill': 'false'}, + ), + ], + ), + }, ); -CiYaml examplePresubmitRescheduleConfig = CiYaml( +final examplePresubmitRescheduleConfig = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: pb.SchedulerConfig( - enabledBranches: [ - Config.defaultBranch(Config.flutterSlug), - ], - targets: [ - pb.Target( - name: 'Linux A', - ), - pb.Target( - name: 'Linux B', - postsubmit: true, - properties: {'presubmit_retry': '1'}, - ), - ], - ), + yamls: { + CiType.any: pb.SchedulerConfig( + enabledBranches: [ + Config.defaultBranch(Config.flutterSlug), + ], + targets: [ + pb.Target( + name: 'Linux A', + ), + pb.Target( + name: 'Linux B', + postsubmit: true, + properties: {'presubmit_retry': '1'}, + ), + ], + ), + }, ); -final CiYaml batchPolicyConfig = CiYaml( +final batchPolicyConfig = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: pb.SchedulerConfig( - enabledBranches: [ - Config.defaultBranch(Config.flutterSlug), - ], - targets: [ - pb.Target( - name: 'Linux_android A', - ), - pb.Target( - name: 'Linux_android B', - ), - pb.Target( - name: 'Linux_android C', - ), - ], - ), + yamls: { + CiType.any: pb.SchedulerConfig( + enabledBranches: [ + Config.defaultBranch(Config.flutterSlug), + ], + targets: [ + pb.Target( + name: 'Linux_android A', + ), + pb.Target( + name: 'Linux_android B', + ), + pb.Target( + name: 'Linux_android C', + ), + ], + ), + }, ); -final CiYaml unsupportedPostsubmitCheckrunConfig = CiYaml( +final unsupportedPostsubmitCheckrunConfig = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: pb.SchedulerConfig( - enabledBranches: [ - Config.defaultBranch(Config.flutterSlug), - ], - targets: [ - pb.Target( - name: 'Linux flutter', - ), - ], - ), + yamls: { + CiType.any: pb.SchedulerConfig( + enabledBranches: [ + Config.defaultBranch(Config.flutterSlug), + ], + targets: [ + pb.Target( + name: 'Linux flutter', + ), + ], + ), + }, ); -final CiYaml nonBringupPackagesConfig = CiYaml( +final nonBringupPackagesConfig = CiYamlSet( slug: Config.packagesSlug, branch: Config.defaultBranch(Config.packagesSlug), - config: pb.SchedulerConfig( - enabledBranches: [ - Config.defaultBranch(Config.packagesSlug), - ], - targets: [ - pb.Target( - name: 'Linux nonbringup', - ), - ], - ), + yamls: { + CiType.any: pb.SchedulerConfig( + enabledBranches: [ + Config.defaultBranch(Config.packagesSlug), + ], + targets: [ + pb.Target( + name: 'Linux nonbringup', + ), + ], + ), + }, ); -final CiYaml bringupPackagesConfig = CiYaml( +final bringupPackagesConfig = CiYamlSet( slug: Config.packagesSlug, branch: Config.defaultBranch(Config.packagesSlug), - config: pb.SchedulerConfig( - enabledBranches: [ - Config.defaultBranch(Config.packagesSlug), - ], - targets: [ - pb.Target( - name: 'Linux bringup', - bringup: true, - ), - ], - ), + yamls: { + CiType.any: pb.SchedulerConfig( + enabledBranches: [ + Config.defaultBranch(Config.packagesSlug), + ], + targets: [ + pb.Target( + name: 'Linux bringup', + bringup: true, + ), + ], + ), + }, ); -final CiYaml totCiYaml = CiYaml( +final totCiYaml = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: pb.SchedulerConfig( - enabledBranches: [ - Config.defaultBranch(Config.flutterSlug), - ], - targets: [ - pb.Target( - name: 'Linux_android B', - ), - pb.Target( - name: 'Linux_android C', - ), - ], - ), + yamls: { + CiType.any: pb.SchedulerConfig( + enabledBranches: [ + Config.defaultBranch(Config.flutterSlug), + ], + targets: [ + pb.Target( + name: 'Linux_android B', + ), + pb.Target( + name: 'Linux_android C', + ), + ], + ), + }, ); -final CiYaml notInToTConfig = CiYaml( +final notInToTConfig = CiYamlSet( slug: Config.flutterSlug, branch: Config.defaultBranch(Config.flutterSlug), - config: pb.SchedulerConfig( - enabledBranches: [ - Config.defaultBranch(Config.flutterSlug), - ], - targets: [ - pb.Target( - name: 'Linux_android A', - ), - ], - ), + yamls: { + CiType.any: pb.SchedulerConfig( + enabledBranches: [ + Config.defaultBranch(Config.flutterSlug), + ], + targets: [ + pb.Target( + name: 'Linux_android A', + ), + ], + ), + }, totConfig: totCiYaml, ); diff --git a/app_dart/test/src/utilities/entity_generators.dart b/app_dart/test/src/utilities/entity_generators.dart index ffffacf0a..8f5645133 100644 --- a/app_dart/test/src/utilities/entity_generators.dart +++ b/app_dart/test/src/utilities/entity_generators.dart @@ -10,7 +10,6 @@ import 'package:cocoon_service/src/model/firestore/commit.dart' as firestore_com import 'package:cocoon_service/src/model/firestore/github_build_status.dart'; import 'package:cocoon_service/src/model/firestore/github_gold_status.dart'; import 'package:cocoon_service/src/model/firestore/task.dart' as firestore; -import 'package:cocoon_service/src/model/ci_yaml/target.dart'; import 'package:cocoon_service/src/model/gerrit/commit.dart'; import 'package:cocoon_service/src/model/proto/protos.dart' as pb; import 'package:fixnum/fixnum.dart'; @@ -220,7 +219,7 @@ Target generateTarget( pb.SchedulerSystem? schedulerSystem, String recipe = 'devicelab/devicelab', }) { - final pb.SchedulerConfig config = schedulerConfig ?? exampleConfig.config; + final pb.SchedulerConfig config = schedulerConfig ?? exampleConfig.configFor(CiType.any); if (platformProperties != null && platformDimensions != null) { config.platformProperties[platform.toLowerCase()] = pb.SchedulerConfig_PlatformProperties(properties: platformProperties, dimensions: platformDimensions);