From 6dfacca368406628a747ec6d4c993a3f36b35dd9 Mon Sep 17 00:00:00 2001 From: Christopher Haar Date: Fri, 26 Jan 2024 15:39:26 +0100 Subject: [PATCH] feat(castai): add composition and examples for uptest Signed-off-by: Christopher Haar --- Makefile | 2 +- README.md | 121 +++- apis/fullaccess/composition.yaml | 831 ++++++++++++++++++++++++++ apis/fullaccess/definition.yaml | 85 +++ apis/readonly/composition.yaml | 125 ++++ apis/readonly/definition.yaml | 68 +++ examples/aws-default-provider.yaml | 11 + examples/castai-default-provider.yaml | 11 + examples/configuration.yaml | 6 + examples/eks-xr.yaml | 12 +- examples/fullaccess.yaml | 17 + examples/readonly.yaml | 12 + test/setup.sh | 6 +- 13 files changed, 1300 insertions(+), 7 deletions(-) create mode 100644 apis/fullaccess/composition.yaml create mode 100644 apis/fullaccess/definition.yaml create mode 100644 apis/readonly/composition.yaml create mode 100644 apis/readonly/definition.yaml create mode 100644 examples/aws-default-provider.yaml create mode 100644 examples/castai-default-provider.yaml create mode 100644 examples/configuration.yaml create mode 100644 examples/fullaccess.yaml create mode 100644 examples/readonly.yaml diff --git a/Makefile b/Makefile index cf0b31d..72bb800 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ build.init: $(UP) # $ export UPTEST_CLOUD_CREDENTIALS=$(echo "AWS='$(cat ~/.aws/credentials)'\nCASTAI='$(cat castai.json)'") uptest: $(UPTEST) $(KUBECTL) $(KUTTL) @$(INFO) running automated tests - @KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) CROSSPLANE_NAMESPACE=$(CROSSPLANE_NAMESPACE) $(UPTEST) e2e examples/network-xr.yaml,examples/eks-xr.yaml,examples/readonly.yaml --setup-script=test/setup.sh --default-timeout=2400 || $(FAIL) + @KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) CROSSPLANE_NAMESPACE=$(CROSSPLANE_NAMESPACE) $(UPTEST) e2e examples/network-xr.yaml,examples/eks-xr.yaml,examples/fullaccess.yaml --setup-script=test/setup.sh --default-timeout=2400 || $(FAIL) @$(OK) running automated tests e2e: build controlplane.up local.xpkg.deploy.configuration.$(PROJECT_NAME) uptest diff --git a/README.md b/README.md index e5b20a5..283e613 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,122 @@ -# AWS EKS CastAi Configuration +# AWS EKS CAST AI Configuration +This repository contains a reference CAST AI Configuration for +[Crossplane](https://crossplane.io). It's a great starting point for +building internal cloud platforms with autoscaling and offering a +self-service API to your internal development teams. +This platform offers APIs for setting up fully configured CAST AI +solution with Node Configuration, Node Templates, Autoscaler policies, +and all Castware components such as `castai-agent`, +`castai-cluster-controller`, `castai-evictor`, `castai-spot-handler`. +All these components are built using cloud service tools from the +[Official Upbound AWS Provider](https://marketplace.upbound.io/providers/upbound/provider-family-aws/) and +[Partner Upbound CAST AI Provider](https://marketplace.upbound.io/providers/crossplane-contrib/crossplane-provider-castai) + +## Quickstart + +### Prerequisites + +- CAST AI account +- CAST AI Credentials +- AWS Credentials + +### Configure the CAST AI provider + +Before we can use the reference configuration we need to configure it with CAST AI FullAccess API Access Key: +- Obtained CAST AI [API Access Key](https://docs.cast.ai/docs/authentication#obtaining-api-access-key) + +```console +# Create Secret with CAST AI API Access Key +cat < creds.conf + +# Create a K8s secret with the AWS creds: +kubectl create secret generic aws-creds -n upbound-system --from-file=credentials=./creds.conf + +# Configure the AWS Provider to use the secret: +kubectl apply -f examples/aws-default-provider.yaml +``` + + +## Using the AWS EKS CAST AI Configuration + +Create the Configuration: +```console +kubectl apply -f examples/configuration.yaml +``` + +Create Cluster in ReadOnly mode: +```console +kubectl apply -f examples/readonly.yaml +``` + +Upgrade Cluster to FullAccess mode: +```console +kubectl apply -f examples/fullaccess.yaml +``` + +🎉 Congratulations. You have just installed your CAST AI configuration! + +## Overview + +``` +crossplane beta trace xfullaccess configuration-aws-eks-castai +NAME SYNCED READY STATUS +XFullAccess/configuration-aws-eks-castai True True Available +├─ XReadOnly/configuration-aws-eks-castai True True Available +│ ├─ EksCluster/configuration-aws-eks-castai-5xn9p True True Available +│ └─ Release/configuration-aws-eks-castai-fbklv True True Available +├─ AutoScaler/configuration-aws-eks-castai-md5bq True True Available +├─ EksClusterId/configuration-aws-eks-castai-jcblz True True Available +├─ EksUserArn/configuration-aws-eks-castai-wlqmt True True Available +├─ NodeConfigurationDefault/configuration-aws-eks-castai-8mgsm True True Available +├─ NodeConfiguration/configuration-aws-eks-castai-8nf8j True True Available +├─ NodeTemplate/configuration-aws-eks-castai-xmxtf True True Available +├─ Release/configuration-aws-eks-castai-92869 True True Available +├─ Release/configuration-aws-eks-castai-d6gh2 True True Available +├─ Release/configuration-aws-eks-castai-h2zg8 True True Available +├─ InstanceProfile/cast-eks-configuration-aws-eks-castai-j4xvc-instance-profile True True Available +├─ Policy/cast-eks-configuration-aws-eks-castai-j4xvc-cluster-policy True True Available +├─ Policy/cast-eks-configuration-aws-eks-castai-j4xvc-cluster-policy-restricted True True Available +├─ RolePolicyAttachment/configuration-aws-eks-castai-6bd2w True True Available +├─ RolePolicyAttachment/configuration-aws-eks-castai-6jdrp True True Available +├─ RolePolicyAttachment/configuration-aws-eks-castai-ljf72 True True Available +├─ RolePolicyAttachment/configuration-aws-eks-castai-vsxg8 True True Available +├─ RolePolicyAttachment/configuration-aws-eks-castai-w5npj True True Available +├─ RolePolicyAttachment/configuration-aws-eks-castai-x2vnw True True Available +├─ RolePolicyAttachment/configuration-aws-eks-castai-xkzhx True True Available +├─ Role/cast-eks-configuration-aws-eks-castai-j4xvc-cluster-role True True Available +└─ Role/cast-eks-configuration-aws-eks-castai-j4xvc-instance-profile True True Available +``` + +## Questions? + +For any questions, thoughts and comments don't hesitate to [reach +out](https://www.upbound.io/contact) or drop by +[slack.crossplane.io](https://slack.crossplane.io), and say hi! diff --git a/apis/fullaccess/composition.yaml b/apis/fullaccess/composition.yaml new file mode 100644 index 0000000..2d42ec8 --- /dev/null +++ b/apis/fullaccess/composition.yaml @@ -0,0 +1,831 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: xfullaccess.castai.aws.platform.upbound.io + labels: + provider: aws +spec: + writeConnectionSecretsToNamespace: upbound-system + compositeTypeRef: + apiVersion: castai.aws.platform.upbound.io/v1alpha1 + kind: XFullAccess + mode: Pipeline + pipeline: + - step: patch-and-transform + functionRef: + name: upbound-function-patch-and-transform + input: + apiVersion: pt.fn.crossplane.io/v1beta1 + kind: Resources + patchSets: + - name: common + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.deletionPolicy + toFieldPath: spec.deletionPolicy + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.providerConfigName + toFieldPath: spec.providerConfigRef.name + - name: common-helm + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.deletionPolicy + toFieldPath: spec.deletionPolicy + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.id + toFieldPath: spec.providerConfigRef.name + resources: + - name: cluster-role + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: Role + metadata: + labels: + resource: cast-ai-cluster-role + patches: + - type: PatchSet + patchSetName: common + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.arn + policy: + fromFieldPath: Optional + toFieldPath: status.onboarding.clusterRole + - type: CombineFromComposite + combine: + strategy: string + string: + fmt: cast-eks-%s-cluster-role + variables: + - fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.name + - type: CombineFromComposite + combine: + strategy: string + string: + fmt: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": [ + "%s" + ] + }, + "Action": "sts:AssumeRole" + } + ] + } + variables: + - fromFieldPath: status.onboarding.eksUserArn + policy: + fromFieldPath: Required + toFieldPath: spec.forProvider.assumeRolePolicy + + - name: eks-user-arn + base: + apiVersion: castai.upbound.io/v1alpha1 + kind: EksUserArn + patches: + - type: PatchSet + patchSetName: common + - type: FromCompositeFieldPath + fromFieldPath: status.onboarding.clusterId + toFieldPath: spec.forProvider.clusterId + policy: + fromFieldPath: Required + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.arn + toFieldPath: status.onboarding.eksUserArn + policy: + fromFieldPath: Optional + + - name: cluster-role-ec2-read-only + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicyAttachment + spec: + forProvider: + policyArn: arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess + roleSelector: + matchControllerRef: true + matchLabels: + resource: cast-ai-cluster-role + patches: + - type: PatchSet + patchSetName: common + + - name: cluster-role-iam-read-only + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicyAttachment + spec: + forProvider: + policyArn: arn:aws:iam::aws:policy/IAMReadOnlyAccess + roleSelector: + matchControllerRef: true + matchLabels: + resource: cast-ai-cluster-role + patches: + - type: PatchSet + patchSetName: common + + - name: cluster-role-policy + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: Policy + metadata: + labels: + resource: cast-ai-cluster-role-policy + patches: + - type: PatchSet + patchSetName: common + - type: CombineFromComposite + combine: + strategy: string + string: + fmt: cast-eks-%s-cluster-policy + variables: + - fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.name + - type: CombineFromComposite + combine: + strategy: string + string: + fmt: | + { + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"PassRoleEC2", + "Action":"iam:PassRole", + "Effect":"Allow", + "Resource":"arn:aws:iam::*:role/*", + "Condition":{ + "StringEquals":{ + "iam:PassedToService":"ec2.amazonaws.com" + } + } + }, + { + "Sid":"NonResourcePermissions", + "Effect":"Allow", + "Action":[ + "iam:DeleteInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteRole", + "iam:DetachRolePolicy", + "iam:CreateServiceLinkedRole", + "iam:DeleteServiceLinkedRole", + "ec2:CreateKeyPair", + "ec2:DeleteKeyPair", + "ec2:CreateTags", + "ec2:ImportKeyPair" + ], + "Resource":"*" + }, + { + "Sid":"RunInstancesPermissions", + "Effect":"Allow", + "Action":"ec2:RunInstances", + "Resource":[ + "arn:aws:ec2:*:%[1]s:network-interface/*", + "arn:aws:ec2:*:%[1]s:security-group/*", + "arn:aws:ec2:*:%[1]s:volume/*", + "arn:aws:ec2:*:%[1]s:key-pair/*", + "arn:aws:ec2:*::image/*" + ] + } + ] + } + variables: + - fromFieldPath: status.onboarding.accountId + policy: + fromFieldPath: Required + toFieldPath: spec.forProvider.policy + + - name: cluster-rpa-policy + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicyAttachment + spec: + forProvider: + policyArnSelector: + matchControllerRef: true + matchLabels: + resource: cast-ai-cluster-role-policy + roleSelector: + matchControllerRef: true + matchLabels: + resource: cast-ai-cluster-role + patches: + - type: PatchSet + patchSetName: common + + - name: cluster-role-policy-restricted + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: Policy + metadata: + labels: + resource: cast-ai-cluster-role-policy-restricted + patches: + - type: PatchSet + patchSetName: common + - type: CombineFromComposite + combine: + strategy: string + string: + fmt: cast-eks-%s-cluster-policy-restricted + variables: + - fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.name + - type: CombineFromComposite + combine: + strategy: string + string: + fmt: | + { + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"RunInstancesTagRestriction", + "Effect":"Allow", + "Action":"ec2:RunInstances", + "Resource":"arn:aws:ec2:%[2]s:%[1]s:instance/*", + "Condition":{ + "StringEquals":{ + "aws:RequestTag/kubernetes.io/cluster/%[3]s":"owned" + } + } + }, + { + "Sid":"RunInstancesVpcRestriction", + "Effect":"Allow", + "Action":"ec2:RunInstances", + "Resource":"arn:aws:ec2:%[2]s:%[1]s:subnet/*", + "Condition":{ + "StringEquals":{ + "ec2:Vpc":"arn:aws:ec2:%[2]s:%[1]s:vpc/%[4]s" + } + } + }, + { + "Sid":"InstanceActionsTagRestriction", + "Effect":"Allow", + "Action":[ + "ec2:TerminateInstances", + "ec2:StartInstances", + "ec2:StopInstances", + "ec2:CreateTags" + ], + "Resource":"arn:aws:ec2:%[2]s:%[1]s:instance/*", + "Condition":{ + "StringEquals":{ + "ec2:ResourceTag/kubernetes.io/cluster/%[3]s":[ + "owned", + "shared" + ] + } + } + }, + { + "Sid":"VpcRestrictedActions", + "Effect":"Allow", + "Action":[ + "ec2:DeleteSecurityGroup", + "ec2:DeleteNetworkInterface" + ], + "Resource":"*", + "Condition":{ + "StringEquals":{ + "ec2:Vpc":"arn:aws:ec2:%[2]s:%[1]s:vpc/%[4]s" + } + } + }, + { + "Sid":"AutoscalingActionsTagRestriction", + "Effect":"Allow", + "Action":[ + "autoscaling:UpdateAutoScalingGroup", + "autoscaling:SuspendProcesses", + "autoscaling:ResumeProcesses", + "autoscaling:TerminateInstanceInAutoScalingGroup" + ], + "Resource":"arn:aws:autoscaling:%[2]s:%[1]s:autoScalingGroup:*:autoScalingGroupName/*", + "Condition":{ + "StringEquals":{ + "autoscaling:ResourceTag/kubernetes.io/cluster/%[3]s":[ + "owned", + "shared" + ] + } + } + }, + { + "Sid":"EKS", + "Effect":"Allow", + "Action":[ + "eks:Describe*", + "eks:List*" + ], + "Resource":[ + "arn:aws:eks:%[2]s:%[1]s:cluster/%[3]s", + "arn:aws:eks:%[2]s:%[1]s:nodegroup/%[3]s/*/*" + ] + } + ] + } + variables: + - fromFieldPath: status.onboarding.accountId + - fromFieldPath: spec.parameters.region + - fromFieldPath: spec.parameters.clusterName + - fromFieldPath: spec.parameters.vpc + policy: + fromFieldPath: Required + toFieldPath: spec.forProvider.policy + + - name: cluster-rpa-policy-restricted + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicyAttachment + spec: + forProvider: + policyArnSelector: + matchControllerRef: true + matchLabels: + resource: cast-ai-cluster-role-policy-restricted + roleSelector: + matchControllerRef: true + matchLabels: + resource: cast-ai-cluster-role + patches: + - type: PatchSet + patchSetName: common + + - name: instance-role + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: Role + metadata: + labels: + resource: cast-ai-instance-role + spec: + forProvider: + assumeRolePolicy: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": [ + "ec2.amazonaws.com" + ] + }, + "Action": "sts:AssumeRole" + } + ] + } + patches: + - type: PatchSet + patchSetName: common + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.arn + policy: + fromFieldPath: Optional + toFieldPath: status.onboarding.instanceProfileRole + + - type: CombineFromComposite + combine: + strategy: string + string: + fmt: cast-eks-%s-instance-profile + variables: + - fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.name + + - name: instance-role-eks-worker-node-policy + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicyAttachment + spec: + forProvider: + policyArn: arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy + roleSelector: + matchControllerRef: true + matchLabels: + resource: cast-ai-instance-role + patches: + - type: PatchSet + patchSetName: common + + - name: instance-role-ecr-read-only + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicyAttachment + spec: + forProvider: + policyArn: arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly + roleSelector: + matchControllerRef: true + matchLabels: + resource: cast-ai-instance-role + patches: + - type: PatchSet + patchSetName: common + + - name: instance-role-cni + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicyAttachment + spec: + forProvider: + policyArn: arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy + roleSelector: + matchControllerRef: true + matchLabels: + resource: cast-ai-instance-role + patches: + - type: PatchSet + patchSetName: common + + - name: cluster-instanceProfile + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: InstanceProfile + spec: + forProvider: + roleSelector: + matchControllerRef: true + matchLabels: + resource: cast-ai-instance-role + patches: + - type: PatchSet + patchSetName: common + - type: CombineFromComposite + combine: + strategy: string + string: + fmt: cast-eks-%s-instance-profile + variables: + - fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.name + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.arn + toFieldPath: status.onboarding.instanceProfileArn + policy: + fromFieldPath: Optional + + - name: cluster-readonly-to-fullaccess + base: + apiVersion: castai.aws.platform.upbound.io/v1alpha1 + kind: XReadOnly + spec: + parameters: + deleteNodesOnDisconnect: false + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.id + toFieldPath: metadata.name + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.region + toFieldPath: spec.parameters.region + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.accountId + toFieldPath: spec.parameters.accountId + transforms: + - string: + regexp: + group: 1 + match: (\d+) + type: Regexp + type: string + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.parameters.clusterName + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.id + toFieldPath: spec.parameters.id + - type: FromCompositeFieldPath + fromFieldPath: status.onboarding.clusterRole + toFieldPath: spec.parameters.assumeRoleArn + policy: + fromFieldPath: Required + + - name: cluster-eksClusterId + base: + apiVersion: castai.upbound.io/v1alpha1 + kind: EksClusterId + patches: + - type: PatchSet + patchSetName: common + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.labels[cast-ai-cluster] + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.region + toFieldPath: spec.forProvider.region + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.accountId + toFieldPath: spec.forProvider.accountId + transforms: + - string: + regexp: + group: 1 + match: (\d+) + type: Regexp + type: string + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.clusterName + - type: ToCompositeFieldPath + fromFieldPath: status.atProvider.id + toFieldPath: status.onboarding.clusterId + policy: + fromFieldPath: Optional + - type: ToCompositeFieldPath + fromFieldPath: spec.forProvider.accountId + toFieldPath: status.onboarding.accountId + policy: + fromFieldPath: Optional + + - name: cluster-node-configuration + base: + apiVersion: castai.upbound.io/v1alpha1 + kind: NodeConfiguration + spec: + forProvider: + clusterIdSelector: + matchControllerRef: true + diskCpuRatio: 0 + minDiskSize: 100 + patches: + - type: PatchSet + patchSetName: common + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.labels[cast-ai-cluster] + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.clusterIdSelector.matchLabels[cast-ai-cluster] + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.name + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.subnets + toFieldPath: spec.forProvider.subnets + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.image + toFieldPath: spec.forProvider.image + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.securityGroups + toFieldPath: spec.forProvider.eks[0].securityGroups + - type: FromCompositeFieldPath + fromFieldPath: status.onboarding.instanceProfileArn + toFieldPath: spec.forProvider.eks[0].instanceProfileArn + policy: + fromFieldPath: Required + + - name: cluster-node-configuration-default + base: + apiVersion: castai.upbound.io/v1alpha1 + kind: NodeConfigurationDefault + spec: + forProvider: + clusterIdSelector: + matchControllerRef: true + configurationIdSelector: + matchControllerRef: true + patches: + - patchSetName: common + type: PatchSet + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.labels[cast-ai-cluster] + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.clusterIdSelector.matchLabels[cast-ai-cluster] + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.configurationIdSelector.matchLabels[cast-ai-cluster] + + - name: cluster-autoscaler + base: + apiVersion: castai.upbound.io/v1alpha1 + kind: AutoScaler + spec: + forProvider: + autoscalerPoliciesJson: | + { + "enabled":true, + "unschedulablePods":{ + "enabled":true + }, + "nodeDownscaler":{ + "enabled":true, + "emptyNodes":{ + "enabled":true + }, + "evictor":{ + "aggressiveMode":false, + "cycleInterval":"1m", + "dryRun":false, + "enabled":true, + "nodeGracePeriodMinutes":10, + "scopedMode":false + } + } + } + clusterIdSelector: + matchControllerRef: true + patches: + - patchSetName: common + type: PatchSet + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.labels[cast-ai-cluster] + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.clusterIdSelector.matchLabels[cast-ai-cluster] + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.autoscalerPolicies + toFieldPath: spec.forProvider.autoscalerPoliciesJson + + - name: cluster-default-node-template + base: + apiVersion: castai.upbound.io/v1alpha1 + kind: NodeTemplate + spec: + forProvider: + name: default-by-castai + isEnabled: true + isDefault: true + clusterIdSelector: + matchControllerRef: true + configurationIdSelector: + matchControllerRef: true + constraints: + - architectures: + - amd64 + maxCpu: 64 + minCpu: 2 + minMemory: 4096 + maxMemory: 225280 + onDemand: true + spot: true + spotDiversityPriceIncreaseLimitPercent: 20 + enableSpotDiversity: false + spotInterruptionPredictionsEnabled: true + spotInterruptionPredictionsType: "aws-rebalance-recommendations" + useSpotFallbacks: true + patches: + - patchSetName: common + type: PatchSet + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.labels[cast-ai-cluster] + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.clusterIdSelector.matchLabels[cast-ai-cluster] + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.configurationIdSelector.matchLabels[cast-ai-cluster] + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: match + match: + patterns: + - type: regexp + regexp: .* + result: default-by-castai + + - name: cluster-helm-chart-controller + base: + apiVersion: helm.crossplane.io/v1beta1 + kind: Release + spec: + forProvider: + chart: + name: castai-cluster-controller + repository: https://castai.github.io/helm-charts/ + version: 0.55.4 + namespace: castai-agent + set: + - name: castai.apiKey + valueFrom: + secretKeyRef: + key: attribute.cluster_token + namespace: upbound-system + optional: false + - name: castai.clusterID + valueFrom: + secretKeyRef: + key: attribute.cluster_id + namespace: upbound-system + optional: false + rollbackLimit: 3 + patches: + - type: PatchSet + patchSetName: common-helm + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.set[0].valueFrom.secretKeyRef.name + transforms: + - string: + fmt: castai-%s + type: Format + type: string + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.set[1].valueFrom.secretKeyRef.name + transforms: + - string: + fmt: castai-%s + type: Format + type: string + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - match: + fallbackValue: null + patterns: + - regexp: .* + result: castai-cluster-controller + type: regexp + type: match + + - name: cluster-helm-chart-evictor + base: + apiVersion: helm.crossplane.io/v1beta1 + kind: Release + spec: + forProvider: + chart: + name: castai-evictor + repository: https://castai.github.io/helm-charts/ + version: 0.24.11 + namespace: castai-agent + rollbackLimit: 3 + patches: + - type: PatchSet + patchSetName: common-helm + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: match + match: + fallbackValue: null + patterns: + - type: regexp + regexp: .* + result: castai-evictor + + - name: cast-ai-cluster-helm-chart-spot + base: + apiVersion: helm.crossplane.io/v1beta1 + kind: Release + spec: + forProvider: + chart: + name: castai-spot-handler + repository: https://castai.github.io/helm-charts/ + version: 0.19.4 + namespace: castai-agent + set: + - name: castai.clusterID + valueFrom: + secretKeyRef: + key: attribute.cluster_id + namespace: upbound-system + optional: false + values: + castai: + provider: aws + rollbackLimit: 3 + patches: + - type: PatchSet + patchSetName: common-helm + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.set[0].valueFrom.secretKeyRef.name + transforms: + - type: string + string: + fmt: castai-%s + type: Format + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: match + match: + patterns: + - type: regexp + regexp: .* + result: castai-spot-handler diff --git a/apis/fullaccess/definition.yaml b/apis/fullaccess/definition.yaml new file mode 100644 index 0000000..47af142 --- /dev/null +++ b/apis/fullaccess/definition.yaml @@ -0,0 +1,85 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: CompositeResourceDefinition +metadata: + name: xfullaccess.castai.aws.platform.upbound.io +spec: + group: castai.aws.platform.upbound.io + names: + kind: XFullAccess + plural: xfullaccess + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + parameters: + type: object + description: Defines the castai integration + properties: + providerConfigName: + description: Crossplane ProviderConfig to use for provisioning this resources + type: string + default: default + deletionPolicy: + description: Delete the external resources when the Claim/XR is deleted. Defaults to Delete + enum: + - Delete + - Orphan + type: string + default: Delete + id: + type: string + description: ID of this Cluster that other objects will use to refer to it. + clusterName: + type: string + description: "EKS ClusterName" + autoscalerPolicies: + type: string + description: "Autoscaler Policies for this Cluster" + vpc: + type: string + description: "VPC for this EKS Cluster" + region: + type: string + description: "Region for this EKS Cluster" + accountId: + type: string + description: "AWS AccountId for this EKS Cluster" + subnets: + type: array + items: + type: string + description: subnets for this EKS Cluster Nodes + image: + type: string + description: Image to be used while provisioning the Nodes to this EKS Cluster + securityGroups: + type: array + items: + type: string + description: securityGroups for this EKS Cluster Nodes + required: + - id + - accountId + - clusterName + - image + - region + - securityGroups + - subnets + - vpc + required: + - parameters + status: + description: status represents the observed state of cast.ai onboarding. + properties: + onboarding: + description: Freeform field containing information about the onboarding configured by this Setup. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object diff --git a/apis/readonly/composition.yaml b/apis/readonly/composition.yaml new file mode 100644 index 0000000..d45ccde --- /dev/null +++ b/apis/readonly/composition.yaml @@ -0,0 +1,125 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: xreadonly.castai.aws.platform.upbound.io + labels: + provider: aws +spec: + writeConnectionSecretsToNamespace: upbound-system + compositeTypeRef: + apiVersion: castai.aws.platform.upbound.io/v1alpha1 + kind: XReadOnly + mode: Pipeline + pipeline: + - step: patch-and-transform + functionRef: + name: upbound-function-patch-and-transform + input: + apiVersion: pt.fn.crossplane.io/v1beta1 + kind: Resources + patchSets: + - name: common + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.deletionPolicy + toFieldPath: spec.deletionPolicy + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.providerConfigName + toFieldPath: spec.providerConfigRef.name + - name: common-helm + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.deletionPolicy + toFieldPath: spec.deletionPolicy + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.id + toFieldPath: spec.providerConfigRef.name + resources: + - name: ekscluster + base: + apiVersion: castai.upbound.io/v1alpha1 + kind: EksCluster + spec: + writeConnectionSecretToRef: + namespace: upbound-system + patches: + - type: PatchSet + patchSetName: common + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.labels[cast-ai-cluster] + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.writeConnectionSecretToRef.name + transforms: + - type: string + string: + fmt: castai-%s + type: Format + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.region + toFieldPath: spec.forProvider.region + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.accountId + toFieldPath: spec.forProvider.accountId + transforms: + - type: string + string: + regexp: + group: 1 + match: (\d+) + type: Regexp + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.name + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.deleteNodesOnDisconnect + toFieldPath: spec.forProvider.deleteNodesOnDisconnect + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.assumeRoleArn + toFieldPath: spec.forProvider.assumeRoleArn + + - name: agent + base: + apiVersion: helm.crossplane.io/v1beta1 + kind: Release + spec: + forProvider: + chart: + name: castai-agent + repository: https://castai.github.io/helm-charts/ + version: 0.67.0 + namespace: castai-agent + set: + - name: apiKey + valueFrom: + secretKeyRef: + key: attribute.cluster_token + namespace: upbound-system + optional: false + values: + createNamespace: false + provider: eks + rollbackLimit: 3 + patches: + - type: PatchSet + patchSetName: common-helm + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: spec.forProvider.set[0].valueFrom.secretKeyRef.name + transforms: + - type: string + string: + fmt: castai-%s + type: Format + - type: FromCompositeFieldPath + fromFieldPath: spec.parameters.clusterName + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: match + match: + fallbackValue: null + patterns: + - regexp: .* + result: castai-agent + type: regexp diff --git a/apis/readonly/definition.yaml b/apis/readonly/definition.yaml new file mode 100644 index 0000000..d60ac6e --- /dev/null +++ b/apis/readonly/definition.yaml @@ -0,0 +1,68 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: CompositeResourceDefinition +metadata: + name: xreadonly.castai.aws.platform.upbound.io +spec: + group: castai.aws.platform.upbound.io + names: + kind: XReadOnly + plural: xreadonly + versions: + - name: v1alpha1 + served: true + referenceable: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + parameters: + type: object + description: Defines the castai integration + properties: + providerConfigName: + description: Crossplane ProviderConfig to use for provisioning this resources + type: string + default: default + deletionPolicy: + description: Delete the external resources when the Claim/XR is deleted. Defaults to Delete + enum: + - Delete + - Orphan + type: string + default: Delete + id: + type: string + description: ID of this Cluster that other objects will use to refer to it. + clusterName: + type: string + description: "EKS ClusterName" + region: + type: string + description: "Region for this EKS Cluster" + accountId: + type: string + description: "AWS AccountId for this EKS Cluster" + deleteNodesOnDisconnect: + type: boolean + description: "Delete Nodes on Disconnect" + assumeRoleArn: + type: string + description: "assumeRoleArn for this EKS Cluster Access for castai" + required: + - id + - accountId + - clusterName + - region + required: + - parameters + status: + description: status represents the observed state of cast.ai onboarding. + properties: + onboarding: + description: Freeform field containing information about the onboarding configured by this Setup. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object diff --git a/examples/aws-default-provider.yaml b/examples/aws-default-provider.yaml new file mode 100644 index 0000000..df67b73 --- /dev/null +++ b/examples/aws-default-provider.yaml @@ -0,0 +1,11 @@ +apiVersion: aws.upbound.io/v1beta1 +kind: ProviderConfig +metadata: + name: default +spec: + credentials: + source: Secret + secretRef: + namespace: upbound-system + name: aws-creds + key: credentials diff --git a/examples/castai-default-provider.yaml b/examples/castai-default-provider.yaml new file mode 100644 index 0000000..2ea51f9 --- /dev/null +++ b/examples/castai-default-provider.yaml @@ -0,0 +1,11 @@ +apiVersion: castai.upbound.io/v1beta1 +kind: ProviderConfig +metadata: + name: default +spec: + credentials: + source: Secret + secretRef: + namespace: upbound-system + name: castai-creds + key: credentials diff --git a/examples/configuration.yaml b/examples/configuration.yaml new file mode 100644 index 0000000..8728fce --- /dev/null +++ b/examples/configuration.yaml @@ -0,0 +1,6 @@ +apiVersion: pkg.crossplane.io/v1 +kind: Configuration +metadata: + name: configuration-aws-eks-castai +spec: + package: xpkg.upbound.io/upbound/configuration-aws-eks-castai:v0.1.0 diff --git a/examples/eks-xr.yaml b/examples/eks-xr.yaml index 7b96d62..600b8fb 100644 --- a/examples/eks-xr.yaml +++ b/examples/eks-xr.yaml @@ -8,9 +8,17 @@ spec: region: us-west-2 version: "1.27" iam: - # replace with your custom arn like: + # Important: Please specify an iamRoleArn to access the AWS EKS Cluster deployed as part of CNOE. + # Without specifying a valid roleArn, you will not be able to log in to the EKS cluster. + # If you are using AWS SSO Roles, ensure to remove 'aws-reserved/sso.amazonaws.com/' from the ARN. + # For example, convert this: + # arn:aws:iam::123456789:role/aws-reserved/sso.amazonaws.com/AWSReservedSSO_AdministratorAccess_d703c73ed340fde7 + # To this: + # arn:aws:iam::123456789:role/AWSReservedSSO_AdministratorAccess_d703c73ed340fde7 # roleArn: arn:aws:iam::123456789:role/AWSReservedSSO_AdministratorAccess_d703c73ed340fde7 - roleArn: ${data.aws_eks_iam_default_admin} + roleArn: # set-me + # Please provide the instance role ARN for Cast AI here + autoscalerArn: # set-me nodes: count: 3 instanceType: t3.small diff --git a/examples/fullaccess.yaml b/examples/fullaccess.yaml new file mode 100644 index 0000000..b7769a1 --- /dev/null +++ b/examples/fullaccess.yaml @@ -0,0 +1,17 @@ +apiVersion: castai.aws.platform.upbound.io/v1alpha1 +kind: XFullAccess +metadata: + name: configuration-aws-eks-castai +spec: + parameters: + id: configuration-aws-eks-castai + clusterName: # set-me + vpc: vpc-0b0b0b0b0b0b0b0b0 + region: us-west-2 + accountId: "12345678910" + subnets: + - subnet-0b0b0b0b0b0b0b0b0 + - subnet-1c1c1c1c1c1c1c1c1 + securityGroups: + - sg-00000000000000000 + image: bottlerocket-aws-k8s-1.28-x86_64* diff --git a/examples/readonly.yaml b/examples/readonly.yaml new file mode 100644 index 0000000..b27cab0 --- /dev/null +++ b/examples/readonly.yaml @@ -0,0 +1,12 @@ +apiVersion: castai.aws.platform.upbound.io/v1alpha1 +kind: XReadOnly +metadata: + name: configuration-aws-eks-castai + annotations: + crossplane.io/composition-resource-name: cluster-readonly-to-fullaccess +spec: + parameters: + id: configuration-aws-eks-castai + clusterName: # set-me + region: us-west-2 + accountId: "12345678910" diff --git a/test/setup.sh b/test/setup.sh index 2a7fe2b..8b97e24 100755 --- a/test/setup.sh +++ b/test/setup.sh @@ -2,9 +2,9 @@ set -aeuo pipefail echo "Running setup.sh" -echo "Waiting until configuration package is healthy/installed..." -"${KUBECTL}" wait configuration.pkg platform-ref-gcp --for=condition=Healthy --timeout 5m -"${KUBECTL}" wait configuration.pkg platform-ref-gcp --for=condition=Installed --timeout 5m +echo "Waiting until all configuration packages are healthy/installed..." +"${KUBECTL}" wait configuration.pkg --all --for=condition=Healthy --timeout 5m +"${KUBECTL}" wait configuration.pkg --all --for=condition=Installed --timeout 5m "${KUBECTL}" wait configurationrevisions.pkg --all --for=condition=Healthy --timeout 5m echo "Waiting until all installed provider packages are healthy..."