diff --git a/agent/01_agent_requirements.sh b/agent/01_agent_requirements.sh index 1206e8d43..bd6880036 100755 --- a/agent/01_agent_requirements.sh +++ b/agent/01_agent_requirements.sh @@ -55,5 +55,7 @@ if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISCSI" ]]; then fi if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ]]; then + curl -O https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm + sudo dnf install -y ./google-chrome-stable_current_x86_64.rpm sudo dnf -y install xorriso coreos-installer syslinux skopeo fi diff --git a/agent/06_agent_create_cluster.sh b/agent/06_agent_create_cluster.sh index 05fb0a872..bcd31d17b 100755 --- a/agent/06_agent_create_cluster.sh +++ b/agent/06_agent_create_cluster.sh @@ -646,41 +646,15 @@ case "${AGENT_E2E_TEST_BOOT_MODE}" in check_assisted_install_UI - # Temporarily create a dummy kubeconfig and kubeadmin-password file for the CI - auth_dir=$SCRIPTDIR/$OCP_DIR/auth - mkdir -p $auth_dir - cfg=$auth_dir/kubeconfig - cat << EOF >> ${cfg} -clusters: -- cluster: - certificate-authority-data: LS0tLS1CRUdJTiBGSUNBVLS0tLQo= - server: https://api.test.redhat.com:6443 - name: test -contexts: -- context: - cluster: test - user: admin - name: admin -current-context: admin -preferences: {} -users: -- name: admin - user: - client-certificate-data: LS0tLS1CRUdJTiBNBVEUtLS0tLQo= - client-key-data: LS0tLS1CRUdJTiURSBVktLS0tLQo= -EOF - echo "dummy-kubeadmin-password" > $auth_dir/kubeadmin-password + mkdir -p $OCP_DIR/auth + rendezvousIP=$(getRendezvousIP) + get_vips + # Simulate user actions as done on the webUI and start cluster installation + CLUSTER_NAME=$CLUSTER_NAME BASE_DOMAIN=$BASE_DOMAIN RENDEZVOUS_IP=$rendezvousIP OCP_DIR=$OCP_DIR INGRESS_VIP=$INGRESS_VIPS API_VIP=$API_VIPS go run agent/isobuilder/ui_driven_cluster_installation.go + exit 0 ;; esac -if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ]]; then - # Current goal is to only verify if the nodes are booted fine, - # TUI sets the rendezvous IP correctly and UI is accessible. - # The next goal is to simulate adding the cluster details via UI - # and complete the cluster installation. - exit 0 -fi - if [ ! -z "${AGENT_TEST_CASES:-}" ]; then run_agent_test_cases fi diff --git a/agent/agent_post_install_validation.sh b/agent/agent_post_install_validation.sh index 817457c56..a28adde86 100755 --- a/agent/agent_post_install_validation.sh +++ b/agent/agent_post_install_validation.sh @@ -5,9 +5,10 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" source $SCRIPTDIR/common.sh -# Temp code skip the execution flow as cluster is not really installed if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ]]; then - exit 0 + oc wait clusterversion version --for=condition=Available=True --timeout=60m + oc get csv -A + oc get packagemanifests -n openshift-marketplace fi installed_control_plane_nodes=$(oc get nodes --selector=node-role.kubernetes.io/master | grep -v AGE | wc -l) @@ -18,3 +19,5 @@ if (( $NUM_MASTERS != $installed_control_plane_nodes )); then echo "Post install validation failed. Expected $NUM_MASTERS control plane nodes but found $installed_control_plane_nodes." exit 1 fi + +oc get clusterversion diff --git a/agent/isobuilder/ui_driven_cluster_installation.go b/agent/isobuilder/ui_driven_cluster_installation.go new file mode 100644 index 000000000..1067591c0 --- /dev/null +++ b/agent/isobuilder/ui_driven_cluster_installation.go @@ -0,0 +1,321 @@ +package main + +import ( + "fmt" + "log" + "os" + "path" + "path/filepath" + + "errors" + "strings" + "time" + + resty "github.com/go-resty/resty/v2" + "github.com/go-rod/rod" + + "github.com/go-rod/rod/lib/launcher" + "github.com/go-rod/rod/lib/proto" + "github.com/go-rod/rod/lib/utils" + + "github.com/sirupsen/logrus" +) + +var ( + clusterName = os.Getenv("CLUSTER_NAME") + baseDomain = os.Getenv("BASE_DOMAIN") + rendezvousIP = os.Getenv("RENDEZVOUS_IP") + ocpDir = os.Getenv("OCP_DIR") + baseURL = fmt.Sprintf("http://%s:3001", rendezvousIP) + clustersURL = fmt.Sprintf("%s%s", baseURL, path.Join("/api/assisted-install/v2/clusters")) +) + +func main() { + logrus.Info("Launching headless browser...") + url := launcher.New().NoSandbox(true).Headless(true).MustLaunch() + browser := rod.New().ControlURL(url).MustConnect() + + defer browser.MustClose() + + page := browser.MustPage(baseURL) + page.MustWaitLoad() + + cwd, err := os.Getwd() + if err != nil { + log.Fatal(err) + } + screenshotPath := filepath.Join(cwd, ocpDir) + + logrus.Info("Enter cluster details") + err = clusterDetails(page, filepath.Join(screenshotPath, "01-cluster-details.png")) + if err != nil { + log.Fatalf("failed to enter cluster details: %v", err) + } + + next(page) + + logrus.Info("Select virtualization bundle") + err = virtualizationBundle(page, filepath.Join(screenshotPath, "02-operators.png")) + if err != nil { + log.Fatalf("failed to select virtualization bundle: %v", err) + } + + next(page) + + logrus.Info("Await host discovery") + err = hostDiscovery(page, filepath.Join(screenshotPath, "03-hostDiscovery.png")) + if err != nil { + log.Fatalf("failed awaiting host discovery: %v", err) + } + + next(page) + + logrus.Info("Verify storage") + err = verifyStorage(page, filepath.Join(screenshotPath, "04-storage.png")) + if err != nil { + log.Fatalf("failed awaiting host discovery: %v", err) + } + + next(page) + + logrus.Info("Enter networking details") + err = networkingDetails(page, filepath.Join(screenshotPath, "05-networking.png")) + if err != nil { + log.Fatalf("failed entering networking details: %v", err) + } + + next(page) + + logrus.Info("Download credentials") + client := resty.New() + err = downloadCredentials(page, client, filepath.Join(screenshotPath, "06-credentials.png")) + if err != nil { + log.Fatalf("failed downloading credentials: %v", err) + } + + next(page) + + logrus.Info("Review and start cluster installation") + err = review(page, filepath.Join(screenshotPath, "07-review.png")) + if err != nil { + log.Fatalf("failed review page: %v", err) + } + + logrus.Info("Cluster installation started successfully.") + page.MustElementR("h2", "Installation progress") + + err = waitForClusterConsoleLink(page, filepath.Join(screenshotPath, "08-installation-progress")) + if err != nil { + log.Fatalf("%v", err) + } +} + +func clusterDetails(page *rod.Page, path string) error { + page.MustElement("#form-input-name-field").MustInput(clusterName) + page.MustElement("#form-input-baseDnsDomain-field").MustInput(baseDomain) + + pullSecretPath := os.Getenv("PULL_SECRET_FILE") + secretBytes, err := os.ReadFile(pullSecretPath) + if err != nil { + return fmt.Errorf("failed to read pull secret file: %v", err) + } + pullSecret := strings.TrimSpace(string(secretBytes)) + page.MustElement("#form-input-pullSecret-field").MustInput(pullSecret) + + err = saveFullPageScreenshot(page, path) + if err != nil { + return err + } + return nil +} + +func virtualizationBundle(page *rod.Page, path string) error { + page.MustElement("#bundle-virtualization").MustClick().MustWaitEnabled() + err := saveFullPageScreenshot(page, path) + if err != nil { + return err + } + return nil +} + +func hostDiscovery(page *rod.Page, path string) error { + page.MustElement("button[name='next']").MustWaitEnabled() + err := saveFullPageScreenshot(page, path) + if err != nil { + return err + } + return nil +} + +func verifyStorage(page *rod.Page, path string) error { + err := saveFullPageScreenshot(page, path) + if err != nil { + return err + } + return nil +} + +func networkingDetails(page *rod.Page, path string) error { + apiVip := os.Getenv("API_VIP") + ingressVip := os.Getenv("INGRESS_VIP") + page.MustElement("#form-input-apiVips-0-ip-field").MustInput(apiVip) + page.MustElement("#form-input-ingressVips-0-ip-field").MustInput(ingressVip) + page.MustElement(`button[name="next"]`).MustWaitEnabled() + + err := saveFullPageScreenshot(page, path) + if err != nil { + return err + } + return nil +} + +func downloadCredentials(page *rod.Page, client *resty.Client, path string) error { + clusterID, err := getClusterID(client, clustersURL) + if err != nil { + return err + } + + logrus.Info("Download credentials via api request") + + fileURL := fmt.Sprintf("%s/%s/downloads/credentials?file_name=", clustersURL, clusterID) + saveCredentials(client, fileURL, "kubeadmin-password") + time.Sleep(15 * time.Second) + + saveCredentials(client, fileURL, "kubeconfig") + time.Sleep(15 * time.Second) + + page.MustElement("#credentials-download-agreement").MustClick() + time.Sleep(5 * time.Second) + + page.MustElementR("button", "Download credentials").MustWaitEnabled() + + err = saveFullPageScreenshot(page, path) + if err != nil { + return err + } + return nil +} + +func review(page *rod.Page, path string) error { + installBtn := page.MustElementR("button", "Install cluster") + + err := saveFullPageScreenshot(page, path) + if err != nil { + return err + } + + installBtn.MustClick() + logrus.Info("Install button clicked") + + return nil +} + +func waitForClusterConsoleLink(page *rod.Page, path string) error { + var i = 0 + for { + failMsg, _ := page.Timeout(5*time.Second).ElementR("#cluster-progress-status-value", "Failed on") + if failMsg != nil { + if visible, _ := failMsg.Visible(); visible { + if err := saveFullPageScreenshot(page, fmt.Sprintf("%s-%d.png", path, i)); err != nil { + return err + } + return errors.New("cluster installation failed") + } + } + finalizingPage, _ := page.Timeout(5*time.Second).ElementR("h4", "Finalizing") + if finalizingPage != nil { + if visible, _ := finalizingPage.Visible(); visible { + logrus.Info("Console URL is available.") + i++ + if err := saveFullPageScreenshot(page, fmt.Sprintf("%s-%d.png", path, i)); err != nil { + return err + } + break + } + } + logrus.Info("Cluster installation in progress. Waiting for console URL to be available.") + err := saveFullPageScreenshot(page, fmt.Sprintf("%s-%d.png", path, i)) + if err != nil { + return err + } + i++ + time.Sleep(5 * time.Minute) + } + + return nil +} + +func next(page *rod.Page) { + page.MustElement("button[name='next']").MustWaitEnabled().MustClick() +} + +// saveFullPageScreenshot captures a full-page screenshot and saves it to the given path. +func saveFullPageScreenshot(page *rod.Page, path string) error { + result, err := page.Evaluate(rod.Eval(`() => { + return { + width: document.body.scrollWidth, + height: document.body.scrollHeight + } + }`)) + if err != nil { + return fmt.Errorf("failed to evaluate page size: %w", err) + } + + width := int(result.Value.Get("width").Int()) + height := int(result.Value.Get("height").Int()) + + err = page.SetViewport(&proto.EmulationSetDeviceMetricsOverride{ + Width: int(width), + Height: int(height), + DeviceScaleFactor: 1, + Mobile: false, + }) + if err != nil { + return fmt.Errorf("failed to set viewport: %w", err) + } + + screenshot, err := page.Screenshot(false, nil) + if err != nil { + return fmt.Errorf("failed to take screenshot: %w", err) + } + + if err := utils.OutputFile(path, screenshot); err != nil { + return fmt.Errorf("failed to save screenshot: %w", err) + } + logrus.Info("Screenshot saved to", path, ", with type of image/png") + return nil +} + +// getClusterID fetches the first cluster ID from the given URL using the provided client. +func getClusterID(client *resty.Client, url string) (string, error) { + var clusters []struct { + ID string `json:"id"` + } + + _, err := client.R().SetResult(&clusters).Get(url) + if err != nil { + return "", err + } + + return clusters[0].ID, nil +} + +// saveCredentials downloads a file from the given URL and saves it under the auth directory. +func saveCredentials(client *resty.Client, url, filename string) { + logrus.Info("Downloading ", filename) + + fileURL := fmt.Sprintf("%s%s", url, filename) + resp, err := client.R().Get(fileURL) + if err != nil { + logrus.Info("Request failed:", err) + return + } + + downloadedFile := fmt.Sprintf("%s/auth/%s", ocpDir, filename) + err = os.WriteFile(downloadedFile, resp.Body(), 0644) + if err != nil { + logrus.Errorf("Failed to save file %s: %v", downloadedFile, err) + return + } + logrus.Info("File ", downloadedFile, " downloaded successfully") +} diff --git a/go.mod b/go.mod index 6b5458ad3..2e0555125 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,19 @@ go 1.22.3 require ( github.com/apparentlymart/go-cidr v1.1.0 + github.com/go-resty/resty/v2 v2.16.5 + github.com/go-rod/rod v0.116.2 github.com/openshift/installer v1.4.17 + github.com/sirupsen/logrus v1.9.3 ) -require github.com/pkg/errors v0.9.1 // indirect +require ( + github.com/pkg/errors v0.9.1 // indirect + github.com/ysmood/fetchup v0.2.3 // indirect + github.com/ysmood/goob v0.4.0 // indirect + github.com/ysmood/got v0.40.0 // indirect + github.com/ysmood/gson v0.7.3 // indirect + github.com/ysmood/leakless v0.9.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.28.0 // indirect +) diff --git a/go.sum b/go.sum index 08adacb78..786a9af94 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,48 @@ github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= +github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA= +github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA= +github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg= github.com/openshift/installer v1.4.17 h1:63iijBBgYqQX/p2+Q74gPqnfBN5VNSWX5LxQKuLlj6g= github.com/openshift/installer v1.4.17/go.mod h1:CtlMEGKJDVMZl4qVBC/xMUXM24YnleT6bakI+KXFAhk= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ= +github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns= +github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= +github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18= +github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg= +github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk= +github.com/ysmood/got v0.40.0 h1:ZQk1B55zIvS7zflRrkGfPDrPG3d7+JOza1ZkNxcc74Q= +github.com/ysmood/got v0.40.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg= +github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY= +github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= +github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE= +github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= +github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU= +github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=