[WIP] Port runtime-ethereum e2e-testing

.buildkite/pipeline.yml

@@ -1,9 +1,10 @@
-################
+################################################################################
 # Build pipeline
-################
+################################################################################
+
 docker_plugin: &docker_plugin_configuration
   oasislabs/docker#v2.1.0-oasis4:
-    image: "oasislabs/rust:latest"
+    image: "oasislabs/testing:0.3.0"
     always_pull: true
     workdir: /workdir
     volumes:
@@ -19,24 +20,22 @@ docker_plugin: &docker_plugin_configuration
       - "LANG=C.UTF-8"
       - "CARGO_INCREMENTAL=/cargo_cache"
       - "CARGO_INSTALL_ROOT=/root/.cargo"
+      - "EKIDEN_BRANCH=master"
+      - "RUNTIME_BRANCH=master"
+      - "DEVELOPER_GATEWAY_BRANCH=master"
     propagate-environment: true
     unconfined: true
 
 steps:
-  - label: Lint and Compile Contracts
+  - label: Run Tests
+    parallelism: 4
     command:
-      - .buildkite/scripts/setup_gitconfig.sh
-      - .buildkite/scripts/run_build.sh /tmp/artifacts
-      - cd /tmp/artifacts
-      - buildkite-agent artifact upload build.zip
-      - buildkite-agent artifact upload mantle.zip
+      - .buildkite/scripts/test_e2e.sh -t e2e-tm-committee-e2e-tests
+    timeout_in_minutes: 25
+    # use default: exit-status: "*" and retry limit of 2
+    retry:
+      automatic: true
+    agents:
+      buildkite_agent_size: large
     plugins:
       <<: *docker_plugin_configuration
-
-  - wait
-
-  - label: Build & publish test-runner docker image (master branches only)
-    branches: master
-    command:
-      - buildkite-agent artifact download build.zip .
-      - .buildkite/scripts/build_tag_push_runner_image.sh

.buildkite/rust/common.sh

@@ -0,0 +1,40 @@
+#######################################
+# Common initialization for Rust builds
+#######################################
+
+source .buildkite/scripts/common.sh
+
+####################
+# Set up environment
+####################
+export EKIDEN_UNSAFE_SKIP_AVR_VERIFY="1"
+export RUST_BACKTRACE="1"
+
+##################################
+# Set up RUSTFLAGS for the build #
+##################################
+if [ -z ${RUSTLINT+x} ]; then
+    RUSTLINT=""
+    for opt in $(cat .buildkite/rust/lint.txt | grep -v '#'); do
+        RUSTLINT=$RUSTLINT" -D "$opt
+    done
+
+    export RUSTLINT
+    if [ -z ${RUSTFLAGS+x} ]; then
+        export RUSTFLAGS=$RUSTLINT
+    else
+        export RUSTFLAGS=$RUSTFLAGS" "$RUSTLINT
+    fi
+
+    echo "Using RUSTFLAGS="$RUSTFLAGS
+fi
+
+########################################
+# Add SSH identity so that `cargo build`
+# can successfully download dependencies
+# from private github repos.
+########################################
+eval `ssh-agent -s`
+trap_add "kill ${SSH_AGENT_PID}" EXIT
+
+ssh-add || true

.buildkite/scripts/common_e2e.sh

@@ -0,0 +1,21 @@
+# Path to runtime-ethereum gateway.
+RUNTIME_GATEWAY=${RUNTIME_GATEWAY:-${WORKDIR}/target/debug/gateway}
+
+# Run a runtime-ethereum gateway node.
+#
+# Arguments:
+#   id - node identifier (default: 1)
+run_gateway() {
+    local id=${1:-1}
+
+    # Generate port numbers.
+    let http_port=id+8544
+    let ws_port=id+8554
+    let prometheus_port=id+3000
+
+    ${RUNTIME_GATEWAY} \
+        --node-address unix:${EKIDEN_CLIENT_SOCKET} \
+        --runtime-id 0000000000000000000000000000000000000000000000000000000000000000 \
+        --http-port ${http_port} \
+        --ws-port ${ws_port} 2>&1 | tee ${EKIDEN_COMMITTEE_DIR}/gateway-$id.log | sed "s/^/[gateway-${id}] /" &
+}

.buildkite/scripts/download_artifact.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+################################################################################
+# A script to download artifacts from an external buildkite pipeline.
+#
+# Uses Buildkite's "List all builds" API (with the appropriate query parameters)
+# to retrieve the raw URL to download the artifact.
+#
+# See https://buildkite.com/docs/apis/rest-api/builds#list-all-builds
+#
+# ./download_artifact.sh <PIPELINE> <BRANCH> <JOB_NAME> <ARTIFACT_NAME> <OUTPUT_DIR>
+#
+# Required Args:
+#
+# - PIPELINE:      The name of the pipeline (e.g., e2e-tests).
+# - BRANCH:        The branch from which to grab the latest version of the artifact.
+# - JOB_NAME:      The "label" given to the buildkite step.
+# - ARTIFACT_NAME: The name of the artifact to download (e.g., build.zip).
+# - OUTPUT_DIR:    The desired destination directory for the artifact to live.
+#
+################################################################################
+
+set -euo pipefail
+
+ORGANIZATION="oasislabs"
+
+BUILDKITE_ACCESS_TOKEN=${BUILDKITE_ACCESS_TOKEN:-$(cat ~/.buildkite/read_only_buildkite_api_access_token)}
+
+################################################################################
+# Required arguments.
+################################################################################
+
+# The name of the pipeline to download from.
+PIPELINE=$1
+# The branch of the pipeline to download from.
+BRANCH=$2
+# The job name within the pipeline responsible for uploading the artifact.
+# This is the "label" specified in the pipeline's definition.
+JOB_NAME=$3
+# The name of the artifact we want to download.
+ARTIFACT_NAME=$4
+# The output filename of the artifact
+OUTPUT_DIR=$5
+
+################################################################################
+# First let's fetch the correct build.
+################################################################################
+
+# Query param ensuring we only fetch builds that passed.
+STATE="passed"
+# Query request to be issued.
+BUILDS_QUERY="https://api.buildkite.com/v2/organizations/$ORGANIZATION/pipelines/$PIPELINE/builds?state=$STATE&branch=$BRANCH&access_token=$BUILDKITE_ACCESS_TOKEN"
+# All recent builds passing our query.
+BUILDS_ARRAY=$(curl $BUILDS_QUERY)
+# Take the first build given (since it's the latest).
+BUILD=$(echo $BUILDS_ARRAY | jq '.[0]')
+
+################################################################################
+# Now let's parse the jobs in the build to get all the ARTIFACTs.
+################################################################################
+
+# Extract url to query the artifacts associated with the given job.
+QUOTED_ARTIFACTS_URL=$(echo $BUILD | jq '.jobs | first(.[] | if .name == "'"${JOB_NAME}"'" then .artifacts_url else empty end)')
+# Remove quotes so we can append the access token.
+ARTIFACTS_URL=$(echo "$QUOTED_ARTIFACTS_URL" | tr -d '"')
+# Append the access token to finish constructing the URL.
+ARTIFACTS_QUERY=$ARTIFACTS_URL?access_token=$BUILDKITE_ACCESS_TOKEN
+# Fetch the artifacts
+ARTIFACTS_ARRAY=$(curl $ARTIFACTS_QUERY)
+
+################################################################################
+# Extract the correct ARTIFACT and extract its aws S3 url.
+################################################################################
+
+# Extract the url to download the correct artifact.
+QUOTED_ARTIFACT_DOWNLOAD_URL=$(echo $ARTIFACTS_ARRAY | jq 'first(.[] | if .filename == "'"${ARTIFACT_NAME}"'" then .download_url else empty end)')
+# Remove quotes so we can append the access token.
+ARTIFACT_DOWNLOAD_URL=$(echo "$QUOTED_ARTIFACT_DOWNLOAD_URL" | tr -d '"')
+# Construct the query.
+QUERY_DOWNLOAD_URL=$ARTIFACT_DOWNLOAD_URL?access_token=$BUILDKITE_ACCESS_TOKEN
+# Issue the request and extract the S3 url.
+QUOTED_RAW_S3_DOWNLOAD_URL=$(curl $QUERY_DOWNLOAD_URL | jq '.url')
+# Remove quotes
+RAW_S3_DOWNLOAD_URL=$(echo "$QUOTED_RAW_S3_DOWNLOAD_URL" | tr -d '"')
+
+################################################################################
+# Execute the final download.
+################################################################################
+
+curl ${RAW_S3_DOWNLOAD_URL} --output "$OUTPUT_DIR/$ARTIFACT_NAME"

.buildkite/scripts/download_common_artifacts.sh

@@ -0,0 +1,39 @@
+#! /bin/bash
+
+###############################################
+# Download common build artifacts and make sure
+# they are in the correct directories for tests
+# to run, etc, etc.
+#
+# This script is intended to have buildkite
+# specific things, like env vars and calling
+# the buildkite-agent binary. Keeping this
+# separate from the generic script that gets
+# called allows us to use and test the generic
+# scripts easily on a local dev box.
+###############################################
+
+# Helpful tips on writing build scripts:
+# https://buildkite.com/docs/pipelines/writing-build-scripts
+set -euxo pipefail
+
+source .buildkite/scripts/download_utils.sh
+
+# Create directory to put artifacts into.
+mkdir -p \
+    go/ekiden \
+    go/developer-gateway \
+    target/debug \
+    target/x86_64-fortanix-unknown-sgx/debug
+
+###########################################
+# Download artifacts from other pipelines
+###########################################
+download_ekiden_node go/ekiden
+download_ekiden_runtime_loader target/debug
+download_keymanager_runtime target/debug
+download_keymanager_runtime_sgx target/x86_64-fortanix-unknown-sgx/debug
+download_runtime_sgx target/x86_64-fortanix-unknown-sgx/debug
+download_runtime target/debug/
+download_gateway target/debug/
+download_developer_gateway go/developer-gateway

.buildkite/scripts/download_ekiden_test_scripts.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -euo pipefail
+
+E2E_SCRIPTS_DIR=.e2e
+
+# Check if test scripts already exist and do nothing if they do.
+if [ -e ${E2E_SCRIPTS_DIR}/ekiden_common_e2e.sh ]; then
+    echo "Found existing ekiden E2E scripts in ${E2E_SCRIPTS_DIR}. Not updating."
+    exit 0
+fi
+
+# Detect current ekiden branch.
+# TODO: Make this more robust.
+: ${EKIDEN_REPO:=$(grep 'ekiden-runtime =' Cargo.toml | grep -Eo 'git = "(.+)"' | cut -d '"' -f 2)}
+: ${EKIDEN_BRANCH:=$(grep 'ekiden-runtime =' Cargo.toml | grep -Eo 'branch = "(.+)"' | cut -d '"' -f 2)}
+if [ "$EKIDEN_BRANCH" == "" ]; then
+    echo "ERROR: Unable to determine the ekiden branch."
+    exit 1
+fi
+
+echo "Updating ekiden E2E scripts (repo \"${EKIDEN_REPO}\", branch \"${EKIDEN_BRANCH}\")..."
+
+# Download Ekiden test scripts.
+rm -rf ${E2E_SCRIPTS_DIR}
+mkdir -p ${E2E_SCRIPTS_DIR}
+pushd ${E2E_SCRIPTS_DIR}
+    git clone ${EKIDEN_REPO} -b ${EKIDEN_BRANCH} --depth 1
+    ln -s ekiden/.buildkite/scripts/common_e2e.sh ekiden_common_e2e.sh
+popd
+
+echo "Ekiden E2E scripts updated."

.buildkite/scripts/download_utils.sh

@@ -0,0 +1,56 @@
+download_oasis_binaries() {
+	local out_dir=$1
+	download_ekiden_node $out_dir
+	download_ekiden_runtime_loader $out_dir
+	download_keymanager_runtime $out_dir
+	download_keymanager_runtime_sgx $out_dir
+	download_runtime $out_dir
+	download_runtime_sgx $out_dir
+	download_gateway $out_dir
+}
+
+download_ekiden_node() {
+	local out_dir=$1
+	.buildkite/scripts/download_artifact.sh ekiden $EKIDEN_BRANCH "Build Go node" ekiden $out_dir
+	chmod +x $out_dir/ekiden
+}
+
+download_ekiden_runtime_loader() {
+	local out_dir=$1
+	.buildkite/scripts/download_artifact.sh ekiden $EKIDEN_BRANCH "Build Rust runtime loader" ekiden-runtime-loader $out_dir
+	chmod +x $out_dir/ekiden-runtime-loader
+}
+
+download_keymanager_runtime() {
+	local out_dir=$1
+	.buildkite/scripts/download_artifact.sh ekiden $EKIDEN_BRANCH "Build key manager runtime" ekiden-keymanager-runtime $out_dir
+	chmod +x $out_dir/ekiden-keymanager-runtime
+}
+
+download_keymanager_runtime_sgx() {
+	local out_dir=$1
+	.buildkite/scripts/download_artifact.sh ekiden $EKIDEN_BRANCH "Build key manager runtime" ekiden-keymanager-runtime.sgxs $out_dir
+}
+
+download_gateway() {
+	local out_dir=$1
+	.buildkite/scripts/download_artifact.sh runtime-ethereum $RUNTIME_BRANCH "Build web3 gateway" gateway $out_dir
+	chmod +x $out_dir/gateway
+}
+
+download_runtime() {
+	local out_dir=$1
+	.buildkite/scripts/download_artifact.sh runtime-ethereum $RUNTIME_BRANCH "Build runtime (ELF)" runtime-ethereum $out_dir
+	chmod +x $out_dir/runtime-ethereum
+}
+
+download_runtime_sgx() {
+	local out_dir=$1
+	.buildkite/scripts/download_artifact.sh runtime-ethereum $RUNTIME_BRANCH "Build runtime (SGXS)" runtime-ethereum.sgxs $out_dir
+}
+
+download_developer_gateway() {
+	local out_dir=$1
+	.buildkite/scripts/download_artifact.sh developer-gateway $DEVELOPER_GATEWAY_BRANCH "Build" developer-gateway $out_dir
+	chmod +x $out_dir/developer-gateway
+}