openstack-crowbar: Add pci passthough option

Add Jenkins parameter 'want_pci_passthrough' to allow running tests to validate
pci passthough.

To test pci passthrough locally with mkcloud, simply just export
'want_pci_passthrouh=1' and append 'rebootcloud testpcipassthru' to 'plain'
step, e.g. your_mkcloud.sh plain rebootcloud testpcipassthru

The rebootcloud step will ensure intel_iommu=on kernel param enabled.

The testpcipassthru setp will prepare the environment and spin up a nested vm
(L1) on compute (L0) with a virtio disk passing from compute vm (L0). Noted
that this is not passing any device from the host itself.

1. add environment prep and verify function in qa_crowbarsetup.sh
2. create ci job template

jenkins/ci.suse.de/templates/cloud-mkcloud-job-pcipassthru-template.yml

@@ -0,0 +1,29 @@
+- job-template:
+    name: 'cloud-mkcloud{version}-job-pcipassthru-{arch}'
+    node: cloud-trigger
+    disabled: '{obj:disabled}'
+
+    triggers:
+      - timed: 'H 20 * * *'
+
+    logrotate:
+      numToKeep: -1
+      daysToKeep: 7
+
+    builders:
+      - trigger-builds:
+        - project: openstack-mkcloud
+          condition: SUCCESS
+          block: true
+          current-parameters: true
+          predefined-parameters: |
+            TESTHEAD=1
+            cloudsource=develcloud{version}
+            libvirt_type=kvm
+            nodenumber=2
+            mkcloudtarget=all
+            want_pci_passthrough=1
+            vcpus=4
+            label={label}
+            job_name=cloud-mkcloud{version}-job-pcipassthru-{arch}
+

scripts/jenkins/cloud/ansible/deploy-crowbar.yml

@@ -47,6 +47,13 @@
             - reboot_after_deploy
             - not update_after_deploy
 
+        - include_role:
+            name: crowbar_setup
+          vars:
+            qa_crowbarsetup_cmd: onadmin_verify_pci_passthru
+          when:
+            - want_pci_passthrough|int == 1
+
       rescue:
         - include_role:
             name: rocketchat_notify

scripts/mkcloud

@@ -1029,6 +1029,14 @@ function rebootcrowbar
     return $?
 }
 
+# Need to run rebootcloud step to enable iommu (e.g., plain rebootcloud
+# testpcipassthru)
+function testpcipassthru
+{
+    onadmin verify_pci_passthru
+    return $?
+}
+
 function rebootcloud
 {
     # reboot compute nodes
@@ -1455,6 +1463,8 @@ Optional
         If set, does not use the --insecure flag in openstack CLI commands.
     want_monasca_tsdb (Cloud9+ only)
         Allows setting time-series DB used for Monasca [influxdb|cassandra].
+    want_pci_passthrough (default=0)
+        Deploy node with an extra disk to test pci passthrough. Cloud9+ only.
     want_ipv6 (Currently in development)
         Prepare crowbar to use a full IPv6 single stack control plane. Enabling will also set
         the firmware_type to UEFI as it's required to netboot nodes over IPv6.
@@ -1619,7 +1629,7 @@ allcmds="$step_aliases _test_setuphost \
     lonelynode_nfs_server setupironicnodes\
     restoreadminfromsnapshot createcloudsnapshot restorecloudfromsnapshot \
     cct steps batch setup_aliases onadmin onhost devsetup plain_with_upgrade_test \
-    testpreupgrade testpostupgrade deployexternalceph"
+    testpreupgrade testpostupgrade deployexternalceph testpcipassthru"
 wantedcmds=$@
 
 function expand_steps

scripts/qa_crowbarsetup.sh

@@ -57,10 +57,12 @@ fi
 
 # global variables that are set within this script
 novacontroller=
+novacompute_kvm=
 horizonserver=
 horizonservice=
 manila_service_vm_uuid=
 manila_tenant_vm_ip=
+pci_passthru_vm_name="pci-passthru-instance"
 clusternodesdrbd=
 clusternodesdata=
 clusternodesnetwork=
@@ -3315,6 +3317,37 @@ function onadmin_proposal
     done
 
     set_dashboard_alias
+
+    prepare_pci_passthru_env
+}
+
+function prepare_pci_passthru_env()
+{
+    if iscloudver 9plus && [[ $want_pci_passthrough = 1 ]] ; then
+        # Place nova configuration to verify pci_passthru
+        get_novacontroller
+        safely oncontroller create_pci_passthru_conf
+        append_pci_passthru_filter
+        get_novacompute_kvm
+        safely oncompute create_pci_passthru_conf
+        safely oncompute load_vfio_module_onboot
+        safely oncompute bind_vfio_module_onboot
+        # TODO: need support for amd param
+        safely oncompute add_intel_iommu_param
+    fi
+}
+
+function onadmin_verify_pci_passthru()
+{
+    # spin up vm and pass device from compute host to vm to verify pci
+    # passthru functionality.
+    get_novacontroller
+    safely oncontroller setup_pci_passthru_vm
+    # Get instance name (e.g.,instance-00000001)
+    pci_passthru_instance_name=$(ssh "$novacontroller" "source /root/.openrc;
+        openstack server show -c OS-EXT-SRV-ATTR:instance_name -f value $pci_passthru_vm_name")
+    get_novacompute_kvm
+    safely oncompute parse_vm_xml "$pci_passthru_instance_name"
 }
 
 function set_node_alias
@@ -3417,6 +3450,15 @@ function get_novacontroller
     novacontroller=`resolve_element_to_hostname "$element"`
 }
 
+function get_novacompute_kvm
+{
+    local element=`crowbar nova proposal show default | \
+        rubyjsonparse "
+                    puts j['deployment']['nova']\
+                        ['elements']['nova-compute-kvm']"`
+    novacompute_kvm=`resolve_element_to_hostname "$element"`
+}
+
 function get_horizon
 {
     local element=`crowbar horizon proposal show default | \
@@ -3860,6 +3902,81 @@ function oncontroller_heat_image_setup()
     fi
 }
 
+function oncompute_parse_vm_xml()
+{
+    local instance_name=$1
+    echo $instance_name
+    virsh list --all | grep $instance_name
+    [ $? != 0 ] && complain 53 "failed to find pci-passthru-instance"
+
+    # verify node xml to see if it has pci passthru section which has the
+    # specific address passing from compute host
+    virsh dumpxml $instance_name | grep -E "<hostdev" -A 7 | grep "0x1d"
+    [ $? != 0 ] && complain 63 "failed to find pci-passthru block in node xml"
+
+    return 0
+}
+
+function oncontroller_setup_pci_passthru_vm()
+{
+    local cirros_image_url=$imageserver_url/$arch/openstack/cirros-0.4.0-x86_64-disk.img
+    local cirros_image_name=cirros-0.4.0-x86_64-disk.img
+    local cirros_image_params="--disk-format qcow2 --property hypervisor_type=kvm"
+
+    local sec_group="pci-passthru"
+    local local_image_path=""
+    if [ -n "${localreposdir_target}" ] ; then
+        local_image_path=$localreposdir_target/images/$arch/openstack/$cirros_image_name
+    fi
+
+    if [ -n "${local_image_path}" -a -f "${local_image_path}" ] ; then
+        cirros_image_name=$local_image_path
+    else
+        local ret=$(wget -N --progress=dot:mega "$cirros_image_url" 2>&1 >/dev/null)
+        if [[ $ret =~ "200 OK" ]]; then
+            echo $ret
+        elif [[ $ret =~ "Not Found" ]]; then
+            complain 73 "cirros image not found: $ret"
+        else
+            complain 74 "failed to retrieve cirros image: $ret"
+        fi
+    fi
+
+    . .openrc
+
+    # using list subcommand because show requires an ID
+    if ! openstack image list --format value -c Name | grep -q "^cirros-0.4.0-x86_64-disk$"; then
+        openstack image create --file $cirros_image_name \
+            $cirros_image_params --container-format bare --public \
+            cirros-0.4.0-x86_64-disk
+    fi
+
+    # create flavor with pci-passthru property so that the vm using this flavor
+    # will aquire the pci device passing from the host (compute)
+    openstack flavor create --id 200 --ram 512 --ephemeral 0 \
+        --vcpus 1 --property "pci_passthrough:alias"="a1:1" \
+        pci-passthru-flavor
+
+    if iscloudver 9plus && ! openstack security group show $sec_group 2>/dev/null ; then
+        openstack security group create --description "$sec_group description" $sec_group
+        openstack security group rule create --protocol icmp $sec_group
+        openstack security group rule create --protocol tcp --dst-port 22 $sec_group
+    fi
+
+    fixed_net_id=`neutron net-show fixed -f value -c id`
+    timeout 10m openstack server create --flavor 200 \
+        --image cirros-0.4.0-x86_64-disk \
+        --security-group $sec_group \
+        --nic net-id=$fixed_net_id $pci_passthru_vm_name
+
+    [ $? != 0 ] && complain 43 "nova boot pci-passthru-instance failed"
+
+    # Check status of the vm
+    wait_for 60 1 "openstack server show -c status -f value pci-passthru-instance | grep '^ACTIVE$'" \
+        "pci passthru VM booted and is in ACTIVE state" \
+        "echo \"ERROR: pci passthru VM is not in ACTIVE state.\""
+}
+
 function oncontroller_manila_generic_driver_setup()
 {
     if [[ $wantxenpv ]] ; then
@@ -4478,6 +4595,93 @@ function oncontroller
     run_on "$novacontroller" "oncontroller_$func $@"
 }
 
+function oncompute
+{
+    local func=$1 ; shift
+    run_on "$novacompute_kvm" "oncompute_$func $@"
+}
+
+function oncompute_add_intel_iommu_param
+{
+    # enabled intel_iommu in kernel param
+    sed -in "s/\(^GRUB_CMDLINE_LINUX_DEFAULT=.*\)\"$/\1 intel_iommu=on\"/g" /etc/default/grub
+    update-bootloader
+}
+
+function oncompute_load_vfio_module_onboot
+{
+    # vfio-pci module is not loaded by default, however, we need it to bind to
+    # our candidate device to verify pci passthru
+    cat > /etc/modules-load.d/pci-passthru.conf <<EOF
+vfio_pci
+EOF
+    modprobe vfio-pci
+}
+
+function oncompute_bind_vfio_module_onboot
+{
+    # We need to bind vfio-pci to our candidate device to verify pci passthru
+    cat > /etc/udev/rules.d/99-pci-passthru.rules <<EOF
+KERNELS=="0000:00:1d.0", SUBSYSTEMS=="pci", ATTRS{device}=="0x1001", ATTRS{vendor}=="0x1af4", \
+RUN+="/bin/bash -c 'echo  0000:00:1d.0 > /sys/bus/pci/devices/0000:00:1d.0/driver/unbind'", \
+RUN+="/bin/bash -c 'echo 1af4 1001 > /sys/bus/pci/drivers/vfio-pci/new_id'"
+EOF
+}
+
+function oncompute_create_pci_passthru_conf
+{
+    # drop new configuration file under /etc/nova/nova.conf.d/ on compute
+    # [pci]
+    # passthrough_whitelist = { "address": "0000:41:00.0" }
+    # alias = { "vendor_id":"1af4", "product_id":"1001", "device_type":"type-PCI", "name":"a1" }
+    cat > /etc/nova/nova.conf.d/200-nova-pci-passthru.conf <<EOF
+[pci]
+passthrough_whitelist = { "address": "0000:00:1d.0" }
+alias = { "vendor_id":"1af4", "product_id":"1001", "device_type":"type-PCI", "name":"a1" }
+EOF
+    # Make the changes effective
+    systemctl restart openstack-nova-compute.service
+}
+
+function oncontroller_create_pci_passthru_conf
+{
+    # drop new configuration file under /etc/nova/nova.conf.d/ on controller
+    # [pci]
+    # alias = { "vendor_id":"1af4", "product_id":"1001", "device_type":"type-PCI", "name":"a1" }
+    cat > /etc/nova/nova.conf.d/200-nova-pci-passthru.conf <<EOF
+[pci]
+alias = { "vendor_id":"1af4", "product_id":"1001", "device_type":"type-PCI", "name":"a1" }
+EOF
+    # Make the changes effective
+    systemctl restart openstack-nova-api.service
+}
+
+function append_pci_passthru_filter
+{
+    set -x
+    prepare_proposal_for_modification nova
+    local pfile=$(get_proposal_filename nova default)
+    local nova_conf="/etc/nova/nova.conf.d/100-nova.conf"
+    local enabled_filters=
+    enabled_filters=$(ssh "$novacontroller" "grep enabled_filters $nova_conf | cut -d' ' -f3")
+
+    enabled_filters="${enabled_filters},PciPassthroughFilter"
+
+    # append PciPassthroughFilter to enabled_filters
+    proposal_set_value nova default \
+        "['attributes']['nova']['scheduler']['enabled_filters']" \
+        "'$enabled_filters'"
+
+    crowbar nova proposal --file=$pfile edit default ||\
+        complain 88 "'crowbar nova proposal --file=$pfile edit default' failed with exit code: $?"
+
+    crowbar_proposal_commit nova
+
+    # Make the changes effective right away
+    run_on "$novacontroller" 'sed -in "s/\(enabled_filters = .*\)$/\1,PciPassthroughFilter/g"' $nova_conf
+    run_on "$novacontroller" "systemctl restart openstack-nova-scheduler.service"
+}
+
 function install_suse_ca
 {
     # trust build key - workaround https://bugzilla.opensuse.org/show_bug.cgi?id=935020