Add mocked disk device to test pci passthrough

We need to specify q35 as the machine type in order to support IOMMU and
tweaking pcie controllers to place our virtio device properly.
1. Add option for pci passthrough
2. Add script to construct node template

scripts/lib/libvirt/compute-config

@@ -47,9 +47,11 @@ def main():
                         help="Local Repository Directory Target")
     parser.add_argument("--ipmi", action='store_true',
                         help="Whether to simulate IPMI BMC devices")
+    parser.add_argument("--pcipassthrough", action='store_true',
+                        help="Whether to simulate PCI passthrough devices")
     args = parser.parse_args()
 
-    print(libvirt_setup.compute_config(args, libvirt_setup.cpuflags()))
+    print(libvirt_setup.compute_config(args, libvirt_setup.cpuflags(args.pcipassthrough)))
 
 
 if __name__ == "__main__":

scripts/lib/libvirt/libvirt_setup.py

@@ -28,13 +28,13 @@ def remove_files(files):
         os.remove(f)
 
 
-def cpuflags():
+def cpuflags(pcipassthrough=False):
     cpu_template = "cpu-default.xml"
     cpu_info = readfile("/proc/cpuinfo")
     if re.search("^CPU architecture.* 8", cpu_info, re.MULTILINE):
         cpu_template = "cpu-arm64.xml"
     elif re.search("^vendor_id.*GenuineIntel", cpu_info, re.MULTILINE):
-        cpu_template = "cpu-intel.xml"
+        cpu_template = get_intel_cputemplate(pcipassthrough)
     elif re.search("^vendor_id.*AuthenticAMD", cpu_info, re.MULTILINE):
         cpu_template = "cpu-amd.xml"
     elif re.search("^vendor_id.*IBM/S390", cpu_info, re.MULTILINE):
@@ -43,6 +43,13 @@ def cpuflags():
     return readfile(os.path.join(TEMPLATE_DIR, cpu_template))
 
 
+def get_intel_cputemplate(pcipassthrough=False):
+    cpu_template = "cpu-intel.xml"
+    if pcipassthrough:
+        cpu_template = "cpu-intel-pcipassthrough.xml"
+    return cpu_template
+
+
 def hypervisor_has_virtio(libvirt_type):
     return libvirt_type == "kvm"
 
@@ -257,7 +264,7 @@ def compute_config(args, cpu_flags=cpuflags()):
             "bus='{0}' target='0' unit='0'/>"
 
     # override nic model for ironic setups
-    if args.ironicnic >= 0:
+    if args.ironicnic >= 0 and not args.pcipassthrough:
         configopts['nicmodel'] = 'e1000'
 
     controller_raid_volumes = args.controller_raid_volumes
@@ -320,7 +327,32 @@ def compute_config(args, cpu_flags=cpuflags()):
             'target_address': target_address.format('0x1f')},
             configopts))
 
-    if args.ipmi:
+    machine = ""
+    machine = get_default_machine(args.emulator)
+    pciecontrollers = ""
+    iommudevice = ""
+    extravolume = ""
+    if args.pcipassthrough and not args.drbdserial:
+        volume_template = string.Template(
+            readfile(os.path.join(TEMPLATE_DIR, "extra-volume.xml")))
+        extravolume = volume_template.substitute(merge_dicts({
+            'volume_serial': "{0}-node{1}-extra".format(
+                args.cloud,
+                args.nodecounter),
+            'source_dev': "{0}/{1}.node{2}-extra".format(
+                args.vdiskdir,
+                args.cloud,
+                args.nodecounter),
+            'target_dev': targetdevprefix + ''.join(next(alldevices)),
+            'target_address': target_address.format('0x1d')},
+            configopts))
+        iommudevice = readfile(os.path.join(
+            TEMPLATE_DIR, 'iommu-device-default.xml'))
+        machine = "q35"
+        pciecontrollers = readfile(os.path.join(
+            TEMPLATE_DIR, 'pcie-root-bridge-default.xml'))
+
+    if args.ipmi and not args.pcipassthrough:
         values = dict(
           nodecounter=args.nodecounter
         )
@@ -329,7 +361,7 @@ def compute_config(args, cpu_flags=cpuflags()):
     else:
         ipmi_config = ''
 
-    if not hypervisor_has_virtio(libvirt_type):
+    if not hypervisor_has_virtio(libvirt_type) and not args.pcipassthrough:
         target_address = target_address.format('0')
         # map virtio addr to ide:
         raidvolume = raidvolume.replace("bus='0x17'", "bus='1'")
@@ -342,13 +374,16 @@ def compute_config(args, cpu_flags=cpuflags()):
         nodememory=nodememory,
         vcpus=args.vcpus,
         march=get_machine_arch(),
-        machine=get_default_machine(args.emulator),
+        machine=machine,
         osloader=get_os_loader(firmware_type=args.firmwaretype),
         cpuflags=cpu_flags,
         consoletype=get_console_type(),
         raidvolume=raidvolume,
         cephvolume=cephvolume,
         drbdvolume=drbdvolume,
+        pciecontrollers=pciecontrollers,
+        extravolume=extravolume,
+        iommudevice=iommudevice,
         nics=net_interfaces_config(args, configopts["nicmodel"]),
         maindiskaddress=get_maindisk_address(),
         videodevices=get_video_devices(),

scripts/lib/libvirt/templates/compute-node.xml

@@ -25,6 +25,9 @@ $cpuflags
 $raidvolume
 $cephvolume
 $drbdvolume
+$extravolume
+$pciecontrollers
+$iommudevice
 $nics
 $serialdevice
     <console type='pty'>

scripts/lib/libvirt/templates/cpu-intel-pcipassthrough.xml

@@ -0,0 +1,11 @@
+  <features>
+    <acpi/>
+    <apic/>
+    <pae/>
+    <ioapic driver='qemu'/>
+  </features>
+  <cpu mode='custom' match='exact'>
+    <model fallback='allow'>Haswell-noTSX</model>
+    <feature policy='require' name='vmx'/>
+    <feature policy='disable' name='monitor'/>
+  </cpu>

scripts/lib/libvirt/templates/iommu-device-default.xml

@@ -0,0 +1,3 @@
+    <iommu model='intel'>
+      <driver intremap='on' caching_mode='on'/>
+    </iommu>

scripts/lib/libvirt/templates/pcie-root-bridge-default.xml

@@ -0,0 +1,14 @@
+    <controller type='pci' index='0' model='pcie-root'>
+      <alias name='pcie.0'/>
+    </controller>
+    <controller type='pci' index='1' model='dmi-to-pci-bridge'>
+      <model name='i82801b11-bridge'/>
+      <alias name='pci.1'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x1e' function='0x0'/>
+    </controller>
+    <controller type='pci' index='2' model='pci-bridge'>
+      <model name='pci-bridge'/>
+      <target chassisNr='2'/>
+      <alias name='pci.2'/>
+      <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
+    </controller>

scripts/lib/mkcloud-driver-libvirt.sh

@@ -285,6 +285,17 @@ function libvirt_do_create_cloud_lvm()
         done
     fi
 
+    # create an extra volume to test pci passthrough
+    if [ $extravolumenumber -gt 0 ] ; then
+        for i in $(nodes ids all) ; do
+            for n in $(seq 1 $extravolumenumber) ; do
+                onhost_get_next_pv_device
+                hdd_size=${extravolume_hdd_size}
+                _lvcreate $cloud.node$i-extra $hdd_size $cloudvg $next_pv_device
+            done
+        done
+    fi
+
     # create volumes for drbd
     if [ $drbd_hdd_size != 0 ] ; then
         for i in `seq 1 2`; do

scripts/mkcloud

@@ -112,6 +112,12 @@ iscloudver 7plus && : ${controller_node_memory:=12582912}
 : ${controller_ceph_hdd_size:=25}
 : ${lonelynode_hdd_size:=20}
 : ${ironicnode_hdd_size:=20}
+: ${want_pci_passthrough:=0}
+if [[ $want_pci_passthrough = 1 ]]; then
+    # Create extra volume to test pci passthrough
+    : ${extravolumenumber:=1}
+    : ${extravolume_hdd_size:=1}
+fi
 
 if [[ $hacloud = 1 ]] && ( [[ "$want_database_sql_engine" == "postgresql" ]] || iscloudver 6minus ) ; then
     : ${drbd_hdd_size:=15}
@@ -739,6 +745,11 @@ function setupnodes
             localrepos_params="--localreposrc ${localreposdir_src} --localrepotgt ${localreposdir_target}"
         fi
 
+        local pcipassthrough_params=""
+        if [[ $want_pci_passthrough = 1 ]]; then
+            pcipassthrough_params="--pcipassthrough"
+        fi
+
         local ipmi_params=""
         if [[ $want_ipmi = 1 ]] ; then
             ipmi_params="--ipmi"
@@ -757,6 +768,7 @@ function setupnodes
                         $mac_params \
                         $ironic_params \
                         $ipmi_params \
+                        $pcipassthrough_params \
                         --cephvolumenumber $cephvolumenumber \
                         --drbdserial "$drbd_serial"\
                         --computenodememory $compute_node_memory \