Merge pull request #3489 from kwu83tw/pci_passII

Add mocked disk device to test pci passthrough

scripts/lib/libvirt/compute-config

@@ -47,9 +47,11 @@ def main():
                         help="Local Repository Directory Target")
     parser.add_argument("--ipmi", action='store_true',
                         help="Whether to simulate IPMI BMC devices")
+    parser.add_argument("--pcipassthrough", action='store_true',
+                        help="Whether to simulate PCI passthrough devices")
     args = parser.parse_args()
 
-    print(libvirt_setup.compute_config(args, libvirt_setup.cpuflags()))
+    print(libvirt_setup.compute_config(args, libvirt_setup.cpuflags(args.pcipassthrough)))
 
 
 if __name__ == "__main__":

scripts/lib/libvirt/libvirt_setup.py

@@ -28,13 +28,13 @@ def remove_files(files):
         os.remove(f)
 
 
-def cpuflags():
+def cpuflags(pcipassthrough=False):
     cpu_template = "cpu-default.xml"
     cpu_info = readfile("/proc/cpuinfo")
     if re.search("^CPU architecture.* 8", cpu_info, re.MULTILINE):
         cpu_template = "cpu-arm64.xml"
     elif re.search("^vendor_id.*GenuineIntel", cpu_info, re.MULTILINE):
-        cpu_template = "cpu-intel.xml"
+        cpu_template = get_intel_cputemplate(pcipassthrough)
     elif re.search("^vendor_id.*AuthenticAMD", cpu_info, re.MULTILINE):
         cpu_template = "cpu-amd.xml"
     elif re.search("^vendor_id.*IBM/S390", cpu_info, re.MULTILINE):
@@ -43,6 +43,13 @@ def cpuflags():
     return readfile(os.path.join(TEMPLATE_DIR, cpu_template))
 
 
+def get_intel_cputemplate(pcipassthrough=False):
+    cpu_template = "cpu-intel.xml"
+    if pcipassthrough:
+        cpu_template = "cpu-intel-pcipassthrough.xml"
+    return cpu_template
+
+
 def hypervisor_has_virtio(libvirt_type):
     return libvirt_type == "kvm"
 
@@ -257,7 +264,7 @@ def compute_config(args, cpu_flags=cpuflags()):
             "bus='{0}' target='0' unit='0'/>"
 
     # override nic model for ironic setups
-    if args.ironicnic >= 0:
+    if args.ironicnic >= 0 and not args.pcipassthrough:
         configopts['nicmodel'] = 'e1000'
 
     controller_raid_volumes = args.controller_raid_volumes
@@ -320,7 +327,32 @@ def compute_config(args, cpu_flags=cpuflags()):
             'target_address': target_address.format('0x1f')},
             configopts))
 
-    if args.ipmi:
+    machine = ""
+    machine = get_default_machine(args.emulator)
+    pciecontrollers = ""
+    iommudevice = ""
+    extravolume = ""
+    if args.pcipassthrough and not args.drbdserial:
+        volume_template = string.Template(
+            readfile(os.path.join(TEMPLATE_DIR, "extra-volume.xml")))
+        extravolume = volume_template.substitute(merge_dicts({
+            'volume_serial': "{0}-node{1}-extra".format(
+                args.cloud,
+                args.nodecounter),
+            'source_dev': "{0}/{1}.node{2}-extra".format(
+                args.vdiskdir,
+                args.cloud,
+                args.nodecounter),
+            'target_dev': targetdevprefix + ''.join(next(alldevices)),
+            'target_address': target_address.format('0x1d')},
+            configopts))
+        iommudevice = readfile(os.path.join(
+            TEMPLATE_DIR, 'iommu-device-default.xml'))
+        machine = "q35"
+        pciecontrollers = readfile(os.path.join(
+            TEMPLATE_DIR, 'pcie-root-bridge-default.xml'))
+
+    if args.ipmi and not args.pcipassthrough:
         values = dict(
           nodecounter=args.nodecounter
         )
@@ -329,7 +361,7 @@ def compute_config(args, cpu_flags=cpuflags()):
     else:
         ipmi_config = ''
 
-    if not hypervisor_has_virtio(libvirt_type):
+    if not hypervisor_has_virtio(libvirt_type) and not args.pcipassthrough:
         target_address = target_address.format('0')
         # map virtio addr to ide:
         raidvolume = raidvolume.replace("bus='0x17'", "bus='1'")
@@ -342,13 +374,16 @@ def compute_config(args, cpu_flags=cpuflags()):
         nodememory=nodememory,
         vcpus=args.vcpus,
         march=get_machine_arch(),
-        machine=get_default_machine(args.emulator),
+        machine=machine,
         osloader=get_os_loader(firmware_type=args.firmwaretype),
         cpuflags=cpu_flags,
         consoletype=get_console_type(),
         raidvolume=raidvolume,
         cephvolume=cephvolume,
         drbdvolume=drbdvolume,
+        pciecontrollers=pciecontrollers,
+        extravolume=extravolume,
+        iommudevice=iommudevice,
         nics=net_interfaces_config(args, configopts["nicmodel"]),
         maindiskaddress=get_maindisk_address(),
         videodevices=get_video_devices(),

scripts/lib/libvirt/templates/compute-node.xml

@@ -25,6 +25,9 @@ $cpuflags
 $raidvolume
 $cephvolume
 $drbdvolume
+$extravolume
+$pciecontrollers
+$iommudevice
 $nics
 $serialdevice
     <console type='pty'>

scripts/lib/libvirt/templates/cpu-intel-pcipassthrough.xml

@@ -0,0 +1,11 @@
+  <features>
+    <acpi/>
+    <apic/>
+    <pae/>
+    <ioapic driver='qemu'/>
+  </features>
+  <cpu mode='custom' match='exact'>
+    <model fallback='allow'>Haswell-noTSX</model>
+    <feature policy='require' name='vmx'/>
+    <feature policy='disable' name='monitor'/>
+  </cpu>

scripts/lib/libvirt/templates/iommu-device-default.xml

@@ -0,0 +1,3 @@
+    <iommu model='intel'>
+      <driver intremap='on' caching_mode='on'/>
+    </iommu>

scripts/lib/libvirt/templates/pcie-root-bridge-default.xml

@@ -0,0 +1,14 @@
+    <controller type='pci' index='0' model='pcie-root'>
+      <alias name='pcie.0'/>
+    </controller>
+    <controller type='pci' index='1' model='dmi-to-pci-bridge'>
+      <model name='i82801b11-bridge'/>
+      <alias name='pci.1'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x1e' function='0x0'/>
+    </controller>
+    <controller type='pci' index='2' model='pci-bridge'>
+      <model name='pci-bridge'/>
+      <target chassisNr='2'/>
+      <alias name='pci.2'/>
+      <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
+    </controller>

scripts/lib/mkcloud-driver-libvirt.sh

@@ -285,6 +285,17 @@ function libvirt_do_create_cloud_lvm()
         done
     fi
 
+    # create an extra volume to test pci passthrough
+    if [ $extravolumenumber -gt 0 ] ; then
+        for i in $(nodes ids all) ; do
+            for n in $(seq 1 $extravolumenumber) ; do
+                onhost_get_next_pv_device
+                hdd_size=${extravolume_hdd_size}
+                _lvcreate $cloud.node$i-extra $hdd_size $cloudvg $next_pv_device
+            done
+        done
+    fi
+
     # create volumes for drbd
     if [ $drbd_hdd_size != 0 ] ; then
         for i in `seq 1 2`; do

scripts/mkcloud

@@ -112,6 +112,12 @@ iscloudver 7plus && : ${controller_node_memory:=12582912}
 : ${controller_ceph_hdd_size:=25}
 : ${lonelynode_hdd_size:=20}
 : ${ironicnode_hdd_size:=20}
+: ${want_pci_passthrough:=0}
+if [[ $want_pci_passthrough = 1 ]]; then
+    # Create extra volume to test pci passthrough
+    : ${extravolumenumber:=1}
+    : ${extravolume_hdd_size:=1}
+fi
 
 if [[ $hacloud = 1 ]] && ( [[ "$want_database_sql_engine" == "postgresql" ]] || iscloudver 6minus ) ; then
     : ${drbd_hdd_size:=15}
@@ -739,6 +745,11 @@ function setupnodes
             localrepos_params="--localreposrc ${localreposdir_src} --localrepotgt ${localreposdir_target}"
         fi
 
+        local pcipassthrough_params=""
+        if [[ $want_pci_passthrough = 1 ]]; then
+            pcipassthrough_params="--pcipassthrough"
+        fi
+
         local ipmi_params=""
         if [[ $want_ipmi = 1 ]] ; then
             ipmi_params="--ipmi"
@@ -757,6 +768,7 @@ function setupnodes
                         $mac_params \
                         $ironic_params \
                         $ipmi_params \
+                        $pcipassthrough_params \
                         --cephvolumenumber $cephvolumenumber \
                         --drbdserial "$drbd_serial"\
                         --computenodememory $compute_node_memory \