0

I have a server with 2*14 cores with Hyperthreading enabled. This means I have (2*14*2=28*2) 56 vCPUs.

[root@supervision1 /]# dmidecode -t 4 | egrep 'Socket Designation|Count'
    Socket Designation: SOCKET 0
    Core Count: 14
    Thread Count: 28
    Socket Designation: SOCKET 1
    Core Count: 14
    Thread Count: 28
[root@supervision1 /]# lscpu
Architecture:          x86_64
CPU op-mode(s):        32-bit, 64-bit
Byte Order:            Little Endian
CPU(s):                56
On-line CPU(s) list:   0-55
Thread(s) per core:    2
Core(s) per socket:    14
Socket(s):             2
NUMA node(s):          2
Vendor ID:             GenuineIntel
CPU family:            6
Model:                 79
Model name:            Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GHz
Stepping:              1
CPU MHz:               1204.980
CPU max MHz:           3300.0000
CPU min MHz:           1200.0000
BogoMIPS:              4788.62
Virtualization:        VT-x
L1d cache:             32K
L1i cache:             32K
L2 cache:              256K
L3 cache:              35840K
NUMA node0 CPU(s):     0-13,28-41
NUMA node1 CPU(s):     14-27,42-55
Flags:                 fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc aperfmperf eagerfpu pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch epb cat_l3 cdp_l3 intel_ppin intel_pt tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm rdt_a rdseed adx smap xsaveopt cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts

[root@supervision1 /]# virsh capabilities | grep 'cell\|cpu'
    <cpu>
    </cpu>
      <cells num='2'>
        <cell id='0'>
          <cpus num='28'>
            <cpu id='0' socket_id='0' core_id='0' siblings='0,28'/>
            <cpu id='1' socket_id='0' core_id='1' siblings='1,29'/>
            <cpu id='2' socket_id='0' core_id='2' siblings='2,30'/>
            <cpu id='3' socket_id='0' core_id='3' siblings='3,31'/>
            <cpu id='4' socket_id='0' core_id='4' siblings='4,32'/>
            <cpu id='5' socket_id='0' core_id='5' siblings='5,33'/>
            <cpu id='6' socket_id='0' core_id='6' siblings='6,34'/>
            <cpu id='7' socket_id='0' core_id='8' siblings='7,35'/>
            <cpu id='8' socket_id='0' core_id='9' siblings='8,36'/>
            <cpu id='9' socket_id='0' core_id='10' siblings='9,37'/>
            <cpu id='10' socket_id='0' core_id='11' siblings='10,38'/>
            <cpu id='11' socket_id='0' core_id='12' siblings='11,39'/>
            <cpu id='12' socket_id='0' core_id='13' siblings='12,40'/>
            <cpu id='13' socket_id='0' core_id='14' siblings='13,41'/>
            <cpu id='28' socket_id='0' core_id='0' siblings='0,28'/>
            <cpu id='29' socket_id='0' core_id='1' siblings='1,29'/>
            <cpu id='30' socket_id='0' core_id='2' siblings='2,30'/>
            <cpu id='31' socket_id='0' core_id='3' siblings='3,31'/>
            <cpu id='32' socket_id='0' core_id='4' siblings='4,32'/>
            <cpu id='33' socket_id='0' core_id='5' siblings='5,33'/>
            <cpu id='34' socket_id='0' core_id='6' siblings='6,34'/>
            <cpu id='35' socket_id='0' core_id='8' siblings='7,35'/>
            <cpu id='36' socket_id='0' core_id='9' siblings='8,36'/>
            <cpu id='37' socket_id='0' core_id='10' siblings='9,37'/>
            <cpu id='38' socket_id='0' core_id='11' siblings='10,38'/>
            <cpu id='39' socket_id='0' core_id='12' siblings='11,39'/>
            <cpu id='40' socket_id='0' core_id='13' siblings='12,40'/>
            <cpu id='41' socket_id='0' core_id='14' siblings='13,41'/>
          </cpus>
        </cell>
        <cell id='1'>
          <cpus num='28'>
            <cpu id='14' socket_id='1' core_id='0' siblings='14,42'/>
            <cpu id='15' socket_id='1' core_id='1' siblings='15,43'/>
            <cpu id='16' socket_id='1' core_id='2' siblings='16,44'/>
            <cpu id='17' socket_id='1' core_id='3' siblings='17,45'/>
            <cpu id='18' socket_id='1' core_id='4' siblings='18,46'/>
            <cpu id='19' socket_id='1' core_id='5' siblings='19,47'/>
            <cpu id='20' socket_id='1' core_id='6' siblings='20,48'/>
            <cpu id='21' socket_id='1' core_id='8' siblings='21,49'/>
            <cpu id='22' socket_id='1' core_id='9' siblings='22,50'/>
            <cpu id='23' socket_id='1' core_id='10' siblings='23,51'/>
            <cpu id='24' socket_id='1' core_id='11' siblings='24,52'/>
            <cpu id='25' socket_id='1' core_id='12' siblings='25,53'/>
            <cpu id='26' socket_id='1' core_id='13' siblings='26,54'/>
            <cpu id='27' socket_id='1' core_id='14' siblings='27,55'/>
            <cpu id='42' socket_id='1' core_id='0' siblings='14,42'/>
            <cpu id='43' socket_id='1' core_id='1' siblings='15,43'/>
            <cpu id='44' socket_id='1' core_id='2' siblings='16,44'/>
            <cpu id='45' socket_id='1' core_id='3' siblings='17,45'/>
            <cpu id='46' socket_id='1' core_id='4' siblings='18,46'/>
            <cpu id='47' socket_id='1' core_id='5' siblings='19,47'/>
            <cpu id='48' socket_id='1' core_id='6' siblings='20,48'/>
            <cpu id='49' socket_id='1' core_id='8' siblings='21,49'/>
            <cpu id='50' socket_id='1' core_id='9' siblings='22,50'/>
            <cpu id='51' socket_id='1' core_id='10' siblings='23,51'/>
            <cpu id='52' socket_id='1' core_id='11' siblings='24,52'/>
            <cpu id='53' socket_id='1' core_id='12' siblings='25,53'/>
            <cpu id='54' socket_id='1' core_id='13' siblings='26,54'/>
            <cpu id='55' socket_id='1' core_id='14' siblings='27,55'/>
          </cpus>
        </cell>
      </cells>
      <bank id='0' level='3' type='both' size='35' unit='MiB' cpus='0-13,28-41'/>
      <bank id='1' level='3' type='both' size='35' unit='MiB' cpus='14-27,42-55'/>
      <cpuselection/>
      <cpuselection/>

I have 2 VMs running on this hypervisor:

  • vm1 with 12 vCPUs
  • vm2 with 6 vCPUs

I did not configure any CPU pinning (not needed). below the some vcpus info of the VM1 and the VM2

[root@supervision1 /]# virsh vcpupin vm1
VCPU: CPU Affinity
----------------------------------
   0: 0-55
   1: 0-55
   2: 0-55
   3: 0-55
   4: 0-55
   5: 0-55
   6: 0-55
   7: 0-55
   8: 0-55
   9: 0-55
  10: 0-55
  11: 0-55

[root@supervision1 /]# virsh vcpupin vm2
VCPU: CPU Affinity
----------------------------------
   0: 0-55
   1: 0-55
   2: 0-55
   3: 0-55
   4: 0-55
   5: 0-55

Now I have to deploy a new VM (vm3) that needs the following (requirements from the VM software supplier):

  • Hypertheading disabled
  • 4 CPUs with CPU pinning

How can I do? it seems isolcpu is deprecated or not generally advised.

  • VCPUPIN: If I use vcpupin in the VM3 XML definition, I see the CPU pinning is ok, but these CPUs are not excluded from the other VMs vCPUs.
[root@supervision1 /]# virsh dumpxml vm3 | grep cpu
  <vcpu placement='static'>4</vcpu>
  <cputune>
    <vcpupin vcpu='0' cpuset='0'/>
    <vcpupin vcpu='1' cpuset='1'/>
    <vcpupin vcpu='2' cpuset='2'/>
    <vcpupin vcpu='3' cpuset='3'/>
  </cputune>
  <cpu>
  </cpu>

[root@supervision1 /]# virsh vcpupin vm3
VCPU: CPU Affinity
----------------------------------
   0: 0
   1: 1
   2: 2
   3: 3

Even if I manage to restrict these vCPUs to VM3, the corresponding siblings (28,29,30,31) won't be isolated from VM1 and VM2. Or I could pin vCPU0 to cpuset 0,28, vCPU1 to cpuset 1,28 etc... But they won't be isolated from VM1 and VM2 (to have a hyperthreading disable like behaviour...)

I suppose there are other options or parameters I don't know? Any advice or help would be greatly appreciated :-) Or maybe I cannot use the same hypervisor for these 3 VMs?

Kind regards,

Emmanuel

Manumammouth
  • 1
  • The logic that you assume with that 2x14 cores mean 56 vCPUs is flawed. Hyperthreading is not the real equivalent of a core. If you want to play things safe you should not allocate 20% more vCPUs than the real number of cores which is 28. – Overmind Nov 08 '19 at 13:57
  • Thanks a lot Overmind for this remark. OK I think you're right, but the hypervisor considers well 56 vCPUs and can load balance the vCPUs thread over the 56 siblings. But concerning my question: how to dedicate 4 CPUs to a VM in a hyperthread hypervisor? do you have any idea? I was maybe thinking of using CPU passthrough, but it seems it could lead to other issues. – Manumammouth Nov 08 '19 at 14:05
  • You have hardware reservation which can make sure the allocated cores will not be used for some other VM. This is a good way to do things: allocate a few cores for your critical VM and then you can safely over-allocate the others to the less important VMs. – Overmind Nov 11 '19 at 06:22
  • Thanks Overmind for your remark. Could you please elaborate regarding the idea of "allocated a few cores for your critical VMs"? How to do that? islocpus, cpuset, CPU host-passthrough, it's not clear to me. – Manumammouth Nov 12 '19 at 08:40
  • Here's a guide on how to allocate cores to a specific VM: https://docs.fedoraproject.org/en-US/Fedora/13/html/Virtualization_Guide/ch25s06.html – Overmind Nov 12 '19 at 11:46
  • Thanks Overmind. How to allocate cores to a specific VMs with this method is now quite clear and it's basically the same method I described in my answer. However how to avoid other VMs to use these resources is not really answered yet (except by explicitly pinning the vCPUs of the other resources to other CPUs as I wrote in the answer). – Manumammouth Nov 19 '19 at 10:06

1 Answers1

0

I found a way of doing what I was looking for, although it's not very nice and scalable. I just pinned the vCPUs of the VM3 to the 4 last CPUs (4 pairs of siblings):

[root@supervision1 ~]# virsh vcpupin vm3
VCPU: CPU Affinity
----------------------------------
   0: 24,52
   1: 25,53
   2: 26,54
   3: 27,55

[root@supervision1 ~]# virsh dumpxml vm3 | grep cpu
  <vcpu placement='static'>4</vcpu>
  <cputune>
    <vcpupin vcpu='0' cpuset='24,52'/>
    <vcpupin vcpu='1' cpuset='25,53'/>
    <vcpupin vcpu='2' cpuset='26,54'/>
    <vcpupin vcpu='3' cpuset='27,55'/>
    <emulatorpin cpuset='0-23,28-51'/>
  </cputune>
  <cpu>
  </cpu>

Then I needed to change the definition of VM1 and VM2 (what it's not so nice...) to use only the other CPUs (0-23,28-51 siblings)

[root@supervision1 ~]# virsh vcpupin vm1
VCPU: CPU Affinity
----------------------------------
   0: 0-23,28-51
   1: 0-23,28-51
   2: 0-23,28-51
   3: 0-23,28-51
   4: 0-23,28-51
   5: 0-23,28-51
   6: 0-23,28-51
   7: 0-23,28-51
   8: 0-23,28-51
   9: 0-23,28-51
  10: 0-23,28-51
  11: 0-23,28-51

[root@supervision1 ~]# virsh vcpupin vm2
VCPU: CPU Affinity
----------------------------------
   0: 0-23,28-51
   1: 0-23,28-51
   2: 0-23,28-51
   3: 0-23,28-51
   4: 0-23,28-51
   5: 0-23,28-51
[root@supervision1 ~]# virsh dumpxml vm1| grep cpu
  <vcpu placement='static'>12</vcpu>
  <cputune>
    <vcpupin vcpu='0' cpuset='0-23,28-51'/>
    <vcpupin vcpu='1' cpuset='0-23,28-51'/>
    <vcpupin vcpu='2' cpuset='0-23,28-51'/>
    <vcpupin vcpu='3' cpuset='0-23,28-51'/>
    <vcpupin vcpu='4' cpuset='0-23,28-51'/>
    <vcpupin vcpu='5' cpuset='0-23,28-51'/>
    <vcpupin vcpu='6' cpuset='0-23,28-51'/>
    <vcpupin vcpu='7' cpuset='0-23,28-51'/>
    <vcpupin vcpu='8' cpuset='0-23,28-51'/>
    <vcpupin vcpu='9' cpuset='0-23,28-51'/>
    <vcpupin vcpu='10' cpuset='0-23,28-51'/>
    <vcpupin vcpu='11' cpuset='0-23,28-51'/>
    <emulatorpin cpuset='0-23,28-51'/>
  </cputune>
  <cpu mode='custom' match='exact' check='full'>
  </cpu>
[root@supervision1 ~]# virsh dumpxml vm2| grep cpu
  <vcpu placement='static'>6</vcpu>
  <cputune>
    <vcpupin vcpu='0' cpuset='0-23,28-51'/>
    <vcpupin vcpu='1' cpuset='0-23,28-51'/>
    <vcpupin vcpu='2' cpuset='0-23,28-51'/>
    <vcpupin vcpu='3' cpuset='0-23,28-51'/>
    <vcpupin vcpu='4' cpuset='0-23,28-51'/>
    <vcpupin vcpu='5' cpuset='0-23,28-51'/>
    <emulatorpin cpuset='0-23,28-51'/>
  </cputune>
  <cpu mode='custom' match='exact' check='full'>
  </cpu>

NB: I also set the emulatorpin to use CPUs not used by the VM3 At least I think it answers my question. If anyone has a better idea, or remarks I would be glad to read!

Manumammouth
  • 1