KubeVirt KVM Real-Time 原理

apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
  # a lot of metadata
spec:
  running: true
  template:
    metadata:
      labels:
        kubevirt.io/vm: ecs-realtime
    spec:
      architecture: amd64
      domain:
        cpu:
          cores: 1
          dedicatedCpuPlacement: true
          model: host-passthrough
          numa:
            guestMappingPassthrough: {}
          realtime: {}
          sockets: 2
          threads: 1
        devices:
          disks:
          - bootOrder: 1
            disk:
              bus: virtio
            name: bootdisk
          - disk:
              bus: virtio
            name: cloudinitdisk
          interfaces:
          - bridge: {}
            name: attachnet1
        machine:
          type: q35
        memory:
          guest: 2Gi
          hugepages:
            pageSize: 2Mi
        resources:
          limits:
            cpu: "2"
            memory: 2Gi
          requests:
            cpu: "2"
            memory: 2Gi
      hostname: ecs-realtime
      networks:
      - multus:
          networkName: mec-nets/attachnet1
        name: attachnet1
      volumes:
      - name: bootdisk
        persistentVolumeClaim:
          claimName: ecs-realtime-bootpvc-hlie0p
      - cloudInitConfigDrive:
          userData: |-
            #cloud-config
            user: root
            password: atomic
            ssh_pwauth: True
            chpasswd: { expire: False }            
        name: cloudinitdisk

func (n *NodeLabeller) prepareLabels(node *v1.Node, cpuModels []string, cpuFeatures cpuFeatures, hostCpuModel hostCPUModel, obsoleteCPUsx86 map[string]bool) map[string]string {
    // a lot of code here
    capable, err := isNodeRealtimeCapable()
    if err != nil {
        n.logger.Reason(err).Error("failed to identify if a node is capable of running realtime workloads")
    }
    if capable {
        newLabels[kubevirtv1.RealtimeLabel] = ""
    }
    // a lot of code here
}

// https://github.com/kubevirt/kubevirt/blob/04a198e5a33cd1369e534f55b26920dce7776f69/pkg/virt-handler/node-labeller/node_labeller.go#L367-L381

const kernelSchedRealtimeRuntimeInMicrosecods = "kernel.sched_rt_runtime_us"

func isNodeRealtimeCapable() (bool, error) {
    ret, err := exec.Command("sysctl", kernelSchedRealtimeRuntimeInMicrosecods).CombinedOutput()
    if err != nil {
        return false, err
    }
    st := strings.Trim(string(ret), "\n")
    return fmt.Sprintf("%s = -1", kernelSchedRealtimeRuntimeInMicrosecods) == st, nil
}

    if newVMI.IsRealtimeEnabled() {
        log.Log.V(4).Info("Add realtime node label selector")
        addNodeSelector(newVMI, v1.RealtimeLabel)
    }

$ kubectl get po virt-launcher-ecs-realtime-wt2k8 -o jsonpath='{.spec.nodeSelector}' | jq
{
  "cpumanager": "true",
  "kubernetes.io/arch": "amd64",
  "kubevirt.io/realtime": "",
  "kubevirt.io/schedulable": "true"
}

$ kubectl get nodes -l "kubevirt.io/realtime"
NAME      STATUS   ROLES                  AGE    VERSION
node164   Ready    control-plane,worker   226d   v1.27.2

<cputune>
  <emulatorpin cpuset="8-9"/>
  <vcpupin vcpu="0" cpuset="12"/>
  <vcpupin vcpu="1" cpuset="13"/>
  <vcpupin vcpu="2" cpuset="14"/>
  <vcpupin vcpu="3" cpuset="15"/>
  <vcpusched vcpus='0-4' scheduler='fifo' priority='1'/>
</cputune>

$ kubectl exec -it virt-launcher-ecs-realtime-wt2k8 -- ps -ef
UID         PID   PPID  C STIME TTY          TIME CMD
qemu          1      0  0 09:09 ?        00:00:00 /usr/bin/virt-launcher-monitor
qemu         12      1  0 09:09 ?        00:00:07 /usr/bin/virt-launcher --qemu-
qemu         19     12  0 09:09 ?        00:00:08 /usr/sbin/virtqemud -f /var/ru
qemu         30     12  0 09:09 ?        00:00:01 /usr/sbin/virtlogd -f /etc/lib
qemu         71      1  0 09:09 ?        00:00:54 /usr/libexec/qemu-kvm -name gu

$ kubectl exec -it virt-launcher-ecs-realtime-wt2k8 -- ps -p 71 -L
PID    LWP TTY          TIME CMD
    71     71 ?        00:00:05 qemu-kvm
    71     73 ?        00:00:00 qemu-kvm
    71     74 ?        00:00:00 TC tc-ram-node0
    71     75 ?        00:00:00 IO iothread1
    71     78 ?        00:00:05 IO mon_iothread
    71     79 ?        00:00:30 CPU 0/KVM
    71     80 ?        00:00:12 CPU 1/KVM
    71     82 ?        00:00:00 vnc_worker

$ kubectl exec -it virt-launcher-ecs-realtime-wt2k8 -- virsh dumpxml 1
<domain type='kvm' id='1'>
  <cputune>
    <vcpupin vcpu='0' cpuset='4'/>
    <vcpupin vcpu='1' cpuset='16'/>
  </cputune>
</domain>

# Real-Time VM
$ kubectl exec -it virt-launcher-ecs-realtime-wt2k8 -- chrt -p 79
pid 79's current scheduling policy: SCHED_FIFO
pid 79's current scheduling priority: 1
$ kubectl exec -it virt-launcher-ecs-realtime-wt2k8 -- chrt -p 80
pid 80's current scheduling policy: SCHED_FIFO
pid 80's current scheduling priority: 1

# 普通 VM
$ kubectl exec -it virt-launcher-ecs-test10-b5rtv -- chrt -p 76
pid 76's current scheduling policy: SCHED_OTHER
pid 76's current scheduling priority: 0
$ kubectl exec -it virt-launcher-ecs-test10-b5rtv -- chrt -p 77
pid 77's current scheduling policy: SCHED_OTHER
pid 77's current scheduling priority: 0

$ kubectl get ds virt-handler -n kubevirt -o jsonpath='{.spec.template.spec.containers[0].securityContext}' | jq
{
  "privileged": true,
  "seLinuxOptions": {
    "level": "s0"
  }
}

// configureRealTimeVCPUs parses the realtime mask value and configured the selected vcpus
// for real time workloads by setting the scheduler to FIFO and process priority equal to 1.
func (d *VirtualMachineController) configureVCPUScheduler(vmi *v1.VirtualMachineInstance) error {
    res, err := d.podIsolationDetector.Detect(vmi)
    if err != nil {
        return err
    }
    qemuProcess, err := res.GetQEMUProcess()
    if err != nil {
        return err
    }
    vcpus, err := getVCPUThreadIDs(qemuProcess.Pid())
    if err != nil {
        return err
    }
    mask, err := parseCPUMask(vmi.Spec.Domain.CPU.Realtime.Mask)
    if err != nil {
        return err
    }
    for vcpuID, threadID := range vcpus {
        if mask.isEnabled(vcpuID) {
            param := schedParam{priority: 1}
            tid, err := strconv.Atoi(threadID)
            if err != nil {
                return err
            }
            err = schedSetScheduler(tid, schedFIFO, param)
            if err != nil {
                return fmt.Errorf("failed to set FIFO scheduling and priority 1 for thread %d: %w", tid, err)
            }
        }
    }
    return nil
}

func (d *VirtualMachineController) affinePitThread(vmi *v1.VirtualMachineInstance) error {
    res, err := d.podIsolationDetector.Detect(vmi)
    if err != nil {
        return err
    }
    // a lot of code here
    pitpid, err := res.KvmPitPid()
    if err != nil {
        return err
    }
    if pitpid == -1 {
        return nil
    }
    if vmi.IsRealtimeEnabled() {
        param := schedParam{priority: 2}
        err = schedSetScheduler(pitpid, schedFIFO, param)
        if err != nil {
            return fmt.Errorf("failed to set FIFO scheduling and priority 2 for thread %d: %w", pitpid, err)
        }
    }
}

func (r *RealIsolationResult) KvmPitPid() (int, error) {
    qemuprocess, err := r.GetQEMUProcess()
    if err != nil {
        return -1, err
    }
    processes, _ := ps.Processes()
    nspid, err := GetNspid(qemuprocess.Pid())
    if err != nil || nspid == -1 {
        return -1, err
    }
    pitstr := "kvm-pit/" + strconv.Itoa(nspid)

    for _, process := range processes {
        if process.Executable() == pitstr {
            return process.Pid(), nil
        }
    }
    return -1, nil
}

$ kubectl exec -it virt-launcher-ecs-realtime-wt2k8 -- ps -ef | grep qemu-kvm
qemu         71      1  0 Apr17 ?        00:03:09 /usr/libexec/qemu-kvm -name gu

$ ps -ef | grep "kvm-pit/71"
root     592956      2  0 Apr17 ?        00:00:00 [kvm-pit/71]

$ chrt -p 592956
pid 592956's current scheduling policy: SCHED_FIFO
pid 592956's current scheduling priority: 2