一文搞懂 GPU 共享方案： NVIDIA Time Slicing

root@liqivm:~# k describe node gpu01|grep Capacity -A 7
Capacity:
  cpu:                128
  ephemeral-storage:  879000896Ki
  hugepages-1Gi:      0
  hugepages-2Mi:      0
  memory:             1056457696Ki
  nvidia.com/gpu:     8
  pods:               110

apiVersion: v1
kind: Pod
metadata:
  name: gpu-pod
spec:
  containers:
  - name: gpu-container
    image: nvidia/cuda:11.0-base   # 一个支持 GPU 的镜像
    resources:
      limits:
        nvidia.com/gpu: 1          # 申请 1 个 GPU
    command: ["nvidia-smi"]         # 示例命令，显示 GPU 的信息
  restartPolicy: OnFailure

root@liqivm:~# k describe node gpu01|grep Capacity -A 7
Capacity:
  cpu:                128
  ephemeral-storage:  879000896Ki
  hugepages-1Gi:      0
  hugepages-2Mi:      0
  memory:             1056457696Ki
  nvidia.com/gpu:     1
  pods:               110

root@liqivm:~# k describe node gpu01|grep Capacity -A 7
Capacity:
  cpu:                128
  ephemeral-storage:  879000896Ki
  hugepages-1Gi:      0
  hugepages-2Mi:      0
  memory:             1056457696Ki
  nvidia.com/gpu:     10
  pods:               110

apiVersion: v1
kind: ConfigMap
metadata:
  name: time-slicing-config-all
data:
  any: |-
    version: v1
    flags:
      migStrategy: none
    sharing:
      timeSlicing:
        renameByDefault: false
        failRequestsGreaterThanOne: false
        resources:
          - name: nvidia.com/gpu
            replicas: 4    

kubectl create -n gpu-operator -f time-slicing-config-all.yaml

kubectl patch clusterpolicies.nvidia.com/cluster-policy \
    -n gpu-operator --type merge \
    -p '{"spec": {"devicePlugin": {"config": {"name": "time-slicing-config-all", "default": "any"}}}}'

kubectl get events -n gpu-operator --sort-by='.lastTimestamp'

kubectl describe node xxx

...
Labels:
                  nvidia.com/gpu.count=4
                  nvidia.com/gpu.product=Tesla-T4-SHARED
                  nvidia.com/gpu.replicas=4
Capacity:
  nvidia.com/gpu: 16
  ...
Allocatable:
  nvidia.com/gpu: 16
  ...

apiVersion: apps/v1
kind: Deployment
metadata:
  name: time-slicing-verification
  labels:
    app: time-slicing-verification
spec:
  replicas: 2
  selector:
    matchLabels:
      app: time-slicing-verification
  template:
    metadata:
      labels:
        app: time-slicing-verification
    spec:
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
      hostPID: true
      containers:
        - name: cuda-sample-vector-add
          image: "nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda11.7.1-ubuntu20.04"
          command: ["/bin/bash", "-c", "--"]
          args:
            - while true; do /cuda-samples/vectorAdd; done
          resources:
           limits:
             nvidia.com/gpu: 1

$ kubectl get pods
NAME                                         READY   STATUS    RESTARTS   AGE
time-slicing-verification-7cdc7f87c5-lkd9d   1/1     Running   0          23s
time-slicing-verification-7cdc7f87c5-rrzq7   1/1     Running   0          23s
time-slicing-verification-7cdc7f87c5-s8qwk   1/1     Running   0          23s
time-slicing-verification-7cdc7f87c5-xhmb7   1/1     Running   0          23s
time-slicing-verification-7cdc7f87c5-zsncp   1/1     Running   0          23s

$ kubectl logs deploy/time-slicing-verification
Found 5 pods, using pod/time-slicing-verification-7cdc7f87c5-s8qwk
[Vector addition of 50000 elements]
Copy input data from the host memory to the CUDA device
CUDA kernel launch with 196 blocks of 256 threads
Copy output data from the CUDA device to the host memory
Test PASSED
Done
[Vector addition of 50000 elements]
Copy input data from the host memory to the CUDA device
CUDA kernel launch with 196 blocks of 256 threads
Copy output data from the CUDA device to the host memory
...

apiVersion: v1
kind: ConfigMap
metadata:
  name: time-slicing-config-fine
data:
  a100-40gb: |-
    version: v1
    flags:
      migStrategy: mixed
    sharing:
      timeSlicing:
        resources:
        - name: nvidia.com/gpu
          replicas: 8
        - name: nvidia.com/mig-1g.5gb
          replicas: 2
        - name: nvidia.com/mig-2g.10gb
          replicas: 2
        - name: nvidia.com/mig-3g.20gb
          replicas: 3
        - name: nvidia.com/mig-7g.40gb
          replicas: 7    
  tesla-t4: |-
    version: v1
    flags:
      migStrategy: none
    sharing:
      timeSlicing:
        resources:
        - name: nvidia.com/gpu
          replicas: 4    

kubectl create -n gpu-operator -f time-slicing-config-all.yaml

kubectl patch clusterpolicies.nvidia.com/cluster-policy \
    -n gpu-operator --type merge \
    -p '{"spec": {"devicePlugin": {"config": {"name": "time-slicing-config-fine"}}}}'

kubectl label node <node-name> nvidia.com/device-plugin.config=tesla-t4

  devicePlugin:
    config:
      default: any
      name: time-slicing-config-all
    enabled: true
    env:
    - name: PASS_DEVICE_SPECS
      value: "true"
    - name: FAIL_ON_INIT_ERROR
      value: "true"

kubectl patch clusterpolicies.nvidia.com/cluster-policy -n gpu-operator --type json -p '[{"op": "remove", "path": "/spec/devicePlugin/config"}]'

kubectl rollout restart -n gpu-operator daemonset/nvidia-device-plugin-daemonset

kubectl get node xxx -oyaml
  addresses:
  - address: 172.18.187.224
    type: InternalIP
  - address: izj6c5dnq07p1ic04ei9vwz
    type: Hostname
  allocatable:
    cpu: "4"
    ephemeral-storage: "189889991571"
    hugepages-1Gi: "0"
    hugepages-2Mi: "0"
    memory: 15246720Ki
    nvidia.com/gpu: "1"
    pods: "110"

// api/config/v1/config.go#L32
// Config is a versioned struct used to hold configuration information.
type Config struct {
	Version   string    `json:"version"             yaml:"version"`
	Flags     Flags     `json:"flags,omitempty"     yaml:"flags,omitempty"`
	Resources Resources `json:"resources,omitempty" yaml:"resources,omitempty"`
	Sharing   Sharing   `json:"sharing,omitempty"   yaml:"sharing,omitempty"`
}

apiVersion: v1
kind: ConfigMap
metadata:
  name: time-slicing-config-all
data:
  any: |-
    version: v1
    flags:
      migStrategy: none
    sharing:
      timeSlicing:
        renameByDefault: false
        failRequestsGreaterThanOne: false
        resources:
          - name: nvidia.com/gpu
            replicas: 4    

        resources:
          - name: nvidia.com/gpu
            replicas: 4

// internal/rm/device_map.go#L282

// updateDeviceMapWithReplicas returns an updated map of resource names to devices with replica
// information from the active replicated resources config.
func updateDeviceMapWithReplicas(replicatedResources *spec.ReplicatedResources, oDevices DeviceMap) (DeviceMap, error) {
	devices := make(DeviceMap)

	// Begin by walking replicatedResources.Resources and building a map of just the resource names.
	names := make(map[spec.ResourceName]bool)
	for _, r := range replicatedResources.Resources {
		names[r.Name] = true
	}

	// Copy over all devices from oDevices without a resource reference in TimeSlicing.Resources.
	for r, ds := range oDevices {
		if !names[r] {
			devices[r] = ds
		}
	}

	// Walk shared Resources and update devices in the device map as appropriate.
	for _, resource := range replicatedResources.Resources {
		r := resource
		// Get the IDs of the devices we want to replicate from oDevices
		ids, err := oDevices.getIDsOfDevicesToReplicate(&r)
		if err != nil {
			return nil, fmt.Errorf("unable to get IDs of devices to replicate for '%v' resource: %v", r.Name, err)
		}
		// Skip any resources not matched in oDevices
		if len(ids) == 0 {
			continue
		}

		// Add any devices we don't want replicated directly into the device map.
		for _, d := range oDevices[r.Name].Difference(oDevices[r.Name].Subset(ids)) {
			devices.insert(r.Name, d)
		}

		// Create replicated devices add them to the device map.
		// Rename the resource for replicated devices as requested.
		name := r.Name
		if r.Rename != "" {
			name = r.Rename
		}
		for _, id := range ids {
			for i := 0; i < r.Replicas; i++ {
				annotatedID := string(NewAnnotatedID(id, i))
				replicatedDevice := *(oDevices[r.Name][id])
				replicatedDevice.ID = annotatedID
				replicatedDevice.Replicas = r.Replicas
				devices.insert(name, &replicatedDevice)
			}
		}
	}

	return devices, nil
}

for _, id := range ids {
  for i := 0; i < r.Replicas; i++ {
    annotatedID := string(NewAnnotatedID(id, i))
    replicatedDevice := *(oDevices[r.Name][id])
    replicatedDevice.ID = annotatedID
    replicatedDevice.Replicas = r.Replicas
    devices.insert(name, &replicatedDevice)
  }
}

// NewAnnotatedID creates a new AnnotatedID from an ID and a replica number.
func NewAnnotatedID(id string, replica int) AnnotatedID {
	return AnnotatedID(fmt.Sprintf("%s::%d", id, replica))
}

// Split splits a AnnotatedID into its ID and replica number parts.
func (r AnnotatedID) Split() (string, int) {
	split := strings.SplitN(string(r), "::", 2)
	if len(split) != 2 {
		return string(r), 0
	}
	replica, _ := strconv.ParseInt(split[1], 10, 0)
	return split[0], int(replica)
}