diff --git a/roles/nvidia_device_plugin/README.md b/roles/nvidia_device_plugin/README.md new file mode 100644 index 0000000..0859155 --- /dev/null +++ b/roles/nvidia_device_plugin/README.md @@ -0,0 +1,63 @@ +# NVIDIA Device Plugin + +Installs NVIDIA Kubernetes Device Plugin via Helm with GPU Feature Discovery support. + +## Requirements + +- Kubernetes cluster with NVIDIA GPU workers +- NVIDIA drivers installed on GPU nodes +- nvidia-container-toolkit configured + +## Role Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `nvidia_device_plugin_chart_version` | `0.17.1` | NVIDIA device plugin Helm chart version | +| `nvidia_device_plugin_namespace` | `nvidia-device-plugin` | Kubernetes namespace | +| `nvidia_device_plugin_config` | `{}` | ConfigMap configuration | +| `nvidia_device_plugin_compat_with_cpu_manager` | `false` | Enable CPUManager compatibility | +| `nvidia_device_plugin_mig_strategy` | `none` | MIG strategy: `none`, `single`, `mixed` | +| `nvidia_device_plugin_fail_on_init_error` | `true` | Fail on init error | +| `nvidia_device_plugin_device_list_strategy` | `envvar` | Device list strategy | +| `nvidia_device_plugin_device_id_strategy` | `uuid` | Device ID strategy | +| `nvidia_device_plugin_nvidia_driver_root` | `/` | NVIDIA driver root path | +| `nvidia_device_plugin_gfd.enabled` | `true` | Deploy GPU Feature Discovery | +| `nvidia_device_plugin_nfd` | see defaults | Node Feature Discovery config | + +## Example Playbook + +```yaml +- hosts: k8s_manager + become: false + roles: + - nvidia_device_plugin +``` + +## Configuration Examples + +### MIG Strategy + +```yaml +nvidia_device_plugin_mig_strategy: "mixed" +``` + +### ConfigMap + +```yaml +nvidia_device_plugin_config: + map: + default: | + version: v1 + flags: + migStrategy: none + mig-mixed: | + version: v1 + flags: + migStrategy: mixed + default: "default" +``` + +## References + +- [NVIDIA k8s-device-plugin GitHub](https://github.com/NVIDIA/k8s-device-plugin) +- [Helm Chart Values](https://github.com/NVIDIA/k8s-device-plugin/blob/v0.17.1/deployments/helm/nvidia-device-plugin/values.yaml) diff --git a/roles/nvidia_device_plugin/defaults/main.yml b/roles/nvidia_device_plugin/defaults/main.yml new file mode 100644 index 0000000..cbf2542 --- /dev/null +++ b/roles/nvidia_device_plugin/defaults/main.yml @@ -0,0 +1,97 @@ +--- +nvidia_device_plugin_chart_version: "0.17.1" +nvidia_device_plugin_namespace: nvidia-device-plugin + +nvidia_device_plugin_config: + map: {} + default: "" + name: "" + +nvidia_device_plugin_compat_with_cpu_manager: false +nvidia_device_plugin_mig_strategy: "none" +nvidia_device_plugin_fail_on_init_error: true +nvidia_device_plugin_device_list_strategy: "envvar" +nvidia_device_plugin_device_id_strategy: "uuid" +nvidia_device_plugin_nvidia_driver_root: "/" + +nvidia_device_plugin_image: + repository: nvcr.io/nvidia/k8s-device-plugin + tag: "" + pull_policy: IfNotPresent + +nvidia_device_plugin_resources: {} + +nvidia_device_plugin_node_selector: {} + +nvidia_device_plugin_affinity: + node_affinity: + required_during_scheduling_ignored_during_execution: + node_selector_terms: + - match_expressions: + - key: feature.node.kubernetes.io/pci-10de.present + operator: In + values: + - "true" + - match_expressions: + - key: feature.node.kubernetes.io/cpu-model.vendor_id + operator: In + values: + - "NVIDIA" + - match_expressions: + - key: nvidia.com/gpu.present + operator: In + values: + - "true" + +nvidia_device_plugin_tolerations: + - key: CriticalAddonsOnly + operator: Exists + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + +nvidia_device_plugin_priority_class_name: "system-node-critical" + +nvidia_device_plugin_runtime_class_name: null + +nvidia_device_plugin_gfd: + enabled: true + name_override: gpu-feature-discovery + no_timestamp: null + sleep_interval: null + security_context: + privileged: true + +nvidia_device_plugin_nfd: + name_override: node-feature-discovery + enable_node_feature_api: false + master: + service_account: + name: node-feature-discovery + create: true + config: + extra_label_ns: + - nvidia.com + worker: + tolerations: + - key: node-role.kubernetes.io/master + operator: Equal + value: "" + effect: NoSchedule + - key: nvidia.com/gpu + operator: Equal + value: present + effect: NoSchedule + config: + sources: + pci: + device_class_whitelist: + - "02" + - "03" + device_label_fields: + - vendor + +nvidia_device_plugin_mps: + root: "/run/nvidia/mps" + +nvidia_device_plugin_kubeconfig: "/home/{{ ansible_user }}/.kube/config" diff --git a/roles/nvidia_device_plugin/meta/main.yml b/roles/nvidia_device_plugin/meta/main.yml new file mode 100644 index 0000000..b6745e6 --- /dev/null +++ b/roles/nvidia_device_plugin/meta/main.yml @@ -0,0 +1,12 @@ +--- +galaxy_info: + role_name: nvidia_device_plugin + author: ops + description: Installs NVIDIA Kubernetes Device Plugin via Helm with GPU Feature Discovery support + license: MIT + min_ansible_version: "2.14" + platforms: + - name: EL + versions: + - "9" + dependencies: [] diff --git a/roles/nvidia_device_plugin/tasks/deploy.yml b/roles/nvidia_device_plugin/tasks/deploy.yml new file mode 100644 index 0000000..70c4c22 --- /dev/null +++ b/roles/nvidia_device_plugin/tasks/deploy.yml @@ -0,0 +1,36 @@ +--- +- name: Helm | Add NVIDIA device plugin chart repository + ansible.builtin.command: helm repo add nvdp https://nvidia.github.io/k8s-device-plugin + environment: + KUBECONFIG: "{{ nvidia_device_plugin_kubeconfig }}" + PATH: "/usr/local/bin:/usr/bin:/bin" + register: _helm_repo_add + changed_when: "'already exists' not in _helm_repo_add.stdout" + failed_when: _helm_repo_add.rc != 0 and 'already exists' not in _helm_repo_add.stdout + +- name: Helm | Update chart repositories + ansible.builtin.command: helm repo update + environment: + KUBECONFIG: "{{ nvidia_device_plugin_kubeconfig }}" + PATH: "/usr/local/bin:/usr/bin:/bin" + changed_when: false + +- name: NVIDIA Device Plugin | Render values + ansible.builtin.template: + src: values.yml.j2 + dest: /tmp/nvidia-device-plugin-values.yml + mode: "0600" + +- name: NVIDIA Device Plugin | Install or upgrade + ansible.builtin.command: > + helm upgrade --install nvidia-device-plugin nvdp/nvidia-device-plugin + --namespace {{ nvidia_device_plugin_namespace }} + --version {{ nvidia_device_plugin_chart_version }} + --values /tmp/nvidia-device-plugin-values.yml + --timeout 15m0s + --wait + environment: + KUBECONFIG: "{{ nvidia_device_plugin_kubeconfig }}" + PATH: "/usr/local/bin:/usr/bin:/bin" + register: _helm_install + changed_when: "'STATUS: deployed' in _helm_install.stdout or 'has been upgraded' in _helm_install.stdout" diff --git a/roles/nvidia_device_plugin/tasks/gfd.yml b/roles/nvidia_device_plugin/tasks/gfd.yml new file mode 100644 index 0000000..e01f276 --- /dev/null +++ b/roles/nvidia_device_plugin/tasks/gfd.yml @@ -0,0 +1,16 @@ +--- +- name: GPU Feature Discovery | Check if already deployed + ansible.builtin.kubectl: + name: "gpu-feature-discovery" + kind: "DaemonSet" + namespace: "{{ nvidia_device_plugin_namespace }}" + state: "list" + environment: + KUBECONFIG: "{{ nvidia_device_plugin_kubeconfig }}" + PATH: "/usr/local/bin:/usr/bin:/bin" + register: _gfd_check + changed_when: false + +- name: GPU Feature Discovery | Deploy via Helm (if enabled in values) + ansible.builtin.include_tasks: deploy.yml + when: nvidia_device_plugin_gfd.enabled and (_gfd_check.stdout | length == 0) diff --git a/roles/nvidia_device_plugin/tasks/main.yml b/roles/nvidia_device_plugin/tasks/main.yml new file mode 100644 index 0000000..d88c363 --- /dev/null +++ b/roles/nvidia_device_plugin/tasks/main.yml @@ -0,0 +1,9 @@ +--- +- name: Create namespace + ansible.builtin.include_tasks: namespace.yml + +- name: Deploy NVIDIA Device Plugin via Helm + ansible.builtin.include_tasks: deploy.yml + +- name: Deploy GPU Feature Discovery + ansible.builtin.include_tasks: gfd.yml diff --git a/roles/nvidia_device_plugin/tasks/namespace.yml b/roles/nvidia_device_plugin/tasks/namespace.yml new file mode 100644 index 0000000..0b24e4f --- /dev/null +++ b/roles/nvidia_device_plugin/tasks/namespace.yml @@ -0,0 +1,10 @@ +--- +- name: Create namespace + ansible.builtin.kubectl: + name: "{{ nvidia_device_plugin_namespace }}" + api_version: "v1" + kind: "Namespace" + state: "present" + environment: + KUBECONFIG: "{{ nvidia_device_plugin_kubeconfig }}" + PATH: "/usr/local/bin:/usr/bin:/bin" diff --git a/roles/nvidia_device_plugin/templates/values.yml.j2 b/roles/nvidia_device_plugin/templates/values.yml.j2 new file mode 100644 index 0000000..9d93b1f --- /dev/null +++ b/roles/nvidia_device_plugin/templates/values.yml.j2 @@ -0,0 +1,128 @@ +{% if nvidia_device_plugin_config.name or nvidia_device_plugin_config.map %} +config: +{% if nvidia_device_plugin_config.name %} + name: "{{ nvidia_device_plugin_config.name }}" +{% endif %} +{% if nvidia_device_plugin_config.map %} + map: +{% for key, value in nvidia_device_plugin_config.map.items() %} + {{ key }}: | +{{ value | indent(6, false) }} +{% endfor %} +{% endif %} +{% if nvidia_device_plugin_config.default %} + default: "{{ nvidia_device_plugin_config.default }}" +{% endif %} +{% if nvidia_device_plugin_config.fallback_strategies %} + fallbackStrategies: {{ nvidia_device_plugin_config.fallback_strategies }} +{% endif %} +{% endif %} + +compatWithCPUManager: {{ nvidia_device_plugin_compat_with_cpu_manager | lower }} + +{% if nvidia_device_plugin_mig_strategy is defined and nvidia_device_plugin_mig_strategy %} +migStrategy: "{{ nvidia_device_plugin_mig_strategy }}" +{% endif %} + +{% if nvidia_device_plugin_fail_on_init_error is defined %} +failOnInitError: {{ nvidia_device_plugin_fail_on_init_error | lower }} +{% endif %} + +{% if nvidia_device_plugin_device_list_strategy is defined and nvidia_device_plugin_device_list_strategy %} +deviceListStrategy: "{{ nvidia_device_plugin_device_list_strategy }}" +{% endif %} + +{% if nvidia_device_plugin_device_id_strategy is defined and nvidia_device_plugin_device_id_strategy %} +deviceIDStrategy: "{{ nvidia_device_plugin_device_id_strategy }}" +{% endif %} + +{% if nvidia_device_plugin_nvidia_driver_root is defined and nvidia_device_plugin_nvidia_driver_root %} +nvidiaDriverRoot: "{{ nvidia_device_plugin_nvidia_driver_root }}" +{% endif %} + +image: + repository: {{ nvidia_device_plugin_image.repository }} + tag: "{{ nvidia_device_plugin_image.tag }}" + pullPolicy: {{ nvidia_device_plugin_image.pull_policy }} + +resources: {{ nvidia_device_plugin_resources | default({}) | to_nice_yaml(indent=2) | indent(2) }} + +nodeSelector: {{ nvidia_device_plugin_node_selector | default({}) | to_nice_yaml(indent=2) | indent(2) }} + +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: +{% for term in nvidia_device_plugin_affinity.node_affinity.required_during_scheduling_ignored_during_execution.node_selector_terms %} + - matchExpressions: +{% for expr in term.match_expressions %} + - key: {{ expr.key }} + operator: {{ expr.operator }} + values: +{% for val in expr.values %} + - {{ val }} +{% endfor %} +{% endfor %} +{% endfor %} + +tolerations: {{ nvidia_device_plugin_tolerations | to_nice_yaml(indent=2) | indent(2) }} + +priorityClassName: "{{ nvidia_device_plugin_priority_class_name }}" + +{% if nvidia_device_plugin_runtime_class_name is defined and nvidia_device_plugin_runtime_class_name %} +runtimeClassName: "{{ nvidia_device_plugin_runtime_class_name }}" +{% endif %} + +devicePlugin: + enabled: true + +{% if nvidia_device_plugin_gfd.enabled %} +gfd: + enabled: true + nameOverride: "{{ nvidia_device_plugin_gfd.name_override }}" +{% if nvidia_device_plugin_gfd.no_timestamp is defined %} + noTimestamp: {{ nvidia_device_plugin_gfd.no_timestamp | lower }} +{% endif %} +{% if nvidia_device_plugin_gfd.sleep_interval is defined %} + sleepInterval: "{{ nvidia_device_plugin_gfd.sleep_interval }}" +{% endif %} + securityContext: + privileged: {{ nvidia_device_plugin_gfd.security_context.privileged | lower }} +{% endif %} + +{% if nvidia_device_plugin_nfd %} +nfd: + nameOverride: "{{ nvidia_device_plugin_nfd.name_override }}" + enableNodeFeatureApi: {{ nvidia_device_plugin_nfd.enable_node_feature_api | lower }} + master: + serviceAccount: + name: "{{ nvidia_device_plugin_nfd.master.service_account.name }}" + create: {{ nvidia_device_plugin_nfd.master.service_account.create | lower }} + config: + extraLabelNs: +{% for ns in nvidia_device_plugin_nfd.master.config.extra_label_ns %} + - {{ ns }} +{% endfor %} + worker: + tolerations: +{% for tol in nvidia_device_plugin_nfd.worker.tolerations %} + - key: "{{ tol.key }}" + operator: "{{ tol.operator }}" + value: "{{ tol.value | default('') }}" + effect: "{{ tol.effect }}" +{% endfor %} + config: + sources: + pci: + deviceClassWhitelist: +{% for cls in nvidia_device_plugin_nfd.worker.config.sources.pci.device_class_whitelist %} + - {{ cls }} +{% endfor %} + deviceLabelFields: +{% for field in nvidia_device_plugin_nfd.worker.config.sources.pci.device_label_fields %} + - {{ field }} +{% endfor %} +{% endif %} + +mps: + root: "{{ nvidia_device_plugin_mps.root }}"