Your A100s Show 40GB Free, But Training Jobs Won't Schedule. Here's Why.

A100 40GB MIG Profiles

Profile          GPU Slices    Memory    Max Instances per GPU
─────────────────────────────────────────────────────────────
1g.5gb           1/7           5GB       7
2g.10gb          2/7           10GB      3
3g.20gb          3/7           20GB      2
4g.20gb          4/7           20GB      1
7g.40gb          7/7           40GB      1

Step 1: Create GKE cluster with A100 GPUs

# Create a GKE Standard cluster with A100 GPUs
# Do NOT use --gpu-partition-size (we want dynamic MIG via GPU Operator)
gcloud container clusters create mig-demo-cluster \
  --zone us-central1-a \
  --machine-type a2-highgpu-1g \
  --accelerator type=nvidia-tesla-a100,count=1 \
  --num-nodes=1 \
  --release-channel rapid

Step 2: Create PriorityClass for GPU Operator

# GKE has ResourceQuota that blocks system-critical PriorityClasses
# Create a custom PriorityClass for GPU Operator components
cat <<EOF | kubectl apply -f -
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: nvidia-gpu-operator
value: 100000
globalDefault: false
description: "Priority for NVIDIA GPU Operator components"
EOF

Step 3: Install NVIDIA GPU Operator with GKE-specific settings

# Add NVIDIA Helm repo
helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
helm repo update

# Install GPU Operator with GKE-specific configuration
# Key settings:
#   - driver.enabled=false: Use GKE's pre-installed NVIDIA drivers
#   - toolkit.installDir: Use exec-capable path (COS mounts /var with noexec)
#   - hostPaths.driverInstallDir: Where GKE installs NVIDIA drivers
#   - Custom PriorityClass to avoid GKE ResourceQuota restrictions
helm install gpu-operator nvidia/gpu-operator \
  --namespace gpu-operator \
  --create-namespace \
  --set driver.enabled=false \
  --set toolkit.installDir=/home/kubernetes/bin/nvidia-toolkit \
  --set hostPaths.driverInstallDir=/home/kubernetes/bin/nvidia \
  --set mig.strategy=mixed \
  --set migManager.enabled=true \
  --set migManager.env[0].name=MIG_PARTED_REBOOT_IF_REQUIRED \
  --set migManager.env[0].value=true \
  --set migManager.env[1].name=WITH_REBOOT \
  --set migManager.env[1].value=true \
  --set dcgmExporter.enabled=false \
  --set daemonsets.priorityClassName=nvidia-gpu-operator \
  --set operator.priorityClassName=nvidia-gpu-operator \
  --set node-feature-discovery.priorityClassName=nvidia-gpu-operator

Step 4: Enable MIG mode on the GPU node

# Wait for GPU Operator pods to be running
kubectl get pods -n gpu-operator -w

# Once nvidia-mig-manager is running, apply MIG configuration
# This creates 7x 1g.5gb MIG instances on the A100
NODE=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
kubectl label node $NODE nvidia.com/mig.config=all-1g.5gb --overwrite

# The MIG Manager will reboot the node to enable MIG mode
# Wait for the node to come back up (2-3 minutes)
kubectl get nodes -w

# Verify MIG is configured
kubectl get node $NODE -o jsonpath='{.metadata.labels}' | grep mig.config.state
# Should show: "nvidia.com/mig.config.state":"success"

# Watch GPU Operator pods come up
kubectl get pods -n gpu-operator -w

# You should see these pods running:
# - gpu-operator (controller)
# - nvidia-container-toolkit-daemonset
# - nvidia-device-plugin-daemonset
# - nvidia-mig-manager
# - gpu-feature-discovery
# - nvidia-dcgm-exporter
# - nvidia-operator-validator

# Check all GPU Operator pods are running
kubectl get pods -n gpu-operator

# Verify MIG manager is active and configured for mixed strategy
kubectl get nodes -o jsonpath='{.items[0].metadata.labels}' | jq . | grep mig

# Check node GPU capacity (will show nvidia.com/gpu: 1 initially)
kubectl get nodes -o json | jq '.items[].status.allocatable | with_entries(select(.key | contains("nvidia")))'

values.yaml - Demo Configuration

# MIG Fragmentation Demo Configuration
# This creates a fragmentation scenario on A100 GPUs

namespace: mig-demo

# Phase 1: Deploy inference workloads that consume small MIG slices
inference:
  enabled: true
  replicas: 4
  resources:
    requests:
      nvidia.com/mig-1g.5gb: 1  # Small slices scattered across GPUs
  image: nvcr.io/nvidia/pytorch:24.01-py3
  command: ["python", "-c", "import torch; torch.cuda.is_available(); import time; time.sleep(86400)"]

# Phase 2: Deploy a training workload that needs a medium MIG slice
# This will get stuck due to fragmentation
training:
  enabled: true
  replicas: 1
  resources:
    requests:
      nvidia.com/mig-2g.10gb: 1  # Medium slice - will fail to schedule
  image: nvcr.io/nvidia/pytorch:24.01-py3
  command: ["python", "-c", "print('Training started'); import time; time.sleep(86400)"]

# Phase 3: Deploy more inference to maximize fragmentation
fragmenter:
  enabled: true
  replicas: 2
  resources:
    requests:
      nvidia.com/mig-1g.5gb: 1
  image: nvcr.io/nvidia/pytorch:24.01-py3
  command: ["sleep", "infinity"]

templates/inference-deployment.yaml

{{- if .Values.inference.enabled }}
apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ .Release.Name }}-inference
  namespace: {{ .Values.namespace }}
  labels:
    app: mig-inference
    demo: mig-fragmentation
spec:
  replicas: {{ .Values.inference.replicas }}
  selector:
    matchLabels:
      app: mig-inference
  template:
    metadata:
      labels:
        app: mig-inference
        workload-type: inference
        priority: low
    spec:
      # Use pod anti-affinity to spread inference pods across GPUs
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchLabels:
                  app: mig-inference
              topologyKey: "nvidia.com/gpu"
      containers:
      - name: inference
        image: {{ .Values.inference.image }}
        command: 
        {{- range .Values.inference.command }}
        - {{ . | quote }}
        {{- end }}
        resources:
          requests:
            nvidia.com/mig-1g.5gb: {{ .Values.inference.resources.requests | dig "nvidia.com/mig-1g.5gb" 1 }}
          limits:
            nvidia.com/mig-1g.5gb: {{ .Values.inference.resources.requests | dig "nvidia.com/mig-1g.5gb" 1 }}
      tolerations:
      - key: "nvidia.com/gpu"
        operator: "Exists"
        effect: "NoSchedule"
{{- end }}

templates/training-job.yaml

{{- if .Values.training.enabled }}
apiVersion: batch/v1
kind: Job
metadata:
  name: {{ .Release.Name }}-training
  namespace: {{ .Values.namespace }}
  labels:
    app: mig-training
    demo: mig-fragmentation
spec:
  backoffLimit: 0  # Don't retry - we want to see the scheduling failure
  template:
    metadata:
      labels:
        app: mig-training
        workload-type: training
        priority: high
    spec:
      restartPolicy: Never
      containers:
      - name: training
        image: {{ .Values.training.image }}
        command:
        {{- range .Values.training.command }}
        - {{ . | quote }}
        {{- end }}
        resources:
          requests:
            # This MIG profile requires 2 contiguous GPU slices
            # Fragmentation from 1g.5gb pods will prevent scheduling
            nvidia.com/mig-2g.10gb: {{ .Values.training.resources.requests | dig "nvidia.com/mig-2g.10gb" 1 }}
          limits:
            nvidia.com/mig-2g.10gb: {{ .Values.training.resources.requests | dig "nvidia.com/mig-2g.10gb" 1 }}
      tolerations:
      - key: "nvidia.com/gpu"
        operator: "Exists"
        effect: "NoSchedule"
{{- end }}

Evict inference pods to free contiguous slices

# Evict 2 inference pods to free contiguous slices
kubectl get pods -n mig-demo -l app=mig-inference -o name | head -2 | xargs kubectl delete -n mig-demo

# Watch the training job transition to Running (30-60 seconds)
kubectl get pods -n mig-demo -w

priorityclass.yaml - Enable automatic preemption

# High priority for training jobs
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: gpu-training-high
value: 1000000
preemptionPolicy: PreemptLowerPriority
globalDefault: false
description: "High priority for GPU training jobs - can preempt inference"
---
# Low priority for inference pods
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: gpu-inference-low
value: 100000
preemptionPolicy: Never
globalDefault: false
description: "Low priority for inference - can be preempted by training"

MIG reconfiguration steps

# Step 1: Drain the node (evicts ALL pods)
kubectl drain gpu-node-1 --ignore-daemonsets --delete-emptydir-data

# Step 2: SSH to node and reconfigure MIG (or use GPU Operator config)
nvidia-smi mig -dgi  # Destroy all GPU instances
nvidia-smi mig -dgci # Destroy all compute instances

# Step 3: Create a balanced MIG configuration
nvidia-smi mig -cgi 9,14,19 -C  # Create 3g.20gb + 2g.10gb + 2g.10gb

# Step 4: Uncordon the node
kubectl uncordon gpu-node-1