Cluster Nodes:
┌─────────────────────────────────────────────────────────────┐
│  GPU Nodes (4x)              High-Memory Nodes (6x)         │
│  label: node-type=gpu         label: node-type=high-memory  │
│  taint: gpu=true:NoSchedule  taint: (none)                  │
│  GPU: 8x A100                RAM: 512Gi                     │
│  AZ: us-east-1a/1b           AZ: us-east-1a/1b/1c           │
│                                                             │
│  Standard Nodes (20x)                                       │
│  label: node-type=standard                                  │
│  taint: (none)                                              │
│  AZ: us-east-1a/1b/1c (balanced)                           │
└─────────────────────────────────────────────────────────────┘

# Label GPU nodes
for node in gpu-node-01 gpu-node-02 gpu-node-03 gpu-node-04; do
  kubectl label node $node \
    node-type=gpu \
    accelerator=nvidia-a100 \
    topology.kubernetes.io/zone=us-east-1a

  # Taint: Only ML workloads with toleration can schedule here
  kubectl taint node $node \
    gpu=true:NoSchedule       # NoSchedule: new pods rejected without toleration
                              # NoExecute: existing pods also evicted
                              # PreferNoSchedule: soft version of NoSchedule
done

# Label high-memory nodes
for node in highmem-node-01 highmem-node-02 highmem-node-03; do
  kubectl label node $node \
    node-type=high-memory \
    memory-tier=xlarge
done

# Label standard nodes with AZ distribution
kubectl label node standard-node-01 node-type=standard topology.kubernetes.io/zone=us-east-1a
kubectl label node standard-node-02 node-type=standard topology.kubernetes.io/zone=us-east-1a
kubectl label node standard-node-03 node-type=standard topology.kubernetes.io/zone=us-east-1b
kubectl label node standard-node-04 node-type=standard topology.kubernetes.io/zone=us-east-1b
kubectl label node standard-node-05 node-type=standard topology.kubernetes.io/zone=us-east-1c
kubectl label node standard-node-06 node-type=standard topology.kubernetes.io/zone=us-east-1c
# ... continue for all 20 standard nodes

# Verify node labels and taints
kubectl get nodes -o custom-columns=\
"NAME:.metadata.name,\
TYPE:.metadata.labels.node-type,\
ZONE:.metadata.labels.topology\.kubernetes\.io/zone,\
TAINTS:.spec.taints[*].key"

# ml-training-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
  name: ml-training-resnet50
  namespace: ml-platform
spec:
  parallelism: 4              # Run on all 4 GPU nodes simultaneously
  completions: 4
  template:
    spec:
      # REQUIRED: Tolerate the GPU taint to schedule on GPU nodes
      tolerations:
        - key: "gpu"
          operator: "Equal"
          value: "true"
          effect: "NoSchedule"

      # REQUIRED: Target GPU nodes specifically
      affinity:
        nodeAffinity:
          # Hard requirement: MUST be on GPU node
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: node-type
                    operator: In
                    values: ["gpu"]
                  - key: accelerator
                    operator: In
                    values: ["nvidia-a100"]    # Specific GPU model required

          # Soft preference: prefer nodes with fewer running jobs
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 50
              preference:
                matchExpressions:
                  - key: workload-count
                    operator: Lt
                    values: ["2"]

        # Spread across different GPU nodes (one job per node)
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchLabels:
                  job-name: ml-training-resnet50
              topologyKey: kubernetes.io/hostname  # One pod per node

      containers:
        - name: training
          image: registry.mycompany.com/ml-trainer:latest
          command: ["python", "train.py", "--model=resnet50", "--epochs=100"]

          # Request GPU resources
          resources:
            requests:
              cpu: "8"
              memory: "64Gi"
              nvidia.com/gpu: "1"     # Request 1 GPU per pod
            limits:
              cpu: "16"
              memory: "128Gi"
              nvidia.com/gpu: "1"     # GPU limits must equal requests

          env:
            - name: CUDA_VISIBLE_DEVICES
              value: "0"
            - name: NCCL_DEBUG
              value: "INFO"

      restartPolicy: OnFailure

      # Give ML jobs priority over regular workloads
      priorityClassName: ml-high-priority

# analytics-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: analytics-engine
  namespace: analytics
spec:
  replicas: 3
  selector:
    matchLabels:
      app: analytics-engine
  template:
    metadata:
      labels:
        app: analytics-engine
    spec:
      affinity:
        nodeAffinity:
          # Soft preference: prefer high-memory nodes
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100           # Highest possible weight — strongly prefer this
              preference:
                matchExpressions:
                  - key: node-type
                    operator: In
                    values: ["high-memory"]
            - weight: 50            # Fallback: standard nodes are acceptable
              preference:
                matchExpressions:
                  - key: node-type
                    operator: In
                    values: ["standard"]

          # Hard requirement: NEVER schedule on GPU nodes
          # (GPU nodes are expensive and reserved for ML)
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: node-type
                    operator: NotIn
                    values: ["gpu"]     # Exclude GPU nodes

        # Spread analytics pods across nodes
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                labelSelector:
                  matchLabels:
                    app: analytics-engine
                topologyKey: kubernetes.io/hostname

      containers:
        - name: analytics
          image: registry.mycompany.com/analytics:latest
          resources:
            requests:
              cpu: "4"
              memory: "128Gi"       # High memory workload
            limits:
              cpu: "8"
              memory: "256Gi"

# frontend-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-frontend
  namespace: production
spec:
  replicas: 12           # 4 per AZ (3 AZs × 4 pods)
  selector:
    matchLabels:
      app: web-frontend
  template:
    metadata:
      labels:
        app: web-frontend
    spec:
      # Topology Spread Constraints (Kubernetes 1.19+)
      # More expressive than podAntiAffinity for spreading
      topologySpreadConstraints:
        # Constraint 1: Spread evenly across AZs
        - maxSkew: 1                  # Max difference in pod count between AZs
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: DoNotSchedule  # Hard constraint
          labelSelector:
            matchLabels:
              app: web-frontend
          # With 12 replicas and 3 AZs: 4 pods per AZ
          # maxSkew: 1 means AZs can have 4 or 5 pods, but not 3 and 6

        # Constraint 2: Spread across nodes within each AZ
        - maxSkew: 1
          topologyKey: kubernetes.io/hostname
          whenUnsatisfiable: ScheduleAnyway   # Soft constraint
          labelSelector:
            matchLabels:
              app: web-frontend

      affinity:
        # Hard requirement: only standard nodes for frontend
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: node-type
                    operator: In
                    values: ["standard"]

      containers:
        - name: frontend
          image: registry.mycompany.com/frontend:latest
          resources:
            requests:
              cpu: "500m"
              memory: "512Mi"
            limits:
              cpu: "1000m"
              memory: "1Gi"

# priority-classes.yaml

# Highest priority: ML training (preempts everything else if needed)
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: ml-high-priority
value: 1000000
globalDefault: false
preemptionPolicy: PreemptLowerPriority
description: "ML training jobs — highest priority, can preempt other workloads"
---
# High priority: Production workloads
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: production-high
value: 100000
globalDefault: false
preemptionPolicy: PreemptLowerPriority
description: "Production web services"
---
# Default priority: Standard workloads
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: default-priority
value: 1000
globalDefault: true
description: "Default priority for all workloads"
---
# Low priority: Batch jobs and background tasks
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: batch-low
value: 100
globalDefault: false
preemptionPolicy: Never           # Cannot preempt others
description: "Batch jobs — lowest priority, never preempt"

# descheduler-policy.yaml
# The descheduler runs periodically and evicts pods that violate
# scheduling constraints (e.g., due to cluster changes after initial scheduling)

apiVersion: "descheduler/v1alpha1"
kind: "DeschedulerPolicy"
strategies:
  # Remove pods that violated their node affinity after node label changes
  RemovePodsViolatingNodeAffinity:
    enabled: true
    params:
      nodeAffinityType:
        - requiredDuringSchedulingIgnoredDuringExecution

  # Rebalance pods that are violating topology spread constraints
  RemovePodsViolatingTopologySpreadConstraint:
    enabled: true
    params:
      constraints:
        - DoNotSchedule

  # Remove duplicate pods on the same node (fix anti-affinity violations)
  RemoveDuplicates:
    enabled: true

  # Evict pods from over-utilized nodes
  LowNodeUtilization:
    enabled: true
    params:
      nodeResourceUtilizationThresholds:
        thresholds:
          cpu: 20          # Node is underutilized if CPU < 20%
          memory: 20
          pods: 20
        targetThresholds:
          cpu: 50          # Target utilization after descheduling
          memory: 50
          pods: 50

# Deploy descheduler as a CronJob
helm repo add descheduler https://kubernetes-sigs.github.io/descheduler/
helm install descheduler descheduler/descheduler \
  --namespace kube-system \
  --set schedule="*/30 * * * *"   # Run every 30 minutes

# Check where pods are scheduled
kubectl get pods -n ml-platform -o wide
# Verify all ml-training pods are on GPU nodes

kubectl get pods -n production -o wide | grep web-frontend
# Verify pods are spread across AZs

# Check why a pod is not scheduling (Pending state)
kubectl describe pod <pending-pod> | grep -A 20 "Events:"
# Common messages:
# "0/26 nodes are available: 4 node(s) had taint {gpu: true}..."
# "0/26 nodes are available: 6 node(s) didn't match node affinity..."
# "0/26 nodes are available: topology.kubernetes.io/zone, maxSkew=1..."

# Simulate scheduling without actually creating pods
kubectl apply --dry-run=server -f ml-training-job.yaml

# Use kube-scheduler simulator for complex scenarios
# https://github.com/kubernetes-sigs/kube-scheduler-simulator