Zero-Downtime Deployments: Architecture & Implementation

1. RollingUpdate Strategy    → Controls how pods are replaced
2. Readiness Probes          → Prevents traffic to unready pods
3. preStop + terminationGracePeriodSeconds → Handles in-flight requests
4. PodDisruptionBudgets      → Protects against simultaneous eviction

apiVersion: apps/v1
kind: Deployment
metadata:
  name: payment-api
  namespace: production
spec:
  replicas: 10
  # Pillar 1: Rolling update strategy
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 2          # Allow 2 extra pods above desired (surge capacity)
      maxUnavailable: 0    # NEVER reduce below desired count (critical for payment API)
      # With 10 replicas: at any point you have 10-12 pods
      # Old pods are only terminated AFTER new pods pass readiness checks
  selector:
    matchLabels:
      app: payment-api
  template:
    metadata:
      labels:
        app: payment-api
    spec:
      # Pillar 3a: Give pods enough time to finish in-flight requests
      terminationGracePeriodSeconds: 60

      containers:
        - name: payment-api
          image: registry.mycompany.com/payment-api:v2.1.0
          ports:
            - containerPort: 8080

          # Pillar 2a: Readiness probe — traffic only sent when ready
          readinessProbe:
            httpGet:
              path: /health/ready    # Must return 200 only when truly ready
              port: 8080
            initialDelaySeconds: 10  # Wait 10s before first check
            periodSeconds: 5         # Check every 5s
            failureThreshold: 3      # Remove from Service after 3 failures
            successThreshold: 1      # Re-add after 1 success

          # Pillar 2b: Liveness probe — restart unhealthy pods
          livenessProbe:
            httpGet:
              path: /health/live
              port: 8080
            initialDelaySeconds: 30
            periodSeconds: 10
            failureThreshold: 3

          # Pillar 2c: Startup probe — for slow-starting apps
          startupProbe:
            httpGet:
              path: /health/startup
              port: 8080
            failureThreshold: 30     # Allow up to 5 minutes to start
            periodSeconds: 10

          # Pillar 3b: preStop hook — drain connections before SIGTERM
          lifecycle:
            preStop:
              exec:
                command:
                  - /bin/sh
                  - -c
                  # Sleep gives kube-proxy time to remove pod from iptables rules
                  # BEFORE the app starts shutting down
                  - "sleep 5 && /app/graceful-shutdown.sh"

          resources:
            requests:
              cpu: "500m"
              memory: "512Mi"
            limits:
              cpu: "1000m"
              memory: "1Gi"
---
# Pillar 4: PodDisruptionBudget — protect against simultaneous eviction
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: payment-api-pdb
  namespace: production
spec:
  # Always keep at least 90% of pods running
  minAvailable: "90%"
  # Alternatively, use maxUnavailable:
  # maxUnavailable: 1
  selector:
    matchLabels:
      app: payment-api

Timeline of pod termination (WITHOUT sleep):

t=0:  Pod receives SIGTERM
t=0:  kube-proxy starts updating iptables rules (async, takes 1-5s)
t=0:  App begins shutdown immediately
t=2:  App stops accepting connections
t=3:  kube-proxy STILL routing new requests to this pod  ← DROPPED REQUESTS

Timeline WITH preStop sleep:

t=0:  Pod receives SIGTERM
t=0:  preStop hook runs: sleep 5
t=0:  kube-proxy starts updating iptables rules
t=5:  kube-proxy finishes updating rules (no more new requests arrive)
t=5:  App receives SIGTERM (after preStop completes)
t=5:  App handles remaining in-flight requests
t=65: terminationGracePeriodSeconds expires, SIGKILL sent if still running
      (5s preStop + 60s termination = 65s total window)

// main.go — Graceful HTTP server shutdown
func main() {
    srv := &http.Server{Addr: ":8080", Handler: router}

    // Channel to listen for OS signals
    quit := make(chan os.Signal, 1)
    signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT)

    go func() {
        if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
            log.Fatal(err)
        }
    }()

    // Block until signal received
    <-quit
    log.Println("Shutdown signal received, draining connections...")

    // Give in-flight requests up to 55 seconds to complete
    ctx, cancel := context.WithTimeout(context.Background(), 55*time.Second)
    defer cancel()

    if err := srv.Shutdown(ctx); err != nil {
        log.Fatal("Server forced to shutdown:", err)
    }
    log.Println("Server exited cleanly")
}

// health.go — Readiness check that reflects true app state
var isReady atomic.Bool

func readinessHandler(w http.ResponseWriter, r *http.Request) {
    // Check database connection pool
    if err := db.PingContext(r.Context()); err != nil {
        http.Error(w, "DB not ready", http.StatusServiceUnavailable)
        return
    }

    // Check downstream service dependencies
    if !paymentGateway.IsHealthy() {
        http.Error(w, "Payment gateway not ready", http.StatusServiceUnavailable)
        return
    }

    // Check internal state (warming up, loading caches, etc.)
    if !isReady.Load() {
        http.Error(w, "App still initializing", http.StatusServiceUnavailable)
        return
    }

    w.WriteHeader(http.StatusOK)
    w.Write([]byte("OK"))
}