How Do You Benchmark and Monitor etcd Health in Production?: Architecture & Implementation

# prometheus-etcd-alerts.yaml
groups:
  - name: etcd.critical
    interval: 30s
    rules:

      # Disk: WAL fsync too slow → writes will back up
      - alert: EtcdWALFsyncLatencyHigh
        expr: |
          histogram_quantile(0.99,
            rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])
          ) > 0.010
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "etcd WAL fsync p99 > 10ms on {{ $labels.instance }}"
          description: |
            Current value: {{ $value | humanizeDuration }}
            Cause: Disk I/O contention or slow underlying storage.
            Action: Check disk scheduler, iostat, and consider NVMe upgrade.

      # Raft: Leader change detected
      - alert: EtcdLeaderChange
        expr: |
          increase(etcd_server_leader_changes_seen_total[10m]) > 0
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "etcd leader change detected on {{ $labels.instance }}"
          description: |
            Leader changes cause write unavailability and API server reconnects.
            Check network between control plane nodes and disk latency.

      # Raft: Proposals failing (writes being rejected)
      - alert: EtcdHighNumberOfFailedProposals
        expr: |
          rate(etcd_server_proposals_failed_total[5m]) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "etcd proposals failing at {{ $value }}/s"
          description: "Quorum may be lost or members unreachable."

      # DB: Size approaching quota (default 2GB, you should set 6GB+)
      - alert: EtcdDatabaseSizeApproachingQuota
        expr: |
          (etcd_mvcc_db_total_size_in_bytes /
           etcd_server_quota_backend_bytes) > 0.80
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "etcd DB is {{ $value | humanizePercentage }} of quota"
          description: "Run compaction and defrag, or increase quota-backend-bytes."

      # DB: Quota exceeded → etcd is read-only
      - alert: EtcdDatabaseQuotaExceeded
        expr: etcd_server_quota_backend_bytes - etcd_mvcc_db_total_size_in_bytes < 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "etcd quota exceeded — cluster is READ ONLY"
          description: |
            Immediately run: etcdctl compact && etcdctl defrag
            Then raise --quota-backend-bytes in etcd config.

      # Network: Peer RTT too high → risk of spurious elections
      - alert: EtcdPeerNetworkLatencyHigh
        expr: |
          histogram_quantile(0.99,
            rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])
          ) > 0.1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "etcd peer RTT p99 > 100ms on {{ $labels.instance }}"

      # Member: etcd member down
      - alert: EtcdMemberDown
        expr: up{job="etcd"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "etcd member {{ $labels.instance }} is down"

# Panel 1: WAL fsync latency (p50, p99, p999)
histogram_quantile(0.99,
  sum by (le, instance) (
    rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])
  )
)

# Panel 2: DB backend commit latency
histogram_quantile(0.99,
  rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])
)

# Panel 3: DB size vs. quota (all members)
etcd_mvcc_db_total_size_in_bytes
etcd_server_quota_backend_bytes

# Panel 4: Proposal rate and failure rate
rate(etcd_server_proposals_committed_total[1m])
rate(etcd_server_proposals_failed_total[1m])
rate(etcd_server_proposals_pending[1m])

# Panel 5: Leader changes (should be near-zero)
increase(etcd_server_leader_changes_seen_total[1h])

# Panel 6: Active connections from kube-apiserver
etcd_grpc_server_started_total
etcd_grpc_server_handled_total

# Panel 7: Key space size (total stored keys)
etcd_debugging_mvcc_keys_total

# Panel 8: Compaction lag
etcd_mvcc_db_total_size_in_bytes - etcd_mvcc_db_total_size_in_use_in_bytes
# Large gap = fragmented space waiting for defrag

#!/bin/bash
# etcd-healthcheck.sh — run daily or include in on-call runbook

ETCDCTL="etcdctl \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/peer.crt \
  --key=/etc/kubernetes/pki/etcd/peer.key"

echo "=== etcd Cluster Health ==="
$ETCDCTL endpoint health --cluster

echo ""
echo "=== Member List ==="
$ETCDCTL member list --write-out=table

echo ""
echo "=== Endpoint Status ==="
$ETCDCTL endpoint status --cluster --write-out=table
# Key columns: DB SIZE, IS LEADER, RAFT TERM, RAFT INDEX
# All members should have same RAFT TERM and nearly same RAFT INDEX

echo ""
echo "=== DB Size vs Quota ==="
$ETCDCTL endpoint status --cluster --write-out=json | jq -r '
  .[] | {
    endpoint: .Endpoint,
    db_size_mb: (.Status.dbSize / 1024 / 1024 | round),
    db_size_in_use_mb: (.Status.dbSizeInUse / 1024 / 1024 | round),
    fragmented_mb: ((.Status.dbSize - .Status.dbSizeInUse) / 1024 / 1024 | round)
  }
'

echo ""
echo "=== WAL fsync latency (last 60s via metrics) ==="
curl -s http://127.0.0.1:2381/metrics | \
  grep etcd_disk_wal_fsync_duration_seconds | \
  grep -v "^#"

echo ""
echo "=== Recent Leader Changes ==="
curl -s http://127.0.0.1:2381/metrics | \
  grep etcd_server_leader_changes_seen_total