Debugging in a Highly Distributed Kubernetes Environment: Architecture & Implementation

Step 1: OBSERVE    → Gather signals without assumptions
Step 2: HYPOTHESIZE → Form specific, testable theories
Step 3: ISOLATE    → Narrow the blast radius
Step 4: CONFIRM    → Prove the root cause
Step 5: REMEDIATE  → Fix and verify
Step 6: HARDEN     → Prevent recurrence

# Get a high-level view of what's unhealthy RIGHT NOW
kubectl get pods -A | grep -v Running | grep -v Completed | grep -v Succeeded

# Check resource pressure across nodes
kubectl top nodes
kubectl top pods -n checkout --sort-by=cpu

# Check for recent events (often the first clue)
kubectl get events -n checkout \
  --sort-by='.lastTimestamp' \
  --field-selector type=Warning | tail -30

# Check pod restarts (OOMKills show up here)
kubectl get pods -n checkout -o json | \
  jq '.items[] | {
    name: .metadata.name,
    restarts: .status.containerStatuses[0].restartCount,
    reason: .status.containerStatuses[0].lastState.terminated.reason
  }' | grep -A3 '"restarts": [^0]'

# Assuming Jaeger is deployed (via Helm)
helm repo add jaegertracing https://jaegertracing.github.io/helm-charts
helm install jaeger jaegertracing/jaeger \
  --namespace observability \
  --set provisionDataStore.cassandra=false \
  --set allInOne.enabled=true \
  --set storage.type=memory   # Use Elasticsearch/Cassandra for production

# Instrument your service with OpenTelemetry (Python example)
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.instrumentation.requests import RequestsInstrumentor
from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor

# Configure tracer
provider = TracerProvider()
otlp_exporter = OTLPSpanExporter(
    endpoint="http://jaeger-collector.observability.svc.cluster.local:4317"
)
provider.add_span_processor(BatchSpanProcessor(otlp_exporter))
trace.set_tracer_provider(provider)

# Auto-instrument HTTP and DB calls
RequestsInstrumentor().instrument()
Psycopg2Instrumentor().instrument()

tracer = trace.get_tracer(__name__)

# Manual span for critical sections
def process_checkout(order_id: str):
    with tracer.start_as_current_span("process_checkout") as span:
        span.set_attribute("order.id", order_id)
        span.set_attribute("order.items", len(order.items))

        # This span will show if DNS resolution is slow
        with tracer.start_as_current_span("resolve_payment_service"):
            payment_url = resolve_service("payment-svc")

        with tracer.start_as_current_span("call_payment_gateway"):
            result = payment_client.charge(order)

        return result

# Query Jaeger for slow traces (using Jaeger API)
curl "http://jaeger.observability.svc.cluster.local:16686/api/traces?\
service=checkout-api&\
operation=process_checkout&\
minDuration=3000000&\
limit=20" | jq '.data[].spans[] | {
  operationName,
  duration: (.duration / 1000),
  tags: .tags
}' | sort -t: -k2 -rn | head -20

# Look for spans with duration > 4900ms (near the 5s timeout)
# The FIRST long span in the trace is your culprit

# Test DNS resolution speed from within a pod
kubectl run dns-test --image=tutum/dnsutils \
  --restart=Never --rm -it -- \
  bash -c "for i in {1..100}; do
    time nslookup payment-svc.production.svc.cluster.local;
  done 2>&1 | grep real"

# Check CoreDNS performance metrics
kubectl get pods -n kube-system -l k8s-app=kube-dns
kubectl top pods -n kube-system -l k8s-app=kube-dns

# Check CoreDNS logs for errors
kubectl logs -n kube-system -l k8s-app=kube-dns --tail=100 | \
  grep -E "SERVFAIL|timeout|i/o timeout"

# Check CoreDNS ConfigMap for misconfiguration
kubectl get configmap coredns -n kube-system -o yaml

# The most common DNS fix: tune ndots and add search domains
# Default /etc/resolv.conf in pods has ndots:5, causing 5 DNS lookups
# before falling back to the absolute name

# Fix 1: Tune dnsConfig in your deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: checkout-api
spec:
  template:
    spec:
      dnsConfig:
        options:
          - name: ndots
            value: "2"        # Reduce from default 5 to 2
                              # checkout-api.production → tries as FQDN first
          - name: single-request-reopen
            value: ""         # Fix for parallel A/AAAA query race condition
          - name: timeout
            value: "2"        # DNS timeout per attempt (default 5s)
          - name: attempts
            value: "3"        # Retry 3 times before failing
      dnsPolicy: ClusterFirst

# Fix 2: CoreDNS caching tuning
# Edit CoreDNS ConfigMap
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
data:
  Corefile: |
    .:53 {
        errors
        health {
           lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
           pods insecure
           fallthrough in-addr.arpa ip6.arpa
           ttl 30
        }
        prometheus :9153
        forward . /etc/resolv.conf {
           max_concurrent 1000
        }
        cache 30 {           # Cache DNS responses for 30 seconds
            success 9984     # Max successful cache entries
            denial 9984      # Max NXDOMAIN cache entries
            prefetch 10      # Prefetch popular entries before expiry
        }
        loop
        reload
        loadbalance
    }

# Fix 3: NodeLocal DNSCache (most impactful for high-traffic clusters)
# Runs a DNS cache DaemonSet on every node, eliminates conntrack issues
kubectl apply -f https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml

# NodeLocal DNSCache intercepts DNS queries at the node level
# Reduces latency from ~5ms to ~0.1ms for cached entries
# Eliminates the conntrack race condition that causes 5s timeouts

# Check for CPU throttling (a hidden cause of latency spikes)
# Throttling happens when a container hits its CPU LIMIT, not just request

# Query Prometheus for CPU throttling rate
curl -s "http://prometheus.monitoring.svc.cluster.local:9090/api/v1/query" \
  --data-urlencode 'query=
    rate(container_cpu_cfs_throttled_seconds_total{
      namespace="checkout",
      container="checkout-api"
    }[5m])
    /
    rate(container_cpu_cfs_periods_total{
      namespace="checkout",
      container="checkout-api"
    }[5m]) * 100' | \
  jq '.data.result[] | {pod: .metric.pod, throttle_pct: .value[1]}'

# If throttle_pct > 25%, you have a CPU limit problem
# Solution: Increase CPU limit or remove it (controversial but effective)
# Better solution: Profile the CPU usage and optimize the hot path

# Check for connection refused / reset errors
kubectl exec -n checkout deploy/checkout-api -- \
  netstat -an | grep -E "CLOSE_WAIT|TIME_WAIT" | wc -l
# High CLOSE_WAIT count = connection pool not releasing connections

# Check network policy isn't dropping packets
kubectl get networkpolicy -n checkout -o yaml

# Use tcpdump to capture actual traffic (requires privileged pod)
kubectl debug node/worker-node-01 -it \
  --image=nicolaka/netshoot -- \
  tcpdump -i eth0 -w /tmp/capture.pcap \
  'host payment-svc and (tcp-rst or tcp-fin)'

# Copy capture for analysis
kubectl cp debug-pod:/tmp/capture.pcap ./capture.pcap
wireshark capture.pcap   # Analyze TCP resets, retransmissions

# Install Parca (continuous profiling)
helm repo add parca https://parca-dev.github.io/parca
helm install parca parca/parca \
  --namespace observability \
  --set parca-agent.enabled=true   # eBPF-based, no code changes needed

# Parca Agent runs as DaemonSet and profiles ALL processes
# No instrumentation required — uses eBPF

# Access Parca UI
kubectl port-forward -n observability svc/parca 7070:7070

# In the UI:
# 1. Select your checkout-api process
# 2. Set time range to when latency spikes occurred
# 3. Look for flame graph hotspots that appear only during spikes
# 4. Compare "normal" vs "spike" profiles to find the delta

# Grafana dashboard panels to correlate symptoms
# Panel 1: Request latency P99 (5m window)
# Query: histogram_quantile(0.99, rate(http_request_duration_seconds_bucket{service="checkout-api"}[5m]))

# Panel 2: DNS resolution time
# Query: histogram_quantile(0.99, rate(coredns_dns_request_duration_seconds_bucket[5m]))

# Panel 3: CPU throttling rate
# Query: rate(container_cpu_cfs_throttled_seconds_total{container="checkout-api"}[5m]) / rate(container_cpu_cfs_periods_total{container="checkout-api"}[5m])

# Panel 4: Connection pool saturation
# Query: db_connection_pool_size{service="checkout-api"} - db_connection_pool_available{service="checkout-api"}

# Panel 5: Error rate by pod (to identify specific bad pods)
# Query: rate(http_requests_total{service="checkout-api",status=~"5.."}[5m]) by (pod)

# Correlate all 5 panels on same time axis
# The panel that spikes FIRST (leading indicator) is your root cause

# Attach a debug container to a running pod WITHOUT restarting it
# (Kubernetes 1.23+ feature)
kubectl debug -it checkout-api-7d9f8b-xk2pq \
  --image=nicolaka/netshoot \
  --target=checkout-api \
  --namespace=checkout

# Inside the debug container, you share the pod's network namespace
# Run network diagnostics
curl -v http://payment-svc.production.svc.cluster.local/health
dig payment-svc.production.svc.cluster.local
ss -tulpn     # Show listening sockets
strace -p 1   # Trace system calls of PID 1 (the main process)

# Profile memory allocations in real time
# (If the target container has perf installed)
perf top -p $(pgrep checkout-api)