Advanced Networking: Network Policies and Service Mesh

┌─────────────────────────────────────────────────────────────┐
│                    Zero-Trust Layers                         │
│                                                              │
│  Layer 4 (CNI): NetworkPolicy                               │
│  ┌──────────┐  DENY ALL  ┌──────────┐                      │
│  │  Pod A   │────────────│  Pod B   │                      │
│  └──────────┘            └──────────┘                      │
│       │ explicit allow only │                               │
│                                                              │
│  Layer 7 (Mesh): Istio mTLS + AuthorizationPolicy          │
│  ┌──────────────┐  mTLS  ┌──────────────┐                  │
│  │ Envoy Sidecar│◄──────►│ Envoy Sidecar│                  │
│  │  (Pod A)     │        │  (Pod B)     │                  │
│  └──────────────┘        └──────────────┘                  │
│       │ SPIFFE identity verified │                          │
│                                                              │
│  Layer 8 (Audit): Envoy Access Logs → Loki/Elasticsearch   │
│  Every request logged with source identity, destination,    │
│  method, path, response code, latency                       │
└─────────────────────────────────────────────────────────────┘

# Step 1: Deploy a CNI that supports NetworkPolicy
# Calico, Cilium, or Weave Net (NOT Flannel — it doesn't enforce NetworkPolicy)

# Install Cilium (recommended — best performance and observability)
helm repo add cilium https://helm.cilium.io/
helm install cilium cilium/cilium \
  --namespace kube-system \
  --set kubeProxyReplacement=strict \
  --set hubble.enabled=true \
  --set hubble.relay.enabled=true \
  --set hubble.ui.enabled=true

# Verify Cilium is running
cilium status
cilium connectivity test

# Step 2: Global deny-all NetworkPolicy (apply to every namespace)
# This is the zero-trust baseline — nothing talks to anything by default
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-all
  namespace: production
spec:
  podSelector: {}      # Applies to ALL pods in namespace
  policyTypes:
    - Ingress
    - Egress
  # No ingress or egress rules = deny everything
---
# Allow DNS egress (required for service discovery — easy to forget!)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-dns-egress
  namespace: production
spec:
  podSelector: {}      # All pods need DNS
  policyTypes:
    - Egress
  egress:
    - ports:
        - port: 53
          protocol: UDP
        - port: 53
          protocol: TCP
      to:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: kube-system
          podSelector:
            matchLabels:
              k8s-app: kube-dns
---
# Allow monitoring egress (Prometheus scraping)
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-prometheus-scrape
  namespace: production
spec:
  podSelector: {}
  policyTypes:
    - Ingress
  ingress:
    - from:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: monitoring
          podSelector:
            matchLabels:
              app: prometheus
      ports:
        - port: 9090
          protocol: TCP

# Step 3: Service-specific allow rules
# Frontend → Backend API only
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-frontend-to-backend
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: backend-api        # This policy applies TO backend pods
  policyTypes:
    - Ingress
  ingress:
    - from:
        - podSelector:
            matchLabels:
              app: frontend   # Only allow traffic FROM frontend pods
      ports:
        - port: 8080
          protocol: TCP
---
# Backend API → Database only
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-backend-to-database
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: postgres           # Policy applies TO postgres pods
  policyTypes:
    - Ingress
  ingress:
    - from:
        - podSelector:
            matchLabels:
              app: backend-api  # Only backend can reach DB
      ports:
        - port: 5432
          protocol: TCP
---
# Backend egress: can only reach payment service and database
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: backend-egress-allowlist
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: backend-api
  policyTypes:
    - Egress
  egress:
    - to:
        - podSelector:
            matchLabels:
              app: postgres
      ports:
        - port: 5432
    - to:
        - podSelector:
            matchLabels:
              app: payment-service
      ports:
        - port: 8080
    - to:                     # Allow egress to Istio control plane
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: istio-system
      ports:
        - port: 15012         # Istio xDS
        - port: 15014         # Istio webhook

# Install Istio with strict mTLS mode
istioctl install --set profile=production \
  --set meshConfig.accessLogFile=/dev/stdout \
  --set meshConfig.accessLogEncoding=JSON \
  --set meshConfig.enableTracing=true \
  --set meshConfig.defaultConfig.tracing.zipkin.address=jaeger-collector.observability:9411

# Enable automatic sidecar injection for namespace
kubectl label namespace production istio-injection=enabled

# Verify sidecar injection
kubectl get pods -n production -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .spec.containers[*]}{.name}{", "}{end}{"\n"}{end}'
# Each pod should show: myapp-pod    myapp, istio-proxy,

# Step 4: Enforce STRICT mTLS across the entire mesh
# Without this, Istio allows both plaintext and mTLS (PERMISSIVE mode)
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: default-strict-mtls
  namespace: production        # Apply to entire namespace
spec:
  mtls:
    mode: STRICT               # Reject any non-mTLS traffic
                               # All communication must use SPIFFE certificates
---
# Global strict mTLS (mesh-wide, apply in istio-system)
apiVersion: security.istio.io/v1beta1
kind: PeerAuthentication
metadata:
  name: global-strict-mtls
  namespace: istio-system      # Mesh-wide policy
spec:
  mtls:
    mode: STRICT

# Step 5: AuthorizationPolicy — L7 zero-trust access control
# This is where zero-trust gets granular: not just "can A talk to B"
# but "can A call POST /api/payment on B with a valid JWT"

# Deny all by default at L7
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: deny-all-default
  namespace: production
spec:
  {}   # Empty spec = deny all traffic at L7
---
# Allow frontend → backend (specific paths only)
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: allow-frontend-to-backend
  namespace: production
spec:
  selector:
    matchLabels:
      app: backend-api
  action: ALLOW
  rules:
    - from:
        - source:
            # SPIFFE identity of the frontend service account
            principals:
              - "cluster.local/ns/production/sa/frontend-sa"
      to:
        - operation:
            methods: ["GET", "POST"]
            paths:
              - "/api/v1/products*"
              - "/api/v1/cart*"
              - "/api/v1/checkout*"
            # NOT allowed: /api/v1/admin*, /api/v1/users*
      when:
        # Require valid JWT from your identity provider
        - key: request.auth.claims[iss]
          values: ["https://auth.mycompany.com"]
---
# Allow backend → payment service (POST only, specific path)
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: allow-backend-to-payment
  namespace: production
spec:
  selector:
    matchLabels:
      app: payment-service
  action: ALLOW
  rules:
    - from:
        - source:
            principals:
              - "cluster.local/ns/production/sa/backend-api-sa"
      to:
        - operation:
            methods: ["POST"]
            paths: ["/api/v1/charge", "/api/v1/refund"]

# Step 6: RequestAuthentication — JWT validation at the mesh level
apiVersion: security.istio.io/v1beta1
kind: RequestAuthentication
metadata:
  name: jwt-validation
  namespace: production
spec:
  selector:
    matchLabels:
      app: backend-api
  jwtRules:
    - issuer: "https://auth.mycompany.com"
      jwksUri: "https://auth.mycompany.com/.well-known/jwks.json"
      audiences:
        - "backend-api"
      forwardOriginalToken: true    # Forward JWT to backend for app-level validation
      outputClaimToHeaders:
        - header: "x-user-id"
          claim: "sub"             # Extract user ID from JWT and add as header
        - header: "x-user-role"
          claim: "role"

# Configure Envoy access logging via Istio Telemetry API
apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: access-logging
  namespace: production
spec:
  accessLogging:
    - providers:
        - name: envoy              # Built-in Envoy access log provider
  tracing:
    - providers:
        - name: jaeger
      randomSamplingPercentage: 100.0   # 100% for financial audit requirements

// Sample Envoy access log entry (JSON format)
// Every request generates one of these — this is your audit trail
{
  "timestamp": "2026-07-09T14:23:01.234Z",
  "source": {
    "workload": "frontend",
    "namespace": "production",
    "principal": "spiffe://cluster.local/ns/production/sa/frontend-sa",
    "ip": "10.0.1.45",
    "port": 52341
  },
  "destination": {
    "workload": "backend-api",
    "namespace": "production",
    "principal": "spiffe://cluster.local/ns/production/sa/backend-api-sa",
    "ip": "10.0.2.67",
    "port": 8080
  },
  "request": {
    "method": "POST",
    "path": "/api/v1/checkout",
    "protocol": "HTTP/1.1",
    "user_agent": "checkout-frontend/2.1.0"
  },
  "response": {
    "code": 200,
    "duration_ms": 45
  },
  "tls": {
    "version": "TLSv1.3",
    "cipher": "TLS_AES_256_GCM_SHA384",
    "peer_certificate": "spiffe://cluster.local/ns/production/sa/frontend-sa"
  }
}

# Ship access logs to Loki for querying and retention
# (Assuming Promtail is deployed as DaemonSet)

# Query recent access logs via LogQL
logcli query '{namespace="production", app="istio-proxy"}
  | json
  | destination_workload = "backend-api"
  | response_code >= 400
  | line_format "{{.timestamp}} {{.source_principal}} → {{.destination_workload}} {{.request_method}} {{.request_path}} {{.response_code}}"'

# Query for audit: who called the payment service in the last hour
logcli query '{namespace="production"}
  | json
  | destination_workload = "payment-service"
  | line_format "{{.timestamp}} FROM={{.source_principal}} METHOD={{.request_method}} PATH={{.request_path}} STATUS={{.response_code}}"'

# Hubble provides real-time network flow visibility (Cilium's eBPF-based observability)

# Install Hubble CLI
export HUBBLE_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/hubble/master/stable.txt)
curl -L --remote-name-all https://github.com/cilium/hubble/releases/download/$HUBBLE_VERSION/hubble-linux-amd64.tar.gz
tar xzvf hubble-linux-amd64.tar.gz
mv hubble /usr/local/bin/

# Enable Hubble relay
cilium hubble enable --ui

# Observe live traffic flows
hubble observe \
  --namespace production \
  --follow \
  --output json | jq '{
    src: .source.workload_name,
    dst: .destination.workload_name,
    verdict: .verdict,
    protocol: .l4 | keys[0],
    port: (.l4 | ..[.destination_port]? // empty | first)
  }'

# Find DROPPED flows (policy violations)
hubble observe \
  --namespace production \
  --verdict DROPPED \
  --follow

# Generate network policy from observed flows (policy assistant)
hubble observe \
  --namespace production \
  --output json | \
  hubble generate-network-policy   # Experimental feature