Multi-Cluster Strategy for a Geographically Distributed Company: Architecture & Implementation

                        ┌─────────────────┐
                        │   Global DNS    │
                        │  (Route53/CF)   │
                        │  GeoDNS routing │
                        └────────┬────────┘
                                 │
              ┌──────────────────┼──────────────────┐
              │                  │                  │
    ┌─────────▼──────┐  ┌────────▼───────┐  ┌──────▼──────────┐
    │   US-East      │  │   EU-West      │  │   APAC          │
    │   Cluster      │  │   Cluster      │  │   Cluster       │
    │                │  │                │  │  (DR + Serve)   │
    │ • US workloads │  │ • EU workloads │  │ • APAC workloads│
    │ • US databases │  │ • EU databases │  │ • Cross-region  │
    │ • SOX compliant│  │ • GDPR compliant│ │   backup target │
    └────────┬───────┘  └────────┬───────┘  └──────┬──────────┘
             │                   │                  │
             └───────────────────┴──────────────────┘
                         Istio Multi-cluster
                      (cross-cluster service mesh)

# argocd-applicationset.yaml
# Deploy the same base config to all clusters with region-specific overlays
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: platform-services
  namespace: argocd
spec:
  generators:
    - list:
        elements:
          - cluster: us-east
            url: https://k8s-us-east.mycompany.com
            region: us-east-1
            environment: production
            dataResidency: us
          - cluster: eu-west
            url: https://k8s-eu-west.mycompany.com
            region: eu-west-1
            environment: production
            dataResidency: eu
          - cluster: apac
            url: https://k8s-apac.mycompany.com
            region: ap-southeast-1
            environment: production
            dataResidency: apac

  template:
    metadata:
      name: "platform-{{cluster}}"
    spec:
      project: platform
      source:
        repoURL: https://github.com/mycompany/k8s-config
        targetRevision: main
        path: "clusters/{{cluster}}"  # Region-specific Kustomize overlay
      destination:
        server: "{{url}}"
        namespace: platform
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
        syncOptions:
          - CreateNamespace=true

# Repository structure for GitOps multi-cluster
k8s-config/
├── base/                          # Shared base resources
│   ├── deployments/
│   ├── services/
│   └── rbac/
├── clusters/
│   ├── us-east/                   # US-specific overlay
│   │   ├── kustomization.yaml
│   │   ├── patches/
│   │   │   ├── replicas.yaml      # Higher replica count for US
│   │   │   └── resources.yaml
│   │   └── region-config.yaml     # US-specific ConfigMap
│   ├── eu-west/                   # EU-specific overlay
│   │   ├── kustomization.yaml
│   │   ├── patches/
│   │   │   └── gdpr-annotations.yaml
│   │   └── data-residency-policy.yaml
│   └── apac/                      # APAC overlay
│       ├── kustomization.yaml
│       └── dr-config.yaml
└── policies/
    ├── us-data-residency.yaml     # OPA policies per region
    └── eu-gdpr-policy.yaml

# EU GDPR: Enforce data stays in EU namespace
# OPA Gatekeeper policy — blocks cross-region database connections
apiVersion: templates.gatekeeper.sh/v1
kind: ConstraintTemplate
metadata:
  name: k8sdataresidency
spec:
  crd:
    spec:
      names:
        kind: K8sDataResidency
      validation:
        openAPIV3Schema:
          type: object
          properties:
            allowedRegions:
              type: array
              items:
                type: string
            dataClassification:
              type: string
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8sdataresidency

        violation[{"msg": msg}] {
          # Check if pod has data-region annotation
          pod := input.review.object
          data_region := pod.metadata.annotations["mycompany.com/data-region"]
          cluster_region := input.parameters.clusterRegion

          # Reject if data region doesn't match cluster region
          data_region != cluster_region

          msg := sprintf(
            "Pod '%v' has data-region '%v' but cluster is in region '%v'. Data residency violation.",
            [pod.metadata.name, data_region, cluster_region]
          )
        }

        violation[{"msg": msg}] {
          # Require data-region annotation on all pods in regulated namespaces
          pod := input.review.object
          regulated_namespaces := {"eu-production", "us-production"}
          regulated_namespaces[pod.metadata.namespace]
          not pod.metadata.annotations["mycompany.com/data-region"]
          msg := sprintf(
            "Pod '%v' in regulated namespace '%v' must have 'mycompany.com/data-region' annotation",
            [pod.metadata.name, pod.metadata.namespace]
          )
        }
---
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sDataResidency
metadata:
  name: eu-data-residency
spec:
  match:
    kinds:
      - apiGroups: [""]
        kinds: ["Pod"]
    namespaces:
      - eu-production
  parameters:
    clusterRegion: "eu-west-1"
    dataClassification: "gdpr-regulated"

# Set up Istio multi-cluster with primary-remote model
# US-East is primary, EU-West and APAC are remotes

# Step 1: Create a shared root CA for all clusters
# (Required for cross-cluster mTLS)
mkdir -p certs && cd certs

# Generate root CA
openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 \
  -subj "/O=MyCompany/CN=Root CA" \
  -keyout root-ca-key.pem \
  -out root-ca-cert.pem

# Generate intermediate CA for each cluster
for cluster in us-east eu-west apac; do
  # Generate intermediate CA key and CSR
  openssl req -new -newkey rsa:4096 -nodes \
    -subj "/O=MyCompany/CN=Intermediate CA $cluster" \
    -keyout $cluster-ca-key.pem \
    -out $cluster-ca-csr.pem

  # Sign with root CA
  openssl x509 -req -days 730 \
    -CA root-ca-cert.pem \
    -CAkey root-ca-key.pem \
    -CAcreateserial \
    -in $cluster-ca-csr.pem \
    -out $cluster-ca-cert.pem

  # Create cert chain
  cat $cluster-ca-cert.pem root-ca-cert.pem > $cluster-ca-chain.pem
done

# Step 2: Install Istio certs in each cluster
for cluster in us-east eu-west apac; do
  kubectl create secret generic cacerts \
    -n istio-system \
    --context=k8s-$cluster \
    --from-file=cacert.pem=$cluster-ca-cert.pem \
    --from-file=cakey.pem=$cluster-ca-key.pem \
    --from-file=root-cert.pem=root-ca-cert.pem \
    --from-file=cert-chain.pem=$cluster-ca-chain.pem
done

# Install Istio on US-East (primary cluster)
# istio-us-east-values.yaml
apiVersion: install.istio.io/v1alpha1
kind: IstioOperator
metadata:
  name: istio-primary
spec:
  profile: default
  values:
    global:
      meshID: global-mesh
      multiCluster:
        clusterName: us-east
      network: us-east-network
  components:
    ingressGateways:
      - name: istio-eastwestgateway
        label:
          istio: eastwestgateway
          app: istio-eastwestgateway
          topology.istio.io/network: us-east-network
        enabled: true
        k8s:
          env:
            - name: ISTIO_META_ROUTER_MODE
              value: "sni-dnat"
          service:
            ports:
              - port: 15021
                targetPort: 15021
                name: status-port
              - port: 15443
                targetPort: 15443
                name: tls
              - port: 15012
                targetPort: 15012
                name: tls-istiod
              - port: 15017
                targetPort: 15017
                name: tls-webhook

# Expose services for cross-cluster discovery
# Apply on each cluster to expose its services to the mesh

kubectl apply -n istio-system \
  --context=k8s-us-east \
  -f - <<EOF
apiVersion: networking.istio.io/v1alpha3
kind: Gateway
metadata:
  name: cross-network-gateway
spec:
  selector:
    istio: eastwestgateway
  servers:
    - port:
        number: 15443
        name: tls
        protocol: TLS
      tls:
        mode: AUTO_PASSTHROUGH
      hosts:
        - "*.local"
EOF

# Enable endpoint discovery between clusters
istioctl x create-remote-secret \
  --context=k8s-eu-west \
  --name=eu-west | \
  kubectl apply -f - --context=k8s-us-east

istioctl x create-remote-secret \
  --context=k8s-apac \
  --name=apac | \
  kubectl apply -f - --context=k8s-us-east

# Route53 GeoDNS configuration (via Terraform)
# terraform/dns/main.tf

resource "aws_route53_health_check" "us_east" {
  fqdn              = "api-us-east.mycompany.com"
  port              = 443
  type              = "HTTPS"
  resource_path     = "/health"
  failure_threshold = "3"
  request_interval  = "10"

  tags = {
    Name = "us-east-health-check"
  }
}

resource "aws_route53_health_check" "eu_west" {
  fqdn              = "api-eu-west.mycompany.com"
  port              = 443
  type              = "HTTPS"
  resource_path     = "/health"
  failure_threshold = "3"
  request_interval  = "10"
}

# GeoDNS routing policy
resource "aws_route53_record" "api_us" {
  zone_id = aws_route53_zone.main.zone_id
  name    = "api.mycompany.com"
  type    = "A"

  geolocation_routing_policy {
    continent = "NA"   # North America → US-East
  }

  set_identifier = "us-east"
  health_check_id = aws_route53_health_check.us_east.id
  ttl     = 60
  records = [var.us_east_ingress_ip]
}

resource "aws_route53_record" "api_eu" {
  zone_id = aws_route53_zone.main.zone_id
  name    = "api.mycompany.com"
  type    = "A"

  geolocation_routing_policy {
    continent = "EU"   # Europe → EU-West
  }

  set_identifier  = "eu-west"
  health_check_id = aws_route53_health_check.eu_west.id
  ttl     = 60
  records = [var.eu_west_ingress_ip]
}

resource "aws_route53_record" "api_apac" {
  zone_id = aws_route53_zone.main.zone_id
  name    = "api.mycompany.com"
  type    = "A"

  geolocation_routing_policy {
    continent = "AS"   # Asia → APAC
  }

  set_identifier = "apac"
  ttl     = 60
  records = [var.apac_ingress_ip]
}

# Fallback: Route all other traffic to US-East
resource "aws_route53_record" "api_default" {
  zone_id = aws_route53_zone.main.zone_id
  name    = "api.mycompany.com"
  type    = "A"

  geolocation_routing_policy {
    country = "*"   # Default catch-all
  }

  set_identifier  = "default"
  health_check_id = aws_route53_health_check.us_east.id
  ttl     = 60
  records = [var.us_east_ingress_ip]
}

# Install Velero on all clusters with S3 cross-region replication
helm repo add vmware-tanzu https://vmware-tanzu.github.io/helm-charts
helm install velero vmware-tanzu/velero \
  --namespace velero \
  --create-namespace \
  --set configuration.provider=aws \
  --set configuration.backupStorageLocation.bucket=mycompany-k8s-backups \
  --set configuration.backupStorageLocation.config.region=us-east-1 \
  --set configuration.volumeSnapshotLocation.config.region=us-east-1 \
  --set initContainers[0].name=velero-plugin-for-aws \
  --set initContainers[0].image=velero/velero-plugin-for-aws:v1.8.0 \
  --set initContainers[0].volumeMounts[0].mountPath=/target \
  --set initContainers[0].volumeMounts[0].name=plugins

# Scheduled backups — EU cluster backs up to S3 with cross-region replication to APAC
apiVersion: velero.io/v1
kind: Schedule
metadata:
  name: eu-production-backup
  namespace: velero
spec:
  schedule: "0 */6 * * *"    # Every 6 hours
  template:
    includedNamespaces:
      - eu-production
    excludedResources:
      - events
      - events.events.k8s.io
    storageLocation: eu-s3-backup
    volumeSnapshotLocations:
      - eu-ebs-snapshots
    ttl: 720h0m0s             # 30 days retention
    labelSelector:
      matchLabels:
        backup: "true"        # Only backup labeled resources
---
# Disaster Recovery: Restore EU workloads to APAC cluster
# Run this when EU-West cluster fails
apiVersion: velero.io/v1
kind: Restore
metadata:
  name: eu-to-apac-restore
  namespace: velero
spec:
  backupName: eu-production-backup-20260709-060000
  includedNamespaces:
    - eu-production
  namespaceMapping:
    eu-production: eu-production-dr   # Restore into DR namespace
  restorePVs: true
  existingResourcePolicy: update

# Submariner: Cross-cluster L3 connectivity without service mesh
# Alternative to Istio multi-cluster for simpler use cases

# Install Submariner broker on US-East (hub cluster)
subctl deploy-broker \
  --kubeconfig ~/.kube/us-east.yaml \
  --globalnet                    # Enable GlobalNet for overlapping CIDRs

# Join EU-West to the broker
subctl join broker-info.subm \
  --kubeconfig ~/.kube/eu-west.yaml \
  --clusterid eu-west \
  --natt=false

# Join APAC to the broker
subctl join broker-info.subm \
  --kubeconfig ~/.kube/apac.yaml \
  --clusterid apac \
  --natt=false

# Export a service for cross-cluster discovery
# Once exported, service is accessible as:
# <svc-name>.<namespace>.svc.clusterset.local
kubectl apply -f - <<EOF
apiVersion: multicluster.x-k8s.io/v1alpha1
kind: ServiceExport
metadata:
  name: payment-service
  namespace: production
  # Applied on US-East cluster
  # Payment service now accessible from EU-West and APAC as:
  # payment-service.production.svc.clusterset.local
EOF

# Thanos for multi-cluster Prometheus federation
# Each cluster runs Prometheus + Thanos Sidecar
# Central Thanos Query aggregates metrics from all clusters

# thanos-values.yaml (deploy on each cluster)
prometheus:
  prometheusSpec:
    externalLabels:
      cluster: us-east          # Region-specific label on all metrics
      region: us-east-1
      environment: production
    thanos:
      objectStorageConfig:
        key: thanos.yaml
        name: thanos-objstore-secret

# Thanos Query (deploy on central monitoring cluster)
thanos-query:
  stores:
    - dnssrv+_grpc._tcp.thanos-store-us-east.monitoring.svc.cluster.local
    - dnssrv+_grpc._tcp.thanos-store-eu-west.monitoring.svc.cluster.local
    - dnssrv+_grpc._tcp.thanos-store-apac.monitoring.svc.cluster.local
  replicaLabels:
    - cluster
    - replica