How to connect stateful workloads across Kubernetes clusters

kubectl apply -f https://raw.githubusercontent.com/k8ssandra/cass-operator/v1.7.0/docs/user/cass-operator-manifests.yaml

kubectl apply -f https://raw.githubusercontent.com/k8ssandra/cass-operator/v1.7.0/operator/k8s-flavors/gke/storage.yaml

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• 51
• 52
• 53
• 54
• 55
• 56
• 57
• 58
• 59
• 60
• 61
• 62
• 63
• 64
• 65
• 66

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: seed-updater
  name: seed-updater
spec:
  selector:
    matchLabels:
      app: seed-updater
  template:
    metadata:
      labels:
        app: seed-updater
spec:
      serviceAccount: seed-updater
      containers:
      - image: bitnami/kubectl
        name: seed-updater
        command:
        - /bin/bash
        - -c
        - |
          set -xeuo pipefail;
          while sleep 10; do
            if kubectl get endpoints cassandra-seed-service; then
              # remove automatic IP placement
              kubectl patch svc cassandra-seed-service -p '{"spec":{"selector":null}}'
              # set virtual IPs
              kubectl patch endpoints cassandra-seed-service -p $(kubectl get po -lcassandra.datastax.com/seed-node=true --no-headers | sed -r 's/.*-([0-9]+)-rack([0-9]+)-sts-([0-9]+) .*/{"addresses":[{"ip":"10.\1.\2.\3"}]}/'|paste -sd "," -|sed -r 's/(.*)/{"subsets":\[\0\]}/')
            fi
          done
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: seed-updater
rules:
- apiGroups:
  - ""
  resources:
  - endpoints
  - services
  - pods
  verbs:
  - get
  - list
  - update
  - patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: seed-updater
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: seed-updater
subjects:
- kind: ServiceAccount
  name: seed-updater
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: seed-updater

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11

apiVersion: v1
kind: Service
metadata:
  name: cross-dc
spec:
  selector:
    cassandra.datastax.com/cluster: cassandra
  ports:
    - port: 17001
  type: LoadBalancer
  publishNotReadyAddresses: true

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• 51
• 52
• 53
• 54
• 55
• 56
• 57
• 58
• 59
• 60
• 61
• 62
• 63
• 64
• 65
• 66
• 67
• 68
• 69
• 70
• 71
• 72
• 73
• 74
• 75
• 76
• 77
• 78

kind: ConfigMap
metadata:
  name: nginx
apiVersion: v1
data:
  nginx.conf: |
    pid /tmp/nginx.pid;
    worker_processes auto;
    events{
        worker_connections 32000;
    }

    stream{
        map $hostname $datacenter {
            ~^cassandra-dc-(\d+) $1;
        }
        map $hostname $virtualIP {
            ~^cassandra-dc-(\d+)-rack(\d+)-sts-(\d+)$ 10.$1.$2.$3;
        }
        log_format mainLogs '$remote_addr:$remote_port -> $server_addr:$server_port ($proxy_protocol_addr:$proxy_protocol_port -> $proxy_protocol_server_addr:$proxy_protocol_server_port)[$upstream_addr] $status $session_time $bytes_sent($upstream_bytes_received) $bytes_received($upstream_bytes_sent)';

        access_log /dev/stdout mainLogs;
        error_log /dev/stderr debug;

        # egress local
        map $server_addr $egressLocalIPtoPod {
            ~^10\.\d+\.(\d+)\.(\d+)$ rack$1-sts-$2;
        }
        server {
            listen   17007;
            resolver kube-dns.kube-system.svc.cluster.local ipv6=off;
            
            proxy_pass            cassandra-dc-$datacenter-$egressLocalIPtoPod.cassandra-dc-$datacenter
-all-pods-service.cass-operator.svc.cluster.local:17000;
            proxy_timeout         60m;
            proxy_connect_timeout 15s;
            proxy_protocol        on;
        }

        # egress remote
        map $server_addr $egressRemoteRoutes {
            ~^10\.1\.\d+\.\d+$    _EXTERNAL_IP_OF_FIRST_DC_;
            ~^10\.2\.\d+\.\d+$    _EXTERNAL_IP_OF_SECOND_DC_;
        }
        server {
            listen     17008;
            resolver   kube-dns.kube-system.svc.cluster.local ipv6=off;
            
            proxy_pass            $egressRemoteRoutes:17001;
            proxy_timeout         60m;
            proxy_connect_timeout 45s;
            proxy_protocol        on;
        }

        # ingress local
        server {
            listen                17000 proxy_protocol;
            proxy_pass            $virtualIP:7000;
            proxy_bind            $proxy_protocol_addr;
            proxy_timeout         60m;
            proxy_connect_timeout 15s;
        }

        # ingress remote
        map $proxy_protocol_server_addr $ingressRemoteIPtoPod {
            ~^10\.\d+\.(\d+)\.(\d+)$ rack$1-sts-$2;
        }
        server {
            listen                17001 proxy_protocol;
            resolver              kube-dns.kube-system.svc.cluster.local ipv6=off;
            proxy_timeout         60m;
            proxy_connect_timeout 15s;
            set_real_ip_from      0.0.0.0/0;
            proxy_pass            
cassandra-dc-$datacenter-$ingressRemoteIPtoPod.cassandra-dc-$datacenter-all-pods-service.cass-operator.svc.cluster.local:17000;
            proxy_protocol        on;
        }
    }

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• 51
• 52
• 53
• 54
• 55
• 56
• 57
• 58
• 59
• 60
• 61
• 62
• 63
• 64
• 65
• 66
• 67
• 68
• 69
• 70
• 71
• 72
• 73
• 74
• 75
• 76
• 77
• 78
• 79
• 80
• 81
• 82
• 83
• 84
• 85
• 86
• 87
• 88

apiVersion: cassandra.datastax.com/v1beta1
kind: CassandraDatacenter
metadata:
  name: dc-1
spec:
  clusterName: cassandra
  serverType: dse
  serverVersion: "6.8.2"
  serverImage: datastax/dse-server:6.8.12
  managementApiAuth:
    insecure: {}
  size: 3
  racks:
  - name: rack0
  - name: rack1
  - name: rack2
  storageConfig:
    cassandraDataVolumeClaimSpec:
      storageClassName: server-storage
      accessModes:
        - ReadWriteOnce
      resources:
        requests:
          storage: 1Gi
  config:    
    jvm-server-options:
      initial_heap_size: "800m"
      max_heap_size: "800m"
  podTemplateSpec:
    spec:
      containers:
      - name: cassandra
      - image: nginx
        name: nginx
        resources: {}
        securityContext:
          capabilities:
            add:
            - NET_ADMIN
        volumeMounts:
        - mountPath: /etc/nginx
          name: config-vol
      initContainers:
      - name: server-config-init    
      - command:
        - /bin/sh
        - -c
        - |
          set -xeuo pipefail;
          # convert hostname to our virtual IP
          VIRTIP=$(hostname|sed -E "s/^.*dc-([0-9]+)-rack([0-9]+)-sts-([0-9]+)$/10.\\1.\\2.\\3/g")
          
          # override the IP for cassandra to listen on (just internode traffic)
          sed -ib -E "s/listen_address: .*/listen_address: $VIRTIP/" /config/cassandra.yaml

          # init container may rerun so be idempotent and optimistically delete rules from previous invocation
          ip route del local 10.0.0.0/8 dev eth0 || true;
          ip addr del $VIRTIP/32 dev eth0 || true;

          # next, we assign the 10 subnet locally
          # this has the same functionality as adding all the IPs in this cidr one by one to the eth0 network device/interface
          ip route add local 10.0.0.0/8 dev eth0;

          # these rules are inserted (-I) so are evaluated in opposite order of occurance (last will be checked first)
          # that means we don't need to worry about deleting them if the init container is rerun, making them also idempotent
          # the first rule is for cross-dc traffic and is changing the destination port from 7000 to 17008(where nginx listens) on any traffic with dest ip in 10 subnet
          # the second rule is for intra-dc traffic and is changing the destination port from 7000 to 17007(where nginx listens) on any traffic with dest ip in subnet of this DC
          # the third rule excludes the current virtual IP from this logic (-j RETURN)
          iptables -t nat -I OUTPUT -p tcp --dport 7000 -d 10.0.0.0/8 -j DNAT --to-destination :17008;
          iptables -t nat -I OUTPUT -p tcp --dport 7000 -d $VIRTIP/16 -j DNAT --to-destination :17007;
          iptables -t nat -I OUTPUT -p tcp --dport 7000 -d $VIRTIP/32 -j RETURN;
        image: praqma/network-multitool
        name: net-init
        resources: {}
        securityContext:
          runAsUser: 0
          runAsGroup: 0
          fsGroup: 0
          capabilities:
            add:
            - NET_ADMIN
        volumeMounts:
        - mountPath: /config
          name: server-config
      volumes:
      - name: config-vol
        configMap:
          name: nginx

• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9
• 10
• 11
• 12
• 13
• 14
• 15
• 16
• 17
• 18
• 19
• 20
• 21
• 22
• 23
• 24
• 25
• 26
• 27
• 28
• 29
• 30
• 31
• 32
• 33
• 34
• 35
• 36
• 37
• 38
• 39
• 40
• 41
• 42
• 43
• 44
• 45
• 46
• 47
• 48
• 49
• 50
• 51
• 52
• 53
• 54
• 55
• 56
• 57
• 58
• 59
• 60
• 61
• 62
• 63
• 64
• 65
• 66
• 67
• 68
• 69
• 70
• 71
• 72
• 73
• 74
• 75
• 76
• 77
• 78
• 79
• 80
• 81
• 82
• 83
• 84
• 85
• 86
• 87
• 88
• 89
• 90
• 91
• 92

apiVersion: cassandra.datastax.com/v1beta1
kind: CassandraDatacenter
metadata:
  name: dc-2
spec:
  additionalSeeds:
  - 10.1.0.0
  - 10.1.1.0
  - 10.1.2.0
  clusterName: cassandra
  serverType: dse
  serverVersion: "6.8.2"
  serverImage: datastax/dse-server:6.8.12
  managementApiAuth:
    insecure: {}
  size: 3
  racks:
  - name: rack0
  - name: rack1
  - name: rack2
  storageConfig:
    cassandraDataVolumeClaimSpec:
      storageClassName: server-storage
      accessModes:
        - ReadWriteOnce
      resources:
        requests:
          storage: 1Gi
  config:    
    jvm-server-options:
      initial_heap_size: "800m"
      max_heap_size: "800m"
  podTemplateSpec:
    spec:
      containers:
      - name: cassandra
      - image: nginx
        name: nginx
        resources: {}
        securityContext:
          capabilities:
            add:
            - NET_ADMIN
        volumeMounts:
        - mountPath: /etc/nginx
          name: config-vol
      initContainers:
      - name: server-config-init    
      - command:
        - /bin/sh
        - -c
        - |
          set -xeuo pipefail;
          # convert hostname to our virtual IP
          VIRTIP=$(hostname|sed -E "s/^.*dc-([0-9]+)-rack([0-9]+)-sts-([0-9]+)$/10.\\1.\\2.\\3/g")
          
          # override the IP for cassandra to listen on (just internode traffic)
          sed -ib -E "s/listen_address: .*/listen_address: $VIRTIP/" /config/cassandra.yaml

          # init container may rerun so be idempotent and optimistically delete rules from previous invocation
          ip route del local 10.0.0.0/8 dev eth0 || true;
          ip addr del $VIRTIP/32 dev eth0 || true;

          # next, we assign the 10 subnet locally
          # this has the same functionality as adding all the IPs in this cidr one by one to the eth0 network device/interface
          ip route add local 10.0.0.0/8 dev eth0;

          # these rules are inserted (-I) so are evaluated in opposite order of occurance (last will be checked first)
          # that means we don't need to worry about deleting them if the init container is rerun, making them also idempotent
          # the first rule is for cross-dc traffic and is changing the destination port from 7000 to 17008(where nginx listens) on any traffic with dest ip in 10 subnet
          # the second rule is for intra-dc traffic and is changing the destination port from 7000 to 17007(where nginx listens) on any traffic with dest ip in subnet of this DC
          # the third rule excludes the current virtual IP from this logic (-j RETURN)
          iptables -t nat -I OUTPUT -p tcp --dport 7000 -d 10.0.0.0/8 -j DNAT --to-destination :17008;
          iptables -t nat -I OUTPUT -p tcp --dport 7000 -d $VIRTIP/16 -j DNAT --to-destination :17007;
          iptables -t nat -I OUTPUT -p tcp --dport 7000 -d $VIRTIP/32 -j RETURN;
        image: praqma/network-multitool
        name: net-init
        resources: {}
        securityContext:
          runAsUser: 0
          runAsGroup: 0
          fsGroup: 0
          capabilities:
            add:
            - NET_ADMIN
        volumeMounts:
        - mountPath: /config
          name: server-config
      volumes:
      - name: config-vol
        configMap:
          name: nginx