[Cilium Study] Cluster mesh

 

Cluster Mesh란?

Cluster Mesh는 여러 개의 Kubernetes 클러스터를 하나의 네트워크처럼 연결해주는 기능입니다. 이를 통해 클러스터 간에도 Pod-to-Pod 직접 통신, 서비스 디스커버리 공유, 그리고 보안 정책 일관성을 유지할 수 있습니다. 특히 Cilium Cluster Mesh는 네트워크 레벨에서 클러스터를 통합해, 멀티클러스터 환경에서도 마치 단일 클러스터처럼 워크로드를 운영할 수 있게 해줍니다.

 

실습

kind 를 사용해서 2 개의 k8s 클러스터를 배포 (east, west)

Cluster Mesh 설정을 완료한 후 서로 다른 클러스터의 pod 들끼리 통신이 되는지 확인하는 것을 목표로 한다..

(kind 설치 및 k8s 배포 과정은 생략)

kinds k8s 클러스터 west, east 배포 

#
kind create cluster --name west --image kindest/node:v1.33.2 --config - <<EOF
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
  extraPortMappings:
  - containerPort: 30000 # sample apps
    hostPort: 30000
  - containerPort: 30001 # hubble ui
    hostPort: 30001
- role: worker
  extraPortMappings:
  - containerPort: 30002 # sample apps
    hostPort: 30002
networking:
  podSubnet: "10.0.0.0/16"
  serviceSubnet: "10.2.0.0/16"
  disableDefaultCNI: true
  kubeProxyMode: none
EOF


# 설치 확인
kubectl ctx
kubectl get node 
kubectl get pod -A

# 노드에 기본 툴 설치
docker exec -it west-control-plane sh -c 'apt update && apt install tree psmisc lsof wget net-tools dnsutils tcpdump ngrep iputils-ping git -y'
docker exec -it west-worker sh -c 'apt update && apt install tree psmisc lsof wget net-tools dnsutils tcpdump ngrep iputils-ping git -y'

#
kind create cluster --name east --image kindest/node:v1.33.2 --config - <<EOF
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
  extraPortMappings:
  - containerPort: 31000 # sample apps
    hostPort: 31000
  - containerPort: 31001 # hubble ui
    hostPort: 31001
- role: worker
  extraPortMappings:
  - containerPort: 31002 # sample apps
    hostPort: 31002
networking:
  podSubnet: "10.1.0.0/16"
  serviceSubnet: "10.3.0.0/16"
  disableDefaultCNI: true
  kubeProxyMode: none
EOF

# 설치 확인
kubectl config get-contexts 
CURRENT   NAME        CLUSTER     AUTHINFO    NAMESPACE
*         kind-east   kind-east   kind-east   
          kind-west   kind-west   kind-west 

kubectl config set-context kind-east
kubectl get node -v=6 --context kind-east
kubectl get node -v=6
kubectl get node -v=6 --context kind-west
cat ~/.kube/config

kubectl get pod -A
kubectl get pod -A --context kind-west

# 노드에 기본 툴 설치
docker exec -it east-control-plane sh -c 'apt update && apt install tree psmisc lsof wget net-tools dnsutils tcpdump ngrep iputils-ping git -y'
docker exec -it east-worker sh -c 'apt update && apt install tree psmisc lsof wget net-tools dnsutils tcpdump ngrep iputils-ping git -y'

각 클러스터에 cilium cni 배포

# cilium cli 설치
brew install cilium-cli # macOS

# cilium cli 로 cilium cni 설치
# west
cilium install --version 1.17.6 --set ipam.mode=kubernetes \
--set kubeProxyReplacement=true --set bpf.masquerade=true \
--set endpointHealthChecking.enabled=false --set healthChecking=false \
--set operator.replicas=1 --set debug.enabled=true \
--set routingMode=native --set autoDirectNodeRoutes=true --set ipv4NativeRoutingCIDR=10.0.0.0/16 \
--set ipMasqAgent.enabled=true --set ipMasqAgent.config.nonMasqueradeCIDRs='{10.1.0.0/16}' \
--set cluster.name=west --set cluster.id=1 \
--context kind-west

watch kubectl get pod -n kube-system --context kind-west

# east
cilium install --version 1.17.6 --set ipam.mode=kubernetes \
--set kubeProxyReplacement=true --set bpf.masquerade=true \
--set endpointHealthChecking.enabled=false --set healthChecking=false \
--set operator.replicas=1 --set debug.enabled=true \
--set routingMode=native --set autoDirectNodeRoutes=true --set ipv4NativeRoutingCIDR=10.1.0.0/16 \
--set ipMasqAgent.enabled=true --set ipMasqAgent.config.nonMasqueradeCIDRs='{10.0.0.0/16}' \
--set cluster.name=east --set cluster.id=2 \
--context kind-east

watch kubectl get pod -n kube-system --context kind-east


## 배포 확인
kwest get pod -A && keast get pod -A
cilium status --context kind-west
cilium status --context kind-east
cilium config view --context kind-west
cilium config view --context kind-east
kwest exec -it -n kube-system ds/cilium -- cilium status --verbose
keast exec -it -n kube-system ds/cilium -- cilium status --verbose

kwest -n kube-system exec ds/cilium -c cilium-agent -- cilium-dbg bpf ipmasq list
keast -n kube-system exec ds/cilium -c cilium-agent -- cilium-dbg bpf ipmasq list

 

Error가 많은 것 같지만 일단 진행

 

Setting up Cluster Mesh

# 라우팅 정보 확인
docker exec -it west-control-plane ip -c route
docker exec -it west-worker ip -c route
docker exec -it east-control-plane ip -c route
docker exec -it east-worker ip -c route


# Specify the Cluster Name and ID : 이미 설정 되어 있음

# Shared Certificate Authority
keast get secret -n kube-system cilium-ca
keast delete secret -n kube-system cilium-ca

kubectl --context kind-west get secret -n kube-system cilium-ca -o yaml | \
kubectl --context kind-east create -f -

keast get secret -n kube-system cilium-ca


# 모니터링 : 신규 터미널 2개
cilium clustermesh status --context kind-west --wait  
cilium clustermesh status --context kind-east --wait


# Enable Cluster Mesh : 간단한 실습 환경으로 NodePort 로 진행
cilium clustermesh enable --service-type NodePort --enable-kvstoremesh=false --context kind-west
cilium clustermesh enable --service-type NodePort --enable-kvstoremesh=false --context kind-east
 

# 32379 NodePort 정보 : clustermesh-apiserver 서비스 정보
kwest get svc,ep -n kube-system clustermesh-apiserver --context kind-west
NAME                            TYPE       CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
service/clustermesh-apiserver   NodePort   10.2.216.182   <none>        2379:32379/TCP   65s

NAME                              ENDPOINTS         AGE
endpoints/clustermesh-apiserver   10.0.0.195:2379   65s # 대상 파드는 clustermesh-apiserver 파드 IP

kwest get pod -n kube-system -owide | grep clustermesh


keast get svc,ep -n kube-system clustermesh-apiserver --context kind-east
NAME                            TYPE       CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
service/clustermesh-apiserver   NodePort   10.3.252.188   <none>        2379:32379/TCP   43s

NAME                              ENDPOINTS         AGE
endpoints/clustermesh-apiserver   10.1.0.206:2379   65s # 대상 파드는 clustermesh-apiserver 파드 IP

keast get pod -n kube-system -owide | grep clustermesh


# 모니터링 : 신규 터미널 2개
watch -d "cilium clustermesh status --context kind-west --wait"
watch -d "cilium clustermesh status --context kind-east --wait"


# Connect Clusters
cilium clustermesh connect --context kind-west --destination-context kind-east

# 확인
cilium clustermesh status --context kind-west --wait
cilium clustermesh status --context kind-east --wait

# 
kubectl exec -it -n kube-system ds/cilium -c cilium-agent --context kind-west -- cilium-dbg troubleshoot clustermesh
kubectl exec -it -n kube-system ds/cilium -c cilium-agent --context kind-east -- cilium-dbg troubleshoot clustermesh

# 확인
kwest get pod -A && keast get pod -A
cilium status --context kind-west
cilium status --context kind-east
cilium clustermesh status --context kind-west
cilium clustermesh status --context kind-east
cilium config view --context kind-west
cilium config view --context kind-east
kwest exec -it -n kube-system ds/cilium -- cilium status --verbose
keast exec -it -n kube-system ds/cilium -- cilium status --verbose
ClusterMesh:   1/1 remote clusters ready, 0 global-services
   east: ready, 2 nodes, 4 endpoints, 3 identities, 0 services, 0 MCS-API service exports, 0 reconnections (last: never)
   └  etcd: 1/1 connected, leases=0, lock leases=0, has-quorum=true: endpoint status checks are disabled, ID: c6ba18866da7dfd8
   └  remote configuration: expected=true, retrieved=true, cluster-id=2, kvstoremesh=false, sync-canaries=true, service-exports=disabled
   └  synchronization status: nodes=true, endpoints=true, identities=true, services=true

#
helm get values -n kube-system cilium --kube-context kind-west 
...
cluster:
  id: 1
  name: west
clustermesh:
  apiserver:
    kvstoremesh:
      enabled: false
    service:
      type: NodePort
    tls:
      auto:
        enabled: true
        method: cronJob
        schedule: 0 0 1 */4 *
  config:
    clusters:
    - ips:
      - 172.18.0.4
      name: east
      port: 32379
    enabled: true
  useAPIServer: true
...

helm get values -n kube-system cilium --kube-context kind-east 
...
cluster:
  id: 2
  name: east
clustermesh:
  apiserver:
    kvstoremesh:
      enabled: false
    service:
      type: NodePort
    tls:
      auto:
        enabled: true
        method: cronJob
        schedule: 0 0 1 */4 *
  config:
    clusters:
    - ips:
      - 172.18.0.3
      name: west
      port: 32379
    enabled: true
  useAPIServer: true
...


# 라우팅 정보 확인 : 클러스터간 PodCIDR 라우팅 주입 확인!
docker exec -it west-control-plane ip -c route
docker exec -it west-worker ip -c route
docker exec -it east-control-plane ip -c route
docker exec -it east-worker ip -c route

이유는 모르겠지만 실패한다.

 

실습이 전혀 진행이 되지 않는 관계로 잠시 중단.