Agfdhyk

I read the calico docs, it says calico will start an etcd instance when it starts, but I noticed that the K8s cluster will start an etcd pod, when the cluster starts. I want calico use that etcd node, so I do the following action:

Use calicoctl do test, create a config file:

# cat myconfig.yml

apiVersion: projectcalico.org/v3

kind: CalicoAPIConfig

metadata:

spec:

  datastoreType: etcdv3

  etcdEndpoints: https://10.100.1.20:2379

  etcdKeyFile: /etc/kubernetes/pki/etcd/server.key

  etcdCertFile: /etc/kubernetes/pki/etcd/server.crt

  etcdCACertFile: /etc/kubernetes/pki/etcd/ca.crt

the etcd config info came from /etc/kubernetes/manifests/etcd.yaml

# cat /etc/kubernetes/manifests/etcd.yaml

apiVersion: v1

kind: Pod

metadata:

  annotations:

    scheduler.alpha.kubernetes.io/critical-pod: ""

  creationTimestamp: null

  labels:

    component: etcd

    tier: control-plane

  name: etcd

  namespace: kube-system

spec:

  containers:

  - command:

    - etcd

    - --advertise-client-urls=https://127.0.0.1:2379

    - --cert-file=/etc/kubernetes/pki/etcd/server.crt

    - --client-cert-auth=true

    - --data-dir=/var/lib/etcd

    - --initial-advertise-peer-urls=https://127.0.0.1:2380

    - --initial-cluster=t-k8s-a1=https://127.0.0.1:2380

    - --key-file=/etc/kubernetes/pki/etcd/server.key

    - --listen-client-urls=https://127.0.0.1:2379

    - --listen-peer-urls=https://127.0.0.1:2380

    - --name=t-k8s-a1

    - --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt

    - --peer-client-cert-auth=true

    - --peer-key-file=/etc/kubernetes/pki/etcd/peer.key

    - --peer-trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt

    - --snapshot-count=10000

    - --trusted-ca-file=/etc/kubernetes/pki/etcd/ca.crt

    image: k8s.gcr.io/etcd-amd64:3.2.18

    imagePullPolicy: IfNotPresent

    livenessProbe:

      exec:

        command:

        - /bin/sh

        - -ec

        - ETCDCTL_API=3 etcdctl --endpoints=https://[127.0.0.1]:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt

          --cert=/etc/kubernetes/pki/etcd/healthcheck-client.crt --key=/etc/kubernetes/pki/etcd/healthcheck-client.key

          get foo

      failureThreshold: 8

      initialDelaySeconds: 15

      timeoutSeconds: 15

    name: etcd

    resources: {}

    volumeMounts:

    - mountPath: /var/lib/etcd

      name: etcd-data

    - mountPath: /etc/kubernetes/pki/etcd

      name: etcd-certs

  hostNetwork: true

  priorityClassName: system-cluster-critical

  volumes:

  - hostPath:

      path: /var/lib/etcd

      type: DirectoryOrCreate

    name: etcd-data

  - hostPath:

      path: /etc/kubernetes/pki/etcd

      type: DirectoryOrCreate

    name: etcd-certs

status: {}

still refused

# calicoctl get nodes --config ./myconfig.yml

Failed to create Calico API client: dial tcp 10.100.1.20:2379: connect: connection refused





# kubectl get pods --all-namespaces -o wide

NAMESPACE     NAME                                READY     STATUS    RESTARTS   AGE       IP             NODE       NOMINATED NODE

kube-system   calico-node-5nbwz                   2/2       Running   0          22h       10.100.1.21    t-k8s-b2   <none>

kube-system   calico-node-m967m                   2/2       Running   0          22h       10.100.1.20    t-k8s-a1   <none>

kube-system   calico-typha-64fc9d86dd-g8m54       1/1       Running   0          22h       10.100.1.21    t-k8s-b2   <none>

kube-system   coredns-78fcdf6894-5thqv            1/1       Running   0          1d        192.168.1.2    t-k8s-b2   <none>

kube-system   coredns-78fcdf6894-gm5zs            1/1       Running   0          1d        192.168.1.3    t-k8s-b2   <none>

kube-system   etcd-t-k8s-a1                       1/1       Running   0          1d        10.100.1.20    t-k8s-a1   <none>

kube-system   kube-apiserver-t-k8s-a1             1/1       Running   0          1d        10.100.1.20    t-k8s-a1   <none>

kube-system   kube-controller-manager-t-k8s-a1    1/1       Running   0          1d        10.100.1.20    t-k8s-a1   <none>

kube-system   kube-proxy-9rgmd                    1/1       Running   0          1d        10.100.1.20    t-k8s-a1   <none>

kube-system   kube-proxy-z75kc                    1/1       Running   0          1d        10.100.1.21    t-k8s-b2   <none>

kube-system   kube-scheduler-t-k8s-a1             1/1       Running   0          1d        10.100.1.20    t-k8s-a1   <none>

testalex      etcd-deployment-5b5d67bb84-nr7vc    1/1       Running   0          1d        192.168.1.26   t-k8s-b2   <none>

testalex      k8s-alert-76f97ccf49-gffgb          1/1       Running   0          1d        192.168.1.18   t-k8s-b2   <none>

testalex      k8s-monitor-7ddcb74b87-75vxb        1/1       Running   0          1d        192.168.1.27   t-k8s-b2   <none>

testalex      mysql-deployment-858464457f-nznq5   1/1       Running   0          1d        192.168.1.16   t-k8s-b2   <none>

When you install Calico on Kubernetes by default it will use the Kubernetes datastore (which uses etcdv3). Your calicoctl config under /etc/calico/calicoctl.cfg should looks something like this:

apiVersion: projectcalico.org/v3

kind: CalicoAPIConfig

metadata:

spec:

  datastoreType: "kubernetes"

  kubeconfig: "/path/to/.kube/config"

It works for me:

calicoctl get nodes

NAME

ip-172-x-x-x.us-west-2.compute.internal

ip-172-x-x-x.us-west-2.compute.internal

ip-172-x-x-x.us-west-2.compute.internal

ip-172-x-x-x.us-west-2.compute.internal

ip-172-x-x-x.us-west-2.compute.internal

ip-172-x-x-x.us-west-2.compute.internal