Tag Archives: governors

The deployment of etcd storage and flannel network configuration for kubernetes / k8s multi node deployment

article directory

    • 1. Project requirement analysis:
    • 2. Project step deployment (master node) :
      • 5 [7] etcd binary upload

        • 6

      • 7 [8] create configuration file, command file, Certificate
      • [9] use another terminal to copy the certificate and systemctl management service script to other nodes
      • [10] and modify the configuration file under CFG
      • [11] to check whether the cluster state is healthy
  • k8s multi-node deployment of flannel network configuration
      • [1] write the allocated subnet segment to etcd, For the use of flannel
      • [2] view written information
      • [3] above all node node deployment flannel component
      • [4] create k8s working directory, copy command file
      • [5] write flannel components executing scripts to start the node node are the same 】
      • [6] open flannel component network function
      • 【 7 】 configuration docker connection flannel components [all node node a
      • [8] view bip subnet
      • [9] restart docker service
      • [10] view flannel network
      • [11] test connectivity between nodes

    • k8s multi-node deployment etcd storage deployment

    1. Project demand analysis: 192.168.60.100 is the node1 node kubelet; kubelet kube-proxy docker flannel etcd
    [3] 192.168.60.60 is the node2 node kubelet kube-proxy docker flannel etcd

    Ii. Project step deployment (master node) :

    //master master node configuration

    [1] download certificate making tool 

    [root@localhost ~]# hostnamectl set-hostname master
[root@localhost ~]# su
[root@master ~]# cd /usr/local/bin
[root@master bin]# chmod +x *
[root@master bin]# ls
cfssl  cfssl-certinfo  cfssljson

    [2] ca certificate

    is defined

    [root@master ~]#mkdir -p k8s/etcd-cert
[root@master etcd-cert]#cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"     
        ]  
      } 
    }         
  }
}
EOF

    [3] realize certificate signature 

    [root@master etcd-cert]#cat > ca-csr.json <<EOF 
{   
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF

    [4] generate certificate 

    [root@master etcd-cert]#cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

    [5] specifies the communication verification between the three etcd nodes 

    [root@master etcd-cert]#cat > server-csr.json <<EOF
{
    "CN": "etcd",
    "hosts": [
    "192.168.60.10",
    "192.168.60.100",
    "192.168.60.60"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing"
        }
    ]
}
EOF

    [6] generate etcd certificate server 

    [root@master etcd-cert]#cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server

    [7] etcd binary upload 

    [root@master k8s]# ls
etcd-cert  etcd-v3.3.10-linux-amd64  etcd-v3.3.10-linux-amd64.tar.gz

    [8] create configuration file, command file, certificate 

    [root@master k8s]# mkdir -p /opt/etcd/{cfg,bin,ssl}
//命令文件
[root@master k8s]# cp etcd-v3.3.10-linux-amd64/etcd etcd-v3.3.10-linux-amd64/etcdctl /opt/etcd/bin/
//证书
[root@master k8s]# cp etcd-cert/*.pem /opt/etcd/ssl/
//上传etcd.sh脚本，配置文件的生成以及systemctl管理服务文件生成
[root@master k8s]# ls
etcd-cert  etcd.sh  etcd-v3.3.10-linux-amd64  etcd-v3.3.10-linux-amd64.tar.gz
[root@master k8s]#sh etcd.sh etcd01 192.168.60.10 etcd02=https://192.168.60.60:2380,etcd03=https://192.168.60.100:2380
//查看etcd的进程是否启动
[root@master ~]# ps -ef | grep etcd

    [9] use another terminal to copy the certificate and systemctl management service script to other nodes 

    [root@master ~]# scp -r /opt/etcd/ [email protected]:/opt/
[root@master ~]# scp -r /opt/etcd/ [email protected]:/opt/
//启动脚本拷贝到其他节点
scp /usr/lib/systemd/system/etcd.service [email protected]:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/etcd.service [email protected]:/usr/lib/systemd/system/

    [10] modify the configuration file under CFG in the other two nodes 

    //在192.168.60.60节点修改,主要是修改name和IP地址
[root@node1 ~]# cd /opt/etcd/cfg/
[root@node1 cfg]# ls
etcd
[root@node1 cfg]# vim etcd 
#[Member]
ETCD_NAME="etcd02"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.60.60:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.60.60:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.60.60:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.60.60:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.60.10:2380,etcd02=https://192.168.60.60:2380,etcd03=https://192.168.60.100:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
[root@node1 cfg]# systemctl start etcd.service 
[root@node1 cfg]# systemctl status etcd.service


//在192.168.60.100节点修改,主要是修改name和IP地址
[root@node2 ~]# cd /opt/etcd/cfg/
[root@node2 cfg]# ls
etcd
[root@node2 cfg]# vim etcd 
#[Member]
ETCD_NAME="etcd03"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.60.100:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.60.100:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.60.100:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.60.100:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.60.10:2380,etcd02=https://192.168.60.60:2380,etcd03=https://192.168.60.100:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
[root@node2 cfg]# systemctl start etcd.service 
[root@node2 cfg]# systemctl status etcd.service

    [11] check whether the cluster state is healthy 

    [root@master etcd-cert]# /opt/etcd//bin/etcdctl --ca-file=ca.pem --cert-file=server.pem --key-file=server-key.pem --endpoint="https://192.168.60.10:2379,https://192.168.60.60:2379,https://192.168.60.100:2379" cluster-health
member 59173e3f8aecc6c3 is healthy: got healthy result from https://192.168.60.100:2379
member 8da25ad72397ec6e is healthy: got healthy result from https://192.168.60.10:2379
member a21e580b9191cb20 is healthy: got healthy result from https://192.168.60.60:2379
cluster is healthy
[root@master etcd-cert]#

    — — — — — — — — — — — — — — — — — — — —

    k8s multi-node deployment of flannel network configuration deployment

    1. Project demand analysis: 192.168.60.100 is the node1 node kubelet; kubelet kube-proxy docker flannel etcd
    [3] 192.168.60.60 is the node2 node kubelet kube-proxy docker flannel etcd

    Ii. Project step deployment:

    [1] write the allocated subnet segment into the etcd for flannel to use 

    [root@master etcd-cert]# /opt/etcd/bin/etcdctl \
--ca-file=ca.pem \
--cert-file=server.pem \
--key-file=server-key.pem \
--endpoint="https://192.168.60.10:2379,https://192.168.60.60:2379,https://192.168.60.100:2379" \
set /coreos.com/network/config '{"Network":"172.17.0.0/16","Backenf":{"Type":"vxlan"}}'

    [2] view the written information 

    [root@master etcd-cert]# /opt/etcd/bin/etcdctl \
--ca-file=ca.pem \
--cert-file=server.pem \
--key-file=server-key.pem \
--endpoint="https://192.168.60.10:2379,https://192.168.60.60:2379,https://192.168.60.100:2379" \
get /coreos.com/network/config

    [3] deploy flannel component

    on all node nodes

    //在192.168.60.60节点
[root@node1 ~]# tar zxvf flannel-v0.10.0-linux-amd64.tar.gz 
flanneld
mk-docker-opts.sh
README.md
//在192.168.60.100节点
[root@node2 ~]# tar zxvf flannel-v0.10.0-linux-amd64.tar.gz 
flanneld
mk-docker-opts.sh
README.md

    [4] create k8s working directory, copy the command file 

    //在192.168.60.60节点下
[root@node1 ~]# mkdir -p /opt/kubernetes/{cfg,bin,ssl}
[root@node1 ~]# mv flanneld mk-docker-opts.sh /opt/kubernetes/bin/
//在192.168.60.100节点下
[root@node2 ~]# mkdir -p /opt/kubernetes/{cfg,bin,ssl}
[root@node2 ~]# mv flanneld mk-docker-opts.sh /opt/kubernetes/bin/

    [5] write the flannel component startup execution script [node node is the same] 

    [root@node1 ~]# vim flannel.sh
#!/bin/bash
ETCD_ENDPOINTS=${1:-"http://127.0.0.1:2379"}
cat <<EOF >/opt/kubernetes/cfg/flanneld
FLANNEL_OPTIONS="--etcd-endpoints=${ETCD_ENDPOINTS} \
-etcd-cafile=/opt/etcd/ssl/ca.pem \
-etcd-certfile=/opt/etcd/ssl/server.pem \
-etcd-keyfile=/opt/etcd/ssl/server-key.pem"
EOF

cat <<EOF >/usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/opt/kubernetes/cfg/flanneld
ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable flanneld
systemctl restart flanneld

    [6] enable the flannel component network function 

    [root@node1 ~]# sh flannel.sh https://192.168.60.10:2379,https://192.168.60.60:2379,https://192.168.60.100:2379

    [7] configure docker to connect flannel component [all node nodes are the same] 

    [root@node1 ~]# vim /usr/lib/systemd/system/docker.service 
14 EnvironmentFile=/run/flannel/subnet.env
15 ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS -H fd:// --containerd=/run/containerd/contain    erd.sock

    [8] view the subnet

    specified for bip startup

    //在192.168.60.60节点node1
[root@node1 ~]# cat /run/flannel/subnet.env 
DOCKER_OPT_BIP="--bip=172.17.39.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=false"
DOCKER_OPT_MTU="--mtu=1472"
DOCKER_NETWORK_OPTIONS=" --bip=172.17.39.1/24 --ip-masq=false --mtu=1472"
//在192.168.60.100节点node2
[root@node2 ~]# cat /run/flannel/subnet.env 
DOCKER_OPT_BIP="--bip=172.17.85.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=false"
DOCKER_OPT_MTU="--mtu=1472"
DOCKER_NETWORK_OPTIONS=" --bip=172.17.85.1/24 --ip-masq=false --mtu=1472"

    [9] restart docker service 

    [root@node1 ~]# systemctl daemon-reload 
[root@node1 ~]# systemctl restart docker.service

    [10] see flannel network 

    //在node1节点192.168.60.60
[root@node1 ~]# ifconfig 
docker0: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500
        inet 172.17.39.1  netmask 255.255.255.0  broadcast 172.17.39.255
        ether 02:42:b1:19:5b:a1  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
//在node2节点192.168.60.100
[root@node2 ~]# ifconfig 
docker0: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500
        inet 172.17.85.1  netmask 255.255.255.0  broadcast 172.17.85.255
        ether 02:42:b5:54:91:f1  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

    [11] test the connectivity between nodes

    // at 192.168.60.60 node 

    [root@node1 ~]# docker run -it centos:7 /bin/bash
[root@2bbac9ebdc96 /]# yum install -y net-tools
[root@2bbac9ebdc96 /]# ifconfig 
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1472
        inet 172.17.39.2  netmask 255.255.255.0  broadcast 172.17.39.255
        ether 02:42:ac:11:27:02  txqueuelen 0  (Ethernet)
        RX packets 15198  bytes 12444271 (11.8 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 7322  bytes 398889 (389.5 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
[root@2bbac9ebdc96 /]# ping 172.17.85.2
PING 172.17.85.2 (172.17.85.2) 56(84) bytes of data.
64 bytes from 172.17.85.2: icmp_seq=1 ttl=60 time=1.08 ms
64 bytes from 172.17.85.2: icmp_seq=2 ttl=60 time=0.523 ms
64 bytes from 172.17.85.2: icmp_seq=3 ttl=60 time=0.619 ms
64 bytes from 172.17.85.2: icmp_seq=4 ttl=60 time=2.24 ms

    // at 192.168.60.100 node 

    [root@node2 ~]# docker run -it centos:7 /bin/bash
[root@79995e04b320 /]# yum install -y net-tools
[root@79995e04b320 /]# ifconfig 
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1472
        inet 172.17.85.2  netmask 255.255.255.0  broadcast 172.17.85.255
        ether 02:42:ac:11:55:02  txqueuelen 0  (Ethernet)
        RX packets 15299  bytes 12447552 (11.8 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 5864  bytes 320081 (312.5 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
[root@79995e04b320 /]# ping 172.17.39.2
PING 172.17.39.2 (172.17.39.2) 56(84) bytes of data.
64 bytes from 172.17.39.2: icmp_seq=1 ttl=60 time=0.706 ms
64 bytes from 172.17.39.2: icmp_seq=2 ttl=60 time=0.491 ms
64 bytes from 172.17.39.2: icmp_seq=3 ttl=60 time=0.486 ms
64 bytes from 172.17.39.2: icmp_seq=4 ttl=60 time=0.528 ms