CentOS7使用KubeSpray搭建多节点K8S集群

kubespray主机搭建

• 依据《CentOS7使用KubeSpray搭建单节点K8S集群》的环境部署和集群配置修改部分搭建kubespray主机 192.168.77.10

HOSTNAME=kubespray
hostnamectl set-hostname "$HOSTNAME"
echo "$HOSTNAME">/etc/hostname
echo "$(grep -E '127|::1' /etc/hosts)">/etc/hosts
echo "$(ip a|grep "inet "|grep -v 127|awk -F'[ /]' '{print $6}') $HOSTNAME">>/etc/hosts

集群节点列表

• 依据《CentOS7实验机模板搭建部署》克隆多台实验机，作为k8s集群的安装主机，数目随意，这里部署7节点的k8s集群环境

kubespray 192.168.77.10  2CPU,4G内存
master01  192.168.77.101 2CPU,4G内存
master01  192.168.77.102 2CPU,4G内存
master01  192.168.77.103 2CPU,4G内存
worker01  192.168.77.111 2CPU,8G内存
worker02  192.168.77.112 2CPU,8G内存
worker03  192.168.77.113 2CPU,8G内存
worker04  192.168.77.114 2CPU,8G内存

对kubespray主机之外的实验机进行预部署

# 如果实验机为虚拟机，确保各个节点product_uuid和mac唯一
cat /sys/class/dmi/id/product_uuid
cat /sys/class/net/enp0s3/address

# 主机名和hosts解析
HOSTNAME=master01
# HOSTNAME=master02
# HOSTNAME=master03
# HOSTNAME=worker01
# HOSTNAME=worker02
# HOSTNAME=worker03
# HOSTNAME=worker04
hostnamectl set-hostname "$HOSTNAME"
echo "$HOSTNAME">/etc/hostname
echo "$(grep -E '127|::1' /etc/hosts)">/etc/hosts
echo "$(ip a|grep "inet "|grep -v 127|awk -F'[ /]' '{print $6}') $HOSTNAME">>/etc/hosts

# 依据《CentOS7使用KubeSpray搭建单节点K8S集群》的主机预部署对各个节点进行部署

# 依据《CentOS7部署安装Docker和Docker Compose工具简录》部署安装docker-18.09.7，并做优化加速
yum -y install docker-ce-18.09.7

# 依据《CentOS7使用KubeSpray搭建单节点K8S集群》的镜像梳理手动预先下载镜像
# 在一个节点上下载，另外节点装载即可
cd /tmp
cat >/tmp/image-list.txt<<EOF
nginx:1.17
coredns/coredns:1.6.0
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/quay.io-coreos-etcd:v3.3.10
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/pause-amd64:3.1
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/pause:3.1
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/kube-scheduler:v1.16.7
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/kubernetes-dashboard-amd64:v2.0.0-rc5
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/kube-proxy:v1.16.7
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/kube-controller-manager:v1.16.7
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/kube-apiserver:v1.16.7
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/k8s-dns-node-cache:1.15.8
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/docker.io-calico-node:v3.7.3
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/docker.io-calico-kube-controllers:v3.7.3
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/docker.io-calico-cni:v3.7.3
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/cluster-proportional-autoscaler-amd64:1.6.0
EOF
mkdir image
for i in $(cat /tmp/image-list.txt)
do
  Name=$(basename $(echo $i|awk -F':' '{print $1}'))
  docker pull $i
  docker save $i -o /tmp/image/${Name}.tar
done
tar -czf image.tar.gz image

for i in 102 103 111 112 113 114
do
  scp -r /tmp/image.tar.gz 192.168.77.${i}:/tmp/
  ssh -o StrictHostKeyChecking=no 192.168.77.${i} \
    'cd /tmp/&& tar -xf image.tar.gz && cd /tmp/image && for i in $(ls);do docker load<$i;done'
done
for i in 101 102 103 111 112 113 114
do
  ssh 192.168.77.${i} 'rm -rf /tmp/image*'
done

操纵kubespray主机进行集群安装

# 追加配置到hosts文件
cat >>/etc/hosts<<EOF
192.168.77.101    master01
192.168.77.102    master02
192.168.77.103    master03
192.168.77.111    worker01
192.168.77.112    worker02
192.168.77.113    worker03
192.168.77.114    worker04
EOF

# 配置ssh免密登录
ssh-******
ssh-copy-id -o StrictHostKeyChecking=no 127.0.0.1
ssh -o StrictHostKeyChecking=no $(hostname) hostname
for i in $(cat /etc/hosts|grep -E 'master|worker'|awk '{print $2}')
do
  ssh-copy-id -o StrictHostKeyChecking=no $i
  ssh -o StrictHostKeyChecking=no $i hostname
done

# 集群安装
cd /opt/kubespray-2.12.3
/bin/cp -rfp inventory/sample inventory/mycluster
declare -a IPS=($(cat /etc/hosts|grep -E 'master|worker'|awk '{printf $1" "}'))
rm -rf inventory/mycluster/hosts.yaml
CONFIG_FILE=inventory/mycluster/hosts.yaml python3 contrib/inventory_builder/inventory.py ${IPS[@]}
sed -i "s/node1/master01/g" inventory/mycluster/hosts.yaml
sed -i "s/node2/master02/g" inventory/mycluster/hosts.yaml
sed -i "s/node3/master03/g" inventory/mycluster/hosts.yaml
sed -i "s/node4/worker01/g" inventory/mycluster/hosts.yaml
sed -i "s/node5/worker02/g" inventory/mycluster/hosts.yaml
sed -i "s/node6/worker03/g" inventory/mycluster/hosts.yaml
sed -i "s/node7/worker04/g" inventory/mycluster/hosts.yaml
# 手动修改hosts.yaml文件，在kube-master中加入master03，将kube-node中master节点剔除

# 最后执行集群安装
cd /opt/kubespray-2.12.3
ansible-playbook -i inventory/mycluster/hosts.yaml --become --become-user=root cluster.yml

# 在任意一个master节点上验证
ssh master01
kubectl get node --all-namespaces -o wide
kubectl get pods --all-namespaces -o wide
kubectl cluster-info
kubectl -n kube-system describe $(kubectl -n kube-system get secret -n kube-system -o name | grep namespace) | grep ^token
exit
# 浏览器访问 https://192.168.77.101:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy
# 需要贴入提取到的token才能最终授权访问

TOC