CentOS7使用KubeSpray搭建单节点K8S集群
程序员文章站
2022-03-07 11:12:42
...
主机预部署
- 依据《CentOS7实验机模板搭建部署》克隆实验机kubespray 192.168.77.10
# 主机名和hosts解析
HOSTNAME=kubespray
hostnamectl set-hostname "$HOSTNAME"
echo "$HOSTNAME">/etc/hostname
echo "$(grep -E '127|::1' /etc/hosts)">/etc/hosts
echo "$(ip a|grep "inet "|grep -v 127|awk -F'[ /]' '{print $6}') $HOSTNAME">>/etc/hosts
# 关闭swap
swapoff -a
sed -i 's/^.*swap.*$/###&/g' /etc/fstab
# 添加br_netfilter模块
lsmod |grep br_netfilter
# 查看模块是否存在
modprobe br_netfilter
cat > /etc/rc.sysinit << EOF
#!/bin/bash
for file in /etc/sysconfig/modules/*.modules
do
[ -x $file ] && $file
done
EOF
cat > /etc/sysconfig/modules/br_netfilter.modules << EOF
modprobe br_netfilter
EOF
chmod 755 /etc/sysconfig/modules/br_netfilter.modules
# 优化内核参数
cd /tmp
cat > kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
net.ipv4.neigh.default.gc_thresh1=1024
net.ipv4.neigh.default.gc_thresh1=2048
net.ipv4.neigh.default.gc_thresh1=4096
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
/bin/cp -av kubernetes.conf /etc/sysctl.d/kubernetes.conf
sysctl -p /etc/sysctl.d/kubernetes.conf
# 安装软件包
cd /tmp
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
yum clean all && yum makecache faster
yum -y install conntrack ipvsadm ipset jq iptables curl sysstat libseccomp wget socat git
# 升级内核
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
yum --enablerepo=elrepo-kernel install -y kernel-lt
grep initrd16 /boot/grub2/grub.cfg
grub2-set-default 0
reboot
环境部署和集群配置修改
# 配置ssh免密登录
ssh-******
ssh-copy-id -o StrictHostKeyChecking=no 127.0.0.1
ssh -o StrictHostKeyChecking=no $(hostname) hostname
# 安装ansible环境
cd /tmp
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
yum clean all && yum makecache faster
yum -y install python36 python36-pip ansible
pip3 install -i https://mirrors.aliyun.com/pypi/simple/ --upgrade pip Jinja2
# 下载配置安装
cd /opt
wget https://github.com/kubernetes-sigs/kubespray/archive/v2.12.3.tar.gz
tar -xf v2.12.3.tar.gz
cd kubespray-2.12.3
pip3 install -i https://mirrors.aliyun.com/pypi/simple/ -r requirements.txt
cp -rfp inventory/sample inventory/mycluster
declare -a IPS=($(hostname -i))
CONFIG_FILE=inventory/mycluster/hosts.yaml python3 contrib/inventory_builder/inventory.py ${IPS[@]}
sed -i "s/node1/k8s-single/g" inventory/mycluster/hosts.yaml
cat inventory/mycluster/hosts.yaml
# 将k8s安装到本机之上，默认节点名是node1，改成k8s-single，该操作最终会造成主机名的更换
# ./roles/container-engine/docker/defaults/main.yml
# 该文件记录了docker的安装信息，诸如版本、相应系统的软件包下载url：
# docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/7/$basearch/stable'
# docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg'
# extras_rh_repo_base_url: "http://mirror.centos.org/centos/$releasever/extras/$basearch/"
# extras_rh_repo_gpgkey: "http://mirror.centos.org/centos/RPM-GPG-KEY-CentOS-7"
# 可以手动安装相应版本docker，建议手动安装并做下载优化，或者提前准备好优化配置：
mkdir -p /etc/docker
cat >/etc/docker/daemon.json<<EOF
{
"registry-mirrors": ["https://cjw7u3gx.mirror.aliyuncs.com","https://docker.mirrors.ustc.edu.cn","http://hub-mirror.c.163.com"]
}
EOF
cd /opt/kubespray-2.12.3/roles/container-engine/docker/defaults
sed -i 's|download.docker.com/linux/centos/7|mirrors.aliyun.com/docker-ce/linux/centos/7|g' main.yml
sed -i 's|download.docker.com/linux/centos/gpg|mirrors.aliyun.com/docker-ce/linux/centos/gpg|g' main.yml
sed -i 's|mirror.centos.org/centos|mirrors.aliyun.com/centos|g' main.yml
# ./roles/download/defaults/main.yml
# 该文件记录了kubelet、kubectl、kubeadm、etcd、cni、calicoctl和crictl的版本和下载地址，这些地址均是不可达的
# 该文件还记录了k8s集群所需要的镜像仓库和tag，依旧是不可达的
# 这两类不可达的软件下载是使用kubespray安装k8s集群的核心问题
# 手动下载以下安装包，在本机之上创建一个nginx共享这些包的下载：
# kubelet_download_url: "https://storage.googleapis.com/kubernetes-release/release/v1.16.7/bin/linux/amd64/kubelet"
# kubectl_download_url: "https://storage.googleapis.com/kubernetes-release/release/v1.16.7/bin/linux/amd64/kubectl"
# kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/v1.16.7/bin/linux/amd64/kubeadm"
# etcd_download_url: "https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz"
# cni_download_url: "https://github.com/containernetworking/plugins/releases/download/v0.8.1/cni-plugins-linux-amd64-v0.8.1.tgz"
# calicoctl_download_url: "https://github.com/projectcalico/calicoctl/releases/download/v3.7.3/calicoctl-linux-amd64"
# crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.16.1/crictl-v1.16.1-linux-amd64.tar.gz"
cd /tmp
yum -y install nginx
sed -i 's/80 default_server/60000 default_server/g' /etc/nginx/nginx.conf
cd /usr/share/nginx/html
rm -rf *
tar -xf /tmp/手动下载的全部包.tar.gz
systemctl start nginx && systemctl enable nginx
# 测试下载
mkdir /tmp/test && cd /tmp/test
wget "http://$(hostname -i):60000/kubelet"
wget "http://$(hostname -i):60000/kubectl"
wget "http://$(hostname -i):60000/kubeadm"
wget "http://$(hostname -i):60000/etcd-v3.3.10-linux-amd64.tar.gz"
wget "http://$(hostname -i):60000/cni-plugins-linux-amd64-v0.8.1.tgz"
wget "http://$(hostname -i):60000/calicoctl-linux-amd64"
wget "http://$(hostname -i):60000/crictl-v1.16.1-linux-amd64.tar.gz"
cd /tmp && rm -rf /tmp/test
# 修改配置
cd /opt/kubespray-2.12.3/roles/download/defaults/
sed -i "s|^kubelet_download_url.*$|#&\nkubelet_download_url: 'http://$(hostname -i):60000/kubelet'|g" main.yml
sed -i "s|^kubectl_download_url.*$|#&\nkubectl_download_url: 'http://$(hostname -i):60000/kubectl'|g" main.yml
sed -i "s|^kubeadm_download_url.*$|#&\nkubeadm_download_url: 'http://$(hostname -i):60000/kubeadm'|g" main.yml
sed -i "s|^etcd_download_url.*$|#&\netcd_download_url: 'http://$(hostname -i):60000/etcd-v3.3.10-linux-amd64.tar.gz'|g" main.yml
sed -i "s|^cni_download_url.*$|#&\ncni_download_url: 'http://$(hostname -i):60000/cni-plugins-linux-amd64-v0.8.1.tgz'|g" main.yml
sed -i "s|^calicoctl_download_url.*$|#&\ncalicoctl_download_url: 'http://$(hostname -i):60000/calicoctl-linux-amd64'|g" main.yml
sed -i "s|^crictl_download_url.*$|#&\ncrictl_download_url: 'http://$(hostname -i):60000/crictl-v1.16.1-linux-amd64.tar.gz'|g" main.yml
# 对文件进行整理，整理出所有的镜像信息
cd /opt/kubespray-2.12.3/roles/download/defaults/
for i in $(cat main.yml |grep 'version }}"'|awk -F'{{' '{print $2}'|awk '{print $1}')
do
info=$(grep ^${i} main.yml|sed 's/"//g')
Name=$(echo ${info}|awk -F':' '{print $1}')
Vers=$(echo ${info}|awk -F': ' '{print $2}')
echo "sed -i 's|{{ ${Name} }}|${Vers}|g' main.yml"
done>/tmp/change.sh
bash /tmp/change.sh
sed -i 's/{{ kube_version | regex_replace.*"/v1.16"/g' main.yml
sed -i 's|^crictl_version.*|crictl_version: "v1.16.1"|g' main.yml
for i in docker_image_repo kube_image_repo quay_image_repo gcr_image_repo
do
info=$(grep ^${i} main.yml|sed 's/"//g')
Name=$(echo ${info}|awk -F':' '{print $1}')
Vers=$(echo ${info}|awk -F': ' '{print $2}')
echo "sed -i 's|{{ ${Name} }}|${Vers}|g' main.yml"
done>/tmp/change.sh
bash /tmp/change.sh
for i in $(cat main.yml |grep 'repo }}"'|awk -F'{{' '{print $2}'|awk '{print $1}')
do
info=$(grep ^${i} main.yml|sed 's/"//g')
Name=$(echo ${info}|awk -F':' '{print $1}')
Vers=$(echo ${info}|awk -F': ' '{print $2}')
echo "sed -i 's|{{ ${Name} }}|${Vers}|g' main.yml"
done>/tmp/change.sh
bash /tmp/change.sh
sed -i 's|^etcd_image_tag:.*|etcd_image_tag: "v3.3.10"|g' main.yml
for i in $(cat main.yml |grep 'tag }}"'|awk -F'{{' '{print $2}'|awk '{print $1}')
do
info=$(grep ^${i} main.yml|sed 's/"//g')
Name=$(echo ${info}|awk -F':' '{print $1}')
Vers=$(echo ${info}|awk -F': ' '{print $2}')
echo "sed -i 's|{{ ${Name} }}|${Vers}|g' main.yml"
done>/tmp/change.sh
bash /tmp/change.sh
sed -i 's|{{ image_arch }}|amd64|g' main.yml
sed -i 's|^kube_proxy_image_repo.*$|kube_proxy_image_repo: "registry.cn-hangzhou.aliyuncs.com/vinc-k8s"|g' main.yml
# 拼出来所需的image镜像信息，其中docker.io中的镜像无需关注
# 近期发现docker.io之上的calico也是需要单独加速处理的，这也是近期该工具一直不好使的原因
for i in $(cat main.yml |grep tag:|grep -v ' tag'|awk -F'_tag:' '{print $1}')
do
Repo=$(grep ${i}_repo main.yml|sed 's/"//g'|awk '{print $2}')
Tag=$(grep ${i}_tag main.yml|sed 's/"//g'|awk '{print $2}')
echo "${Repo}:${Tag}"
done|sort|grep -v 'docker.io'>/tmp/image.txt
# 参照《基于阿里云容器镜像服务加速K8S镜像下载》，将镜像上传到阿里云，最后修改配置文件
sed -i 's|gcr.io/google-containers/addon-resizer|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/addon-resizer|g' main.yml
sed -i 's|gcr.io/google-containers/cluster-proportional-autoscaler-amd64|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/cluster-proportional-autoscaler-amd64|g' main.yml
sed -i 's|gcr.io/google-containers/k8s-dns-node-cache|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/k8s-dns-node-cache|g' main.yml
sed -i 's|gcr.io/google_containers/kube-registry-proxy|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/kube-registry-proxy|g' main.yml
sed -i 's|gcr.io/google_containers/kubernetes-dashboard-amd64|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/kubernetes-dashboard-amd64|g' main.yml
sed -i 's|gcr.io/google_containers/metrics-server-amd64|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/metrics-server-amd64|g' main.yml
sed -i 's|gcr.io/google_containers/pause-amd64|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/pause-amd64|g' main.yml
sed -i 's|gcr.io/kubernetes-helm/tiller|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/tiller|g' main.yml
sed -i 's|quay.io/coreos/etcd|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/quay.io-coreos-etcd|g' main.yml
sed -i 's|quay.io/coreos/flannel-cni|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/flannel-cni|g' main.yml
sed -i 's|quay.io/coreos/flannel|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/flannel|g' main.yml
sed -i 's|quay.io/external_storage/cephfs-provisioner|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/cephfs-provisioner|g' main.yml
sed -i 's|quay.io/external_storage/local-volume-provisioner|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/local-volume-provisioner|g' main.yml
sed -i 's|quay.io/external_storage/rbd-provisioner|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/rbd-provisioner|g' main.yml
sed -i 's|quay.io/jetstack/cert-manager-controller|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/cert-manager-controller|g' main.yml
sed -i 's|quay.io/kubernetes-ingress-controller/nginx-ingress-controller|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/nginx-ingress-controller|g' main.yml
sed -i 's|quay.io/l23network/k8s-netchecker-agent|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/k8s-netchecker-agent|g' main.yml
sed -i 's|quay.io/l23network/k8s-netchecker-server|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/k8s-netchecker-server|g' main.yml
sed -i 's|docker.io/calico/cni|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/docker.io-calico-cni|g' main.yml
sed -i 's|docker.io/calico/kube-controllers|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/docker.io-calico-kube-controllers|g' main.yml
sed -i 's|docker.io/calico/node|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/docker.io-calico-node|g' main.yml
sed -i 's|docker.io/calico/routereflector|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/docker.io-calico-routereflector|g' main.yml
sed -i 's|docker.io/calico/typha|registry.cn-hangzhou.aliyuncs.com/vinc-k8s/docker.io-calico-typha|g' main.yml
# 默认安装的dashboard版本存在兼容性BUG，改为最新版本的 kubernetesui/dashboard:v2.0.0-rc5
# 该BUG会导致访问dashboard报错：浏览器报错 未知服务器错误 (404)
sed -i 's|dashboard_image_tag: "v1.10.1"|dashboard_image_tag: "v2.0.0-rc5"|g' main.yml
# 还有另外一个文件中的镜像
cd /opt/kubespray-2.12.3/inventory/mycluster/group_vars/k8s-cluster
sed -i 's|{{ gcr_image_repo }}/google-containers|registry.cn-hangzhou.aliyuncs.com/vinc-k8s|g' k8s-cluster.yml
集群安装测试和卸载
cd /opt/kubespray-2.12.3
ansible-playbook -i inventory/mycluster/hosts.yaml --become --become-user=root cluster.yml
kubectl get node --all-namespaces -o wide
kubectl get pods --all-namespaces -o wide
kubectl describe pod coredns-xxxx-xxxx --namespace kube-system
kubectl logs -f pods/coredns-xxxx-xxxx -n kube-system
# 单节点的k8s集群中会有一个coredns处于故障状态
kubectl describe pod kubernetes-dashboard-xxxx-dncrl --namespace kube-system
kubectl logs -f pods/kubernetes-dashboard-xxxx-dncrl -n kube-system
kubectl cluster-info
kubectl -n kube-system describe $(kubectl -n kube-system get secret -n kube-system -o name | grep namespace) | grep ^token
# 浏览器访问 https://192.168.199.10:6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy
# 需要贴入提取到的token才能最终授权访问
# 集群卸载：
cd /opt/kubespray-2.12.3
ansible-playbook -i inventory/mycluster/hosts.yaml --become --become-user=root reset.yml
镜像梳理
# 最后查看镜像，下载的镜像列表为：
nginx:1.17
coredns/coredns:1.6.0
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/quay.io-coreos-etcd:v3.3.10
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/pause-amd64:3.1
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/pause:3.1
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/kube-scheduler:v1.16.7
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/kubernetes-dashboard-amd64:v2.0.0-rc5
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/kube-proxy:v1.16.7
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/kube-controller-manager:v1.16.7
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/kube-apiserver:v1.16.7
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/k8s-dns-node-cache:1.15.8
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/docker.io-calico-node:v3.7.3
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/docker.io-calico-kube-controllers:v3.7.3
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/docker.io-calico-cni:v3.7.3
registry.cn-hangzhou.aliyuncs.com/vinc-k8s/cluster-proportional-autoscaler-amd64:1.6.0
[TOC]