欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识！
您现在的位置是： 首页

二进制文件方式安装kubernetes集群

程序员文章站 2024-03-12 12:09:32
...

所有操作全部用root使用者进行，高可用一般建议大于等于3台的奇数,我们使用3台master来做高可用

练习环境说明： 参考GitHub

master： kube-apiserver，kube-controller-manager，kube-scheduler，flanneld

node： kubelet，kube-proxy，flannel

Service_CIDR：10.254.0.0/16 服务网段，部署前路由不可达，部署后集群内部使用IP:Port可达

Cluster_CIDR：172.30.0.0/16 pod网段，部署前路由不可达，部署后路由可达(flanneld 保证)

主机名称 IP地址 部署软件 备注
k8s-m12 192.168.10.12 keepalived+haproxy+etcd+master master
k8s-m13 192.168.10.13 keepalived+haproxy+etcd+master master
k8s-m14 192.168.10.14 keepalived+haproxy+etcd+master master
k8s-n15 192.168.10.15 node+docker work
k8s-n16 192.168.10.16 node+docker work
VIP 192.168.10.100 VIP

2.1、下载安装包

kubernetes的GitHub网址 ： https://github.com/kubernetes/kubernetes/releases

下载Server Binaries中的 kubernetes-server-linux-amd64.tar.gz 安装包

下载Node Binaries中的 kubernetes-node-linux-amd64.tar.gz 安装包

下载Client Binares中的 kubernetes-client-linux-amd64.tar.gz 安装包

各种CA证书类型参考

k8s各版本组件下载地址

https://github.com/kubernetes/kubernetes/tree/v1.14.3

#kubernetes
wget https://storage.googleapis.com/kubernetes-release/release/v1.14.3/kubernetes-node-linux-amd64.tar.gz
wget https://storage.googleapis.com/kubernetes-release/release/v1.14.3/kubernetes-client-linux-amd64.tar.gz
wget https://storage.googleapis.com/kubernetes-release/release/v1.14.3/kubernetes-server-linux-amd64.tar.gz
wget https://storage.googleapis.com/kubernetes-release/release/v1.14.3/kubernetes.tar.gz
#etcd
wget https://github.com/etcd-io/etcd/releases/download/v3.3.13/etcd-v3.3.13-linux-amd64.tar.gz
#flannel
wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz
#cni-plugins
wget https://github.com/containernetworking/plugins/releases/download/v0.8.1/cni-plugins-linux-amd64-v0.8.1.tgz
#docker
wget https://download.docker.com/linux/static/stable/x86_64/docker-18.09.6.tgz
#cfssl
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
#heapster
wget https://github.com/kubernetes-retired/heapster/archive/v1.5.4.tar.gz

2.2、环境准备

#1.12机器上生成**，无密码ssh登陆
ssh-****** -t rsa
ssh-copy-id 192.168.10.13  #依次拷贝到其他节点上

#2.关闭防火墙，以下点所有机器执行
systemctl stop firewalld
systemctl disable firewalld

#3.关闭swap分区
swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

#4.关闭SELinux
sestatus    #查看selinux状态
setenforce 0        #临时关闭selinux
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

#5.升级内核参考：https://www.cnblogs.com/fan-gx/p/11006762.html

#6.修改文件句柄数
cat <<EOF >>/etc/security/limits.conf
* soft nofile 65536
* hard nofile 65536
* soft nproc 65536
* hard nproc 65536
* soft  memlock  unlimited
* hard memlock  unlimited
EOF

#7.安装ipvs
yum install ipvsadm ipset sysstat conntrack libseccomp -y
#开机加载内核模块，并设置开机自动加载
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF

#然后执行脚本
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules
lsmod | grep -e ip_vs -e nf_conntrack_ipv4

#8.修改系统参数
cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
vm.swappiness=0
EOF
sysctl --system
#-----------下面参考别人的---------#
# cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1
net.ipv4.neigh.default.gc_stale_time = 120
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
net.ipv4.ip_forward = 1
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.netfilter.nf_conntrack_max = 2310720
fs.inotify.max_user_watches=89100
fs.may_detach_mounts = 1
fs.file-max = 52706963
fs.nr_open = 52706963
net.bridge.bridge-nf-call-arptables = 1
vm.swappiness = 0
vm.overcommit_memory=1
vm.panic_on_oom=0
EOF

#9.在生产环境建议预留内存,避免由于内存耗尽导致ssh连不上主机（32G的机器留2G，251的留3G, 500G的留5G）。下面是预留5G
echo 'vm.min_free_kbytes=5000000' >> /etc/sysctl.conf
sysctl -p

2.3、部署docker

二进制部署方式可参考：https://www.kubernetes.org.cn/3831.html 这里为了方便直接yum安装所有节点

#1.安装yum源工具包
yum install -y yum-utils device-mapper-persistent-data lvm2

#2.下载docker-ce官方的yum源配置文件
# yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
# yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

#3.禁用docker-c-edge源配edge是不开发版，不稳定，下载stable版
yum-config-manager --disable docker-ce-edge
#4.更新本地YUM源缓存
yum makecache fast
#5.安装Docker-ce相应版本
yum -y install docker-ce
#6.配置daemon, 因为kubelet的启动环境变量要与docker的cgroup-driver驱动相同,以下是官方推荐处理方式（现在新版二进制kubelet就是cgroup了）
#由于国内拉取镜像较慢，配置文件最后追加了阿里云镜像加速配置。
mkdir -p /etc/docker && 
cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "storage-opts": [
    "overlay2.override_kernel_check=true"
  ],
  "registry-mirrors": ["https://uyah70su.mirror.aliyuncs.com"]
}
EOF
#7.设置开机自启动
systemctl restart docker && systemctl enable docker && systemctl status docker

#8.可以先在自己电脑下来安装包，本环境安装的是18.09版本
yum install --downloadonly docker-ce-18.09 --downloaddir=/opt   #yum下载docker-ce
yum localinstall docker-ce -y       #然后安装

2.4、部署etcd

etcd是用来保存集群所有状态的 Key/Value 存储系统，常用于服务发现、共享配置以及并发控制（如 leader 选举、分布式锁等）。kubernetes 使用 etcd 存储所有运行数据。

所有 Kubernetes 组件会通过 API Server 来跟 Etcd 进行沟通从而保存或读取资源状态。有条件的可以单独几台机器跑,不过需要配置apiserver指向etcd集群。

2.4.1、创建etcd证书

如果不希望将cfssl工具安装到部署主机上，可以在其他的主机上进行该步骤，生成以后将证书拷贝到部署etcd的主机上即可。不是要证书也可以部署，etcd.service文件和etcd.conf文件不要有https的URL

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O /usr/local/bin/cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O /usr/local/bin/cfssljson
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -O /usr/local/bin/cfssl-certinfo

chmod +x /usr/local/bin/cfssl*

#配置CA文件
mkdir /root/ssl && cd /root/ssl

cat >  ca-config.json <<EOF
{
"signing": {
"default": {
  "expiry": "8760h"
},
"profiles": {
  "kubernetes": {
    "usages": [
        "signing",
        "key encipherment",
        "server auth",
        "client auth"
    ],
    "expiry": "8760h"
  }
}
}
}
EOF

#2----------------------------------------------
cat >  ca-csr.json <<EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
  "C": "CN",
  "ST": "ShangHai",
  "L": "ShangHai",
  "O": "k8s",
  "OU": "System"
}
]
}
EOF

#3--------------------------------------------------
cat > etcd-csr.json <<EOF
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "192.168.10.12",
    "192.168.10.13",
    "192.168.10.14"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShangHai",
      "L": "ShangHai",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF
# hosts字段的IP地址是指授权使用证书的etcd地址
#------------------------------------
cfssl gencert -initca ca-csr.json | cfssljson -bare ca

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd

#生产后证书包含文件如下，共9个
ca-config.json
ca.csr
ca-csr.json
ca-key.pem
ca.pem
etcd.csr
etcd-csr.json
etcd-key.pem
etcd.pem

#将生成好的etcd.pem和etcd-key.pem以及ca.pem三个文件拷贝到etcd机器上
mkdir -p /etc/kubernetes/ssl && cp *.pem /etc/kubernetes/ssl/
ssh -n 192.168.10.13 "mkdir -p /etc/kubernetes/ssl && exit"
ssh -n 192.168.10.14 "mkdir -p /etc/kubernetes/ssl && exit"

scp -r /etc/kubernetes/ssl/*.pem 192.168.10.13:/etc/kubernetes/ssl/
scp -r /etc/kubernetes/ssl/*.pem 192.168.10.14:/etc/kubernetes/ssl/

2.4.2、部署etcd

将下载的etcd二进制文件上传到etcd节点机器上。

#在etcd的机器上安装etcd程序
mkdir -p /var/lib/etcd

tar -zxvf etcd-v3.3.13-linux-amd64.tar.gz
cp etcd-v3.3.13-linux-amd64/etcd* /usr/local/bin
scp etcd-v3.3.13-linux-amd64/etcd* 192.168.10.13:/usr/local/bin
scp etcd-v3.3.13-linux-amd64/etcd* 192.168.10.14:/usr/local/bin

#1.在12机器上创建etcd.service文件
cat <<EOF >/etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/local/bin/etcd \
  --name k8s-m12 \
  --cert-file=/etc/kubernetes/ssl/etcd.pem \
  --key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
  --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --initial-advertise-peer-urls https://192.168.10.12:2380 \
  --listen-peer-urls https://192.168.10.12:2380 \
  --listen-client-urls https://192.168.10.12:2379,http://127.0.0.1:2379 \
  --advertise-client-urls https://192.168.10.12:2379 \
  --initial-cluster-token etcd-cluster-0 \
  --initial-cluster k8s-m12=https://192.168.10.12:2380,k8s-m13=https://192.168.10.13:2380,k8s-m14=https://192.168.10.14:2380 \
  --initial-cluster-state new \
  --data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

#2.启动etcd服务
systemctl daemon-reload && systemctl enable etcd.service && systemctl start etcd.service && systemctl status etcd
#1.在13机器上创建etcd.service文件
cat <<EOF >/etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/local/bin/etcd \
  --name k8s-m13 \
  --cert-file=/etc/kubernetes/ssl/etcd.pem \
  --key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
  --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --initial-advertise-peer-urls https://192.168.10.13:2380 \
  --listen-peer-urls https://192.168.10.13:2380 \
  --listen-client-urls https://192.168.10.13:2379,http://127.0.0.1:2379 \
  --advertise-client-urls https://192.168.10.13:2379 \
  --initial-cluster-token etcd-cluster-0 \
  --initial-cluster k8s-m12=https://192.168.10.12:2380,k8s-m13=https://192.168.10.13:2380,k8s-m14=https://192.168.10.14:2380 \
  --initial-cluster-state new \
  --data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

#2.启动etcd服务
systemctl daemon-reload && systemctl enable etcd.service && systemctl start etcd.service && systemctl status etcd
#1.在14机器上创建etcd.service文件
cat <<EOF >/etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/local/bin/etcd \
  --name k8s-m14 \
  --cert-file=/etc/kubernetes/ssl/etcd.pem \
  --key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
  --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
  --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --initial-advertise-peer-urls https://192.168.10.14:2380 \
  --listen-peer-urls https://192.168.10.14:2380 \
  --listen-client-urls https://192.168.10.14:2379,http://127.0.0.1:2379 \
  --advertise-client-urls https://192.168.10.14:2379 \
  --initial-cluster-token etcd-cluster-0 \
  --initial-cluster k8s-m12=https://192.168.10.12:2380,k8s-m13=https://192.168.10.13:2380,k8s-m14=https://192.168.10.14:2380 \
  --initial-cluster-state new \
  --data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

#2.启动etcd服务
systemctl daemon-reload && systemctl enable etcd.service && systemctl start etcd.service && systemctl status etcd

2.4.3、验证集群

#1.查看集群状态
etcdctl --ca-file=/etc/kubernetes/ssl/ca.pem --cert-file=/etc/kubernetes/ssl/etcd.pem --key-file=/etc/kubernetes/ssl/etcd-key.pem cluster-health
#返回如下，代表集群正常
member 1af68d968c7e3f22 is healthy: got healthy result from https://192.168.10.12:2379
member 55204c19ed228077 is healthy: got healthy result from https://192.168.10.14:2379
member e8d9a97b17f26476 is healthy: got healthy result from https://192.168.10.13:2379
cluster is healthy

#2.查看集群成员
etcdctl --endpoints=https://192.168.10.12:2379,https://192.168.10.13:2379,https://192.168.10.14:2379  --ca-file=/etc/kubernetes/ssl/ca.pem  --cert-file=/etc/kubernetes/ssl/etcd.pem  --key-file=/etc/kubernetes/ssl/etcd-key.pem member list
#返回如下结果
1af68d968c7e3f22: name=k8s-m12 peerURLs=https://192.168.10.12:2380 clientURLs=https://192.168.10.12:2379 isLeader=false
55204c19ed228077: name=k8s-m14 peerURLs=https://192.168.10.14:2380 clientURLs=https://192.168.10.14:2379 isLeader=false
e8d9a97b17f26476: name=k8s-m13 peerURLs=https://192.168.10.13:2380 clientURLs=https://192.168.10.13:2379 isLeader=true

2.5、部署flannel

所有的节点都需要安装flannel，，主要目的是跨主机的docker能够互相通信，也是保障kubernetes集群的网络基础和保障

2.5.1、创建flannel证书

#1.生产TLS证书，是让kubectl当做client证书使用,(证书只需要生成一次)
cd /root/ssl
cat > flanneld-csr.json <<EOF
{
  "CN": "flanneld",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShangHai",
      "L": "ShangHai",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

#2.生成证书和私钥
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld
#包含以下文件
flanneld.csr
flanneld-csr.json
flanneld-key.pem
flanneld.pem

#3.然后将证书拷贝到所有节点下。
cp flanneld*.pem /etc/kubernetes/ssl
scp flanneld*.pem 192.168.10.13:/etc/kubernetes/ssl
scp flanneld*.pem 192.168.10.14:/etc/kubernetes/ssl
scp flanneld*.pem 192.168.10.15:/etc/kubernetes/ssl
scp flanneld*.pem 192.168.10.16:/etc/kubernetes/ssl

2.5.2、部署flannel

#1.开始安装flannel
tar -zvxf flannel-v0.11.0-linux-amd64.tar.gz
cp flanneld mk-docker-opts.sh /usr/local/bin
scp flanneld mk-docker-opts.sh 192.168.10.13:/usr/local/bin
scp flanneld mk-docker-opts.sh 192.168.10.14:/usr/local/bin
scp flanneld mk-docker-opts.sh 192.168.10.15:/usr/local/bin
scp flanneld mk-docker-opts.sh 192.168.10.16:/usr/local/bin

#2.向etcd写入集群Pod网段信息，在etcd集群中任意一台执行一次即可
etcdctl \
--endpoints=https://192.168.10.12:2379,https://192.168.10.13:2379,https://192.168.10.14:2379 \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--cert-file=/etc/kubernetes/ssl/flanneld.pem \
--key-file=/etc/kubernetes/ssl/flanneld-key.pem \
mk /kubernetes/network/config '{"Network":"172.30.0.0/16", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}'
#----得到返回信息如下，设置的网络是172.30.0.0/16，子网掩码是24位
{"Network":"172.30.0.0/16", "SubnetLen": 24, "Backend": {"Type": "vxlan"}}

#2.1.列出键值存储的目录
etcdctl \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--cert-file=/etc/kubernetes/ssl/flanneld.pem \
--key-file=/etc/kubernetes/ssl/flanneld-key.pem ls -r
#2.2.查看键值存储
etcdctl \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--cert-file=/etc/kubernetes/ssl/flanneld.pem \
--key-file=/etc/kubernetes/ssl/flanneld-key.pem get /kubernetes/network/config
#2.3查看已分配pod的子网列表
etcdctl \
--ca-file=/etc/kubernetes/ssl/ca.pem \
--cert-file=/etc/kubernetes/ssl/flanneld.pem \
--key-file=/etc/kubernetes/ssl/flanneld-key.pem ls  /kubernetes/network/subnets

#3、创建flannel.service文件
cat > /etc/systemd/system/flannel.service << EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service

[Service]
Type=notify
ExecStart=/usr/local/bin/flanneld \
  -etcd-cafile=/etc/kubernetes/ssl/ca.pem \
  -etcd-certfile=/etc/kubernetes/ssl/flanneld.pem \
  -etcd-keyfile=/etc/kubernetes/ssl/flanneld-key.pem \
  -etcd-endpoints=https://192.168.10.12:2379,https://192.168.10.13:2379,https://192.168.10.14:2379 \
  -etcd-prefix=/kubernetes/network
ExecStartPost=/usr/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=on-failure

[Install]
WantedBy=multi-user.target
RequiredBy=docker.service
EOF
#mk-docker-opts.sh 脚本将分配给flanneld的Pod子网网段信息写入到/run/flannel/docker文件中，后续docker启动时使用这个文件中参数值设置docker0网桥。
#flanneld 使用系统缺省路由所在的接口和其它节点通信，对于有多个网络接口的机器（如，内网和公网），可以用 -iface=enpxx 选项值指定通信接口。

#4、启动flannel
systemctl daemon-reload && systemctl enable flannel && systemctl start flannel && systemctl status flannel

#5.验证flannel
cat /run/flannel/docker             #/run/flannel/docker是flannel分配给docker的子网信息，
#显示如下
DOCKER_OPT_BIP="--bip=172.30.7.1/24"
DOCKER_OPT_IPMASQ="--ip-masq=true"
DOCKER_OPT_MTU="--mtu=1450"
DOCKER_NETWORK_OPTIONS=" --bip=172.30.7.1/24 --ip-masq=true --mtu=1450"

cat /run/flannel/subnet.env         #/run/flannel/subnet.env包含了flannel整个大网段以及在此节点上的子网段
#显示如下
FLANNEL_NETWORK=172.30.0.0/16
FLANNEL_SUBNET=172.30.7.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=false

ip add | grep flannel       #查看网卡信息
4: flannel.1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default 
    inet 172.30.7.0/32 scope global flannel.1

2.5.3、配置docker支持flannel

#1.配置docker支持flannel网络,需要在[Service]标签下新加
vim /etc/systemd/system/multi-user.target.wants/docker.service
EnvironmentFile=/run/flannel/docker     #这行新加内容，下面行新加$后面的内容
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock $DOCKER_NETWORK_OPTIONS

#2.重启docker,然后可以查看到已分配pod的子网列表
systemctl daemon-reload && systemctl restart docker && systemctl status docker

ip add | grep docker
#docker0网口IP地址，已改变
4: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN group default 
    inet 172.30.7.1/24 brd 172.30.7.255 scope global docker0

2.5.4、设置CNI插件支持flannel

tar -zxvf cni-plugins-linux-amd64-v0.8.1.tgz -C /opt/cni
mkdir -p /etc/cni/net.d
cat > /etc/cni/net.d/10-default.conf <<EOF
{
    "name": "flannel",
    "type": "flannel",
    "delegate": {
        "bridge": "docker0",
        "isDefaultGateway": true,
        "mtu": 1400
    }
}
EOF

#把相关指令和文件拷贝到所有主机
scp /opt/cni/* 192.168.10.13:/usr/local/bin && scp /etc/cni/net.d/* 192.168.10.13:/etc/cni/net.d/

2.6、部署keepalived+haproxy

keepalived 提供 kube-apiserver 对外服务的 VIP；haproxy 监听 VIP，后端连接所有 kube-apiserver 实例，提供健康检查和负载均衡功能；

本文档复用 master 节点的三台机器，haproxy 监听的端口(8443) 需要与 kube-apiserver 的端口 6443 不同，避免冲突。

keepalived 在运行过程中周期检查本机的 haproxy 进程状态，如果检测到 haproxy 进程异常，则触发重新选主的过程，VIP 将飘移到新选出来的主节点，从而实现 VIP 的高可用。所有组件（如 kubeclt、apiserver、controller-manager、scheduler 等）都通过 VIP 和 haproxy 监听的 8443 端口访问 kube-apiserver 服务。

2.6.1、安装haproxy

yum install -y haproxy

#12机器上配置
cat << EOF > /etc/haproxy/haproxy.cfg
global
    log         127.0.0.1 local2
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

defaults
    mode                    tcp
    log                     global
    retries                 3
    timeout connect         10s
    timeout client          1m
    timeout server          1m

listen  admin_stats
    bind 0.0.0.0:9090
    mode http
    log 127.0.0.1 local0 err
    stats refresh 30s
    stats uri /status
    stats realm welcome login\ Haproxy
    stats auth admin:123456
    stats hide-version
    stats admin if TRUE

frontend kubernetes
    bind *:8443
    mode tcp
    default_backend kubernetes-master

backend kubernetes-master
    balance roundrobin
    server k8s-m12 192.168.10.12:6443 check maxconn 2000
    server k8s-m13 192.168.10.13:6443 check maxconn 2000
    server k8s-m14 192.168.10.14:6443 check maxconn 2000
EOF

#13 和 14机器上配置都一样

# 启动haproxy
systemctl enable haproxy && systemctl start haproxy && systemctl status haproxy

2.6.2、安装keepalived

yum install -y keepalived

#10.12机器上配置

cat <<EOF > /etc/keepalived/keepalived.conf
global_defs {
   router_id LVS_k8s
}

vrrp_script CheckK8sMaster {
    script "curl -k https://192.168.10.100:8443"
    interval 3
    timeout 9
    fall 2
    rise 2
}

vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 100
    priority 100
    advert_int 1
    mcast_src_ip 192.168.10.12
    nopreempt
    authentication {
        auth_type PASS
        auth_pass fana123
    }
    unicast_peer {
        192.168.10.13
        192.168.10.14
    }
    virtual_ipaddress {
        192.168.10.100/24
    }
    track_script {
        CheckK8sMaster
    }

}
EOF

#13机器keepalived配置
cat <<EOF > /etc/keepalived/keepalived.conf
global_defs {
   router_id LVS_k8s
}

vrrp_script CheckK8sMaster {
    script "curl -k https://192.168.10.100:8443"
    interval 3
    timeout 9
    fall 2
    rise 2
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 100
    priority 90
    advert_int 1
    mcast_src_ip 192.168.10.13
    nopreempt
    authentication {
        auth_type PASS
        auth_pass fana123
    }
    unicast_peer {
        192.168.10.12
        192.168.10.14
    }
    virtual_ipaddress {
        192.168.10.100/24
    }
    track_script {
        CheckK8sMaster
    }
}
EOF

#14机器上keepalived配置
cat <<EOF > /etc/keepalived/keepalived.conf
global_defs {
   router_id LVS_k8s
}

vrrp_script CheckK8sMaster {
    script "curl -k https://192.168.10.100:8443"
    interval 3
    timeout 9
    fall 2
    rise 2
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 100
    priority 80
    advert_int 1
    mcast_src_ip 192.168.10.14
    nopreempt
    authentication {
        auth_type PASS
        auth_pass fana123
    }
    unicast_peer {
        192.168.10.12
        192.168.10.13
    }
    virtual_ipaddress {
        192.168.10.100/24
    }
    track_script {
        CheckK8sMaster
    }

}
EOF

#启动keepalived
systemctl restart keepalived && systemctl enable keepalived && systemctl status keepalived

#查看vip
ip add | grep 10.100

2.7、部署master

kube-scheduler，kube-controller-manager 和 kube-apiserver 三者的功能紧密相关；同时kube-scheduler 和 kube-controller-manager 只能有一个进程处于工作状态，如果运行多个，则需要通过选举产生一个 leader；

2.7.1、部署kubectl命令工具

kubectl 是 kubernetes 集群的命令行管理工具，默认从 ~/.kube/config 文件读取 kube-apiserver 地址、证书、用户名等信息，如果没有配置，执行 kubectl 命令时可能会出错。~/.kube/config只需要部署一次，然后拷贝到其他的master。

#1.解压命令
tar -zxvf kubernetes-server-linux-amd64.tar.gz
cd kubernetes/server/bin/
cp kube-apiserver kubeadm kube-controller-manager kubectl kube-scheduler /usr/local/bin
scp kube-apiserver kubeadm kube-controller-manager kubectl kube-scheduler 192.168.10.13:/usr/local/bin
scp kube-apiserver kubeadm kube-controller-manager kubectl kube-scheduler 192.168.10.14:/usr/local/bin

#2.创建CA证书
cd /root/ssl
cat > admin-csr.json <<EOF
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShangHai",
      "L": "ShangHai",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF

#3.生成证书和私钥
cfssl gencert -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes admin-csr.json | cfssljson -bare admin

#4.创建~/.kube/config文件
kubectl config set-cluster kubernetes \
  --certificate-authority=ca.pem \
  --embed-certs=true \
  --server=https://192.168.10.100:8443 \
  --kubeconfig=kubectl.kubeconfig

#4.1.设置客户端认证参数
kubectl config set-credentials admin \
  --client-certificate=admin.pem \
  --client-key=admin-key.pem \
  --embed-certs=true \
  --kubeconfig=kubectl.kubeconfig

#4.2.设置上下文参数
kubectl config set-context kubernetes \
  --cluster=kubernetes \
  --user=admin \
  --kubeconfig=kubectl.kubeconfig
  
#4.3.设置默认上下文
kubectl config use-context kubernetes --kubeconfig=kubectl.kubeconfig

#4.4.拷贝kubectl.kubeconfig文件
cp kubectl.kubeconfig ~/.kube/config
scp kubectl.kubeconfig 192.168.10.13:/root/.kube/config
scp kubectl.kubeconfig 192.168.10.14:/root/.kube/config

cp admin*.pem /etc/kubernetes/ssl/
scp admin*.pem 192.168.10.13:/etc/kubernetes/ssl/
scp admin*.pem 192.168.10.14:/etc/kubernetes/ssl/

2.7.2、部署api-server

#1.创建CA证书,hosts字段指定授权使用该证书的IP或域名列表，这里列出了VIP/apiserver节点IP/kubernetes服务IP和域名
cd /root/ssl
cat > kubernetes-csr.json <<EOF
{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "192.168.10.12",
    "192.168.10.13",
    "192.168.10.14",
    "192.168.10.100",
    "10.254.0.1",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShangHai",
      "L": "ShangHai",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

#2.生成证书和私钥
cfssl gencert -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes

#3.将证书拷贝到其他master节点
cp kubernetes*.pem /etc/kubernetes/ssl/
scp kubernetes*.pem 192.168.10.13:/etc/kubernetes/ssl/
scp kubernetes*.pem 192.168.10.14:/etc/kubernetes/ssl/

#4.创建加密配置文件
cat > encryption-config.yaml <<EOF
kind: EncryptionConfig
apiVersion: v1
resources:
  - resources:
      - secrets
    providers:
      - aescbc:
          keys:
            - name: key1
              secret: $(head -c 32 /dev/urandom | base64)
      - identity: {}
EOF

#4.1创建kube-apiserver使用的客户端令牌文件
cat <<EOF > bootstrap-token.csv
$(head -c 32 /dev/urandom | base64),kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

#5.将加密文件拷贝到其他master节点
cp encryption-config.yaml bootstrap-token.csv /etc/kubernetes/ssl
scp encryption-config.yaml bootstrap-token.csv 192.168.10.13:/etc/kubernetes/ssl
scp encryption-config.yaml bootstrap-token.csv 192.168.10.14:/etc/kubernetes/ssl

#6.创建kube-apiserver.service文件
cat > /etc/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \
  --enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \
  --anonymous-auth=false \
  --experimental-encryption-provider-config=/etc/kubernetes/ssl/encryption-config.yaml \
  --advertise-address=0.0.0.0 \
  --bind-address=0.0.0.0 \
  --insecure-bind-address=127.0.0.1 \
  --secure-port=6443 \
  --insecure-port=0 \
  --authorization-mode=Node,RBAC \
  --runtime-config=api/all \
  --enable-bootstrap-token-auth \
  --service-cluster-ip-range=10.254.0.0/16 \
  --service-node-port-range=30000-32700 \
  --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \
  --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \
  --client-ca-file=/etc/kubernetes/ssl/ca.pem \
  --kubelet-client-certificate=/etc/kubernetes/ssl/kubernetes.pem \
  --kubelet-client-key=/etc/kubernetes/ssl/kubernetes-key.pem \
  --service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
  --etcd-cafile=/etc/kubernetes/ssl/ca.pem \
  --etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem \
  --etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem \
  --etcd-servers=https://192.168.10.12:2379,https://192.168.10.13:2379,https://192.168.10.14:2379 \
  --enable-swagger-ui=true \
  --allow-privileged=true \
  --apiserver-count=3 \
  --audit-log-maxage=30 \
  --audit-log-maxbackup=3 \
  --audit-log-maxsize=100 \
  --audit-log-path=/var/log/kubernetes/kube-apiserver-audit.log \
  --event-ttl=1h \
  --alsologtostderr=true \
  --logtostderr=false \
  --log-dir=/var/log/kubernetes \
  --v=2
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

mkdir -p /var/log/kubernetes        #创建日志目录然后拷贝到其他master
scp /etc/systemd/system/kube-apiserver.service 192.168.10.13:/etc/systemd/system/
scp /etc/systemd/system/kube-apiserver.service 192.168.10.14:/etc/systemd/system/

#7.启动服务

systemctl daemon-reload && systemctl enable kube-apiserver && systemctl start kube-apiserver && systemctl status kube-apiserver

#8.授予kubernetes证书访问kubelet api权限。在执行kubectl exec、run、logs 等命令时，apiserver会转发到kubelet。这里定义 RBAC规则，授权apiserver调用kubelet API。
kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes

#8.1预定义的ClusterRole system:kubelet-api-admin授予访问kubelet所有 API 的权限：
kubectl describe clusterrole system:kubelet-api-admin

#9.检查api-server和集群状态
netstat -ptln | grep kube-apiserver
tcp        0      0 192.168.10.12:6443      0.0.0.0:*               LISTEN      13000/kube-apiserve 

kubectl cluster-info
#显示如下
Kubernetes master is running at https://192.168.10.100:8443
To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.

kubectl get all --all-namespaces
#显示如下
NAMESPACE   NAME                 TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
default     service/kubernetes   ClusterIP   10.254.0.1   <none>        443/TCP   12m

kubectl get componentstatuses
#显示如下,因scheduler和controller-manager还没有部署
NAME                 STATUS      MESSAGE                                ERROR
scheduler            Unhealthy   Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: connect: connection refused   
controller-manager   Unhealthy   Get http://127.0.0.1:10252/healthz: dial tcp 127.0.0.1:10252: connect: connection refused   
etcd-2               Healthy     {"health":"true"}
etcd-1               Healthy     {"health":"true"} 
etcd-0               Healthy     {"health":"true"}

2.7.3、部署kube-controller-manager

该集群包含 3 个节点，启动后将通过竞争选举机制产生一个 leader 节点，其它节点为阻塞状态。当 leader 节点不可用后，剩余节点将再次进行选举产生新的 leader 节点，从而保证服务的可用性。

#1.创建CA证书
cd /root/ssl
cat > kube-controller-manager-csr.json << EOF
{
    "CN": "system:kube-controller-manager",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "hosts": [
      "127.0.0.1",
      "192.168.10.12",
      "192.168.10.13",
      "192.168.10.14"
    ],
    "names": [
      {
        "C": "CN",
        "ST": "ShangHai",
        "L": "ShangHai",
        "O": "system:kube-controller-manager",
        "OU": "System"
      }
    ]
}
EOF

#2.生成证书
cfssl gencert -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager

#3.将证书拷贝到其他master节点
cp kube-controller-manager*.pem /etc/kubernetes/ssl/
scp kube-controller-manager*.pem 192.168.10.13:/etc/kubernetes/ssl/
scp kube-controller-manager*.pem 192.168.10.14:/etc/kubernetes/ssl/

#4.创建kubeconfig文件
kubectl config set-cluster kubernetes \
  --certificate-authority=ca.pem \
  --embed-certs=true \
  --server=https://192.168.10.100:8443 \
  --kubeconfig=kube-controller-manager.kubeconfig

kubectl config set-credentials system:kube-controller-manager \
  --client-certificate=kube-controller-manager.pem \
  --client-key=kube-controller-manager-key.pem \
  --embed-certs=true \
  --kubeconfig=kube-controller-manager.kubeconfig

kubectl config set-context system:kube-controller-manager \
  --cluster=kubernetes \
  --user=system:kube-controller-manager \
  --kubeconfig=kube-controller-manager.kubeconfig

kubectl config use-context system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig

#5.拷贝kube-controller-manager.kubeconfig到其他master节点
cp kube-controller-manager.kubeconfig /etc/kubernetes/ssl/
scp kube-controller-manager.kubeconfig 192.168.10.13:/etc/kubernetes/ssl/
scp kube-controller-manager.kubeconfig 192.168.10.14:/etc/kubernetes/ssl/

#6.创建kube-controller-manager.service文件
cat > /etc/systemd/system/kube-controller-manager.service  << EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]
ExecStart=/usr/local/bin/kube-controller-manager \
  --address=127.0.0.1 \
  --master=https://192.168.10.100:8443 \
  --kubeconfig=/etc/kubernetes/ssl/kube-controller-manager.kubeconfig \
  --allocate-node-cidrs=true \
  --authentication-kubeconfig=/etc/kubernetes/ssl/kube-controller-manager.kubeconfig \
  --service-cluster-ip-range=10.254.0.0/16 \
  --cluster-cidr=172.30.0.0/16 \
  --cluster-name=kubernetes \
  --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
  --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
  --experimental-cluster-signing-duration=8760h \
  --leader-elect=true \
  --feature-gates=RotateKubeletServerCertificate=true \
  --controllers=*,bootstrapsigner,tokencleaner \
  --horizontal-pod-autoscaler-use-rest-clients=true \
  --horizontal-pod-autoscaler-sync-period=10s \
  --tls-cert-file=/etc/kubernetes/ssl/kube-controller-manager.pem \
  --tls-private-key-file=/etc/kubernetes/ssl/kube-controller-manager-key.pem \
  --service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \
  --root-ca-file=/etc/kubernetes/ssl/ca.pem \
  --use-service-account-credentials=true \
  --alsologtostderr=true \
  --logtostderr=false \
  --log-dir=/var/log/kubernetes \
  --v=2
Restart=on
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

#7.拷贝到其他master节点，然后启动服务
scp /etc/systemd/system/kube-controller-manager.service 192.168.10.13:/etc/systemd/system/
scp /etc/systemd/system/kube-controller-manager.service 192.168.10.14:/etc/systemd/system/

systemctl daemon-reload && systemctl enable kube-controller-manager && systemctl start kube-controller-manager && systemctl status kube-controller-manager

#8.检查服务
netstat -lnpt|grep kube-controll
tcp        0      0 127.0.0.1:10252         0.0.0.0:*               LISTEN      14492/kube-controll 
tcp6       0      0 :::10257                :::*                    LISTEN      14492/kube-controll 

kubectl get cs
#显示如下
NAME                 STATUS      MESSAGE                                               ERROR
scheduler            Unhealthy   Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: connect: connection refused 
controller-manager   Healthy     ok                               
etcd-1               Healthy     {"health":"true"}
etcd-2               Healthy     {"health":"true"}
etcd-0               Healthy     {"health":"true"}

#检查leader所在机器
kubectl get endpoints kube-controller-manager --namespace=kube-system  -o yaml
#显示如下,k8s-m12选为leader
apiVersion: v1
kind: Endpoints
metadata:
  annotations:
    control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"k8s-m12_6f9b09e6-995b-11e9-b2bf-000c29959a05","leaseDurationSeconds":15,"acquireTime":"2019-06-28T04:16:00Z","renewTime":"2019-06-28T04:21:32Z","leaderTransitions":0}'
  creationTimestamp: "2019-06-28T04:16:00Z"
  name: kube-controller-manager
  namespace: kube-system
  resourceVersion: "1481"
  selfLink: /api/v1/namespaces/kube-system/endpoints/kube-controller-manager
  uid: 6f9d838f-995b-11e9-9cb7-000c29959a05

关于 controller 权限和 use-service-account-credentials 参数
kublet 认证和授权

2.7.4、部署kube-scheduler

该集群包含 3 个节点，启动后将通过竞争选举机制产生一个 leader 节点，其它节点为阻塞状态。当 leader 节点不可用后，剩余节点将再次进行选举产生新的 leader 节点，从而保证服务的可用性

#1.创建CA证书
cd /root/ssl
cat > kube-scheduler-csr.json << EOF
{
    "CN": "system:kube-scheduler",
    "hosts": [
      "127.0.0.1",
      "192.168.10.12",
      "192.168.10.13",
      "192.168.10.14"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
      {
        "C": "CN",
        "ST": "ShangHai",
        "L": "ShangHai",
        "O": "system:kube-scheduler",
        "OU": "System"
      }
    ]
}
EOF

#2.生成证书
cfssl gencert -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler

#3.创建kube-scheduler.kubeconfig文件
kubectl config set-cluster kubernetes \
  --certificate-authority=ca.pem \
  --embed-certs=true \
  --server=https://192.168.10.100:8443 \
  --kubeconfig=kube-scheduler.kubeconfig

kubectl config set-credentials system:kube-scheduler \
  --client-certificate=kube-scheduler.pem \
  --client-key=kube-scheduler-key.pem \
  --embed-certs=true \
  --kubeconfig=kube-scheduler.kubeconfig

kubectl config set-context system:kube-scheduler \
  --cluster=kubernetes \
  --user=system:kube-scheduler \
  --kubeconfig=kube-scheduler.kubeconfig

kubectl config use-context system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig

#4.拷贝kubeconfig到其他master节点
cp kube-scheduler.kubeconfig kube-scheduler*.pem /etc/kubernetes/ssl/
scp kube-scheduler.kubeconfig kube-scheduler*.pem 192.168.10.13:/etc/kubernetes/ssl/
scp kube-scheduler.kubeconfig kube-scheduler*.pem 192.168.10.14:/etc/kubernetes/ssl/

#5.创建kube-scheduler.service文件
cat > /etc/systemd/system/kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/GoogleCloudPlatform/kubernetes

[Service]
ExecStart=/usr/local/bin/kube-scheduler \
  --address=127.0.0.1 \
  --master=https://192.168.10.100:8443 \
  --kubeconfig=/etc/kubernetes/ssl/kube-scheduler.kubeconfig \
  --leader-elect=true \
  --alsologtostderr=true \
  --logtostderr=false \
  --log-dir=/var/log/kubernetes \
  --v=2
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

#6.将kube-scheduler.service拷贝到其他master节点，然后启动服务
scp /etc/systemd/system/kube-scheduler.service 192.168.10.13:/etc/systemd/system
scp /etc/systemd/system/kube-scheduler.service 192.168.10.14:/etc/systemd/system

systemctl daemon-reload && systemctl enable kube-scheduler && systemctl start kube-scheduler && systemctl status kube-scheduler

#7.检查服务
netstat -lnpt|grep kube-sche
tcp        0      0 127.0.0.1:10251         0.0.0.0:*               LISTEN      15137/kube-schedule 
tcp6       0      0 :::10259                :::*                    LISTEN      15137/kube-schedule

kubectl get cs
#显示如下
NAME                 STATUS    MESSAGE             ERROR
scheduler            Healthy   ok                  
controller-manager   Healthy   ok                  
etcd-2               Healthy   {"health":"true"}   
etcd-1               Healthy   {"health":"true"}   
etcd-0               Healthy   {"health":"true"} 

kubectl get endpoints kube-scheduler --namespace=kube-system  -o yaml
#显示如下，k8s-m12选为leader
apiVersion: v1
kind: Endpoints
metadata:
  annotations:
    control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"k8s-m12_1c3f7882-995f-11e9-a5c1-000c29959a05","leaseDurationSeconds":15,"acquireTime":"2019-06-28T04:42:19Z","renewTime":"2019-06-28T04:45:18Z","leaderTransitions":0}'
  creationTimestamp: "2019-06-28T04:42:19Z"
  name: kube-scheduler
  namespace: kube-system
  resourceVersion: "2714"
  selfLink: /api/v1/namespaces/kube-system/endpoints/kube-scheduler
  uid: 1cda2b3a-995f-11e9-ac7d-000c2928fce6

2.7.5、在所有master节点上查看功能是否正常

kubectl get componentstatuses
NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok                  
scheduler            Healthy   ok                  
etcd-2               Healthy   {"health":"true"}   
etcd-1               Healthy   {"health":"true"}   
etcd-0               Healthy   {"health":"true"}

2.8、部署node

node节点运行kubelet kube-proxy docker flannel。

2.8.1、部署kubelet

kubelet运行在每个 worker 节点上，接收 kube-apiserver 发送的请求，管理 Pod 容器，执行交互式命令，如 exec、run、logs 等。kubelet 启动时自动向 kube-apiserver注册节点信息，内置的 cadvisor 统计和监控节点的资源使用情况。

#1.解压包，拷贝命令
tar -zxvf kubernetes-node-linux-amd64.tar.gz
cd /opt/kubernetes/node/bin
cp kubectl kubelet kube-proxy /usr/local/bin
scp kubectl kubelet kube-proxy 192.168.10.16:/usr/local/bin

#2.创建kubelet-bootstrap.kubeconfig文件（也是在12机器上执行）要创建3次分别是(k8s-m12，k8s-m13，k8s-m14)

#2.1.创建 token
cd /root/ssl
export BOOTSTRAP_TOKEN=$(kubeadm token create \
  --description kubelet-bootstrap-token \
  --groups system:bootstrappers:k8s-m12 \
  --kubeconfig ~/.kube/config)

#2.2.设置集群参数
kubectl config set-cluster kubernetes \
  --certificate-authority=ca.pem \
  --embed-certs=true \
  --server=https://192.168.10.100:8443 \
  --kubeconfig=kubelet-bootstrap-k8s-m12.kubeconfig

#2.3.设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
  --token=${BOOTSTRAP_TOKEN} \
  --kubeconfig=kubelet-bootstrap-k8s-m12.kubeconfig

#2.4.设置上下文参数
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kubelet-bootstrap \
  --kubeconfig=kubelet-bootstrap-k8s-m12.kubeconfig

#2.5.设置默认上下文
kubectl config use-context default --kubeconfig=kubelet-bootstrap-k8s-m12.kubeconfig

#3.查看kubeadm为各节点创建的token
kubeadm token list --kubeconfig ~/.kube/config
#显示如下
11rq5j.3f628cf6ura1hf2x   20h       2019-06-29T13:01:52+08:00   authentication,signing   kubelet-bootstrap-token   system:bootstrappers:k8s-m14
8zamvk.rfat8wyzh8311f89   20h       2019-06-29T12:59:26+08:00   authentication,signing   kubelet-bootstrap-token   system:bootstrappers:k8s-m12
lhxalz.busnf6izk82e0xqx   20h       2019-06-29T13:01:03+08:00   authentication,signing   kubelet-bootstrap-token   system:bootstrappers:k8s-m13

#3.1.r如果需要删除创建的token
kubeadm token --kubeconfig ~/.kube/config delete lhxalz.busnf6izk82e0xqx        
# 创建的token有效期为 1 天，超期后将不能再被使用，且会被kube-controller-manager的tokencleaner清理(如果启用该 controller 的话)。
# kube-apiserver接收kubelet的bootstrap token后，将请求的user设置为system:bootstrap；group设置为 system:bootstrappers；

#3.2.查看各token关联的secret
kubectl get secrets  -n kube-system     

#4.拷贝bootstrap kubeconfig文件到各个node机器上
scp kubelet-bootstrap-kube12.kubeconfig 192.168.10.15:/etc/kubernetes/ssl/kubelet-bootstrap.kubeconfig
scp kubelet-bootstrap-kube12.kubeconfig 192.168.10.16:/etc/kubernetes/ssl/kubelet-bootstrap.kubeconfig

#5.创建kubelet配置文件
cd /root/ssl
cat > kubelet.config.json <<EOF
{
  "kind": "KubeletConfiguration",
  "apiVersion": "kubelet.config.k8s.io/v1beta1",
  "authentication": {
    "x509": {
      "clientCAFile": "/etc/kubernetes/ssl/ca.pem"
    },
    "webhook": {
      "enabled": true,
      "cacheTTL": "2m0s"
    },
    "anonymous": {
      "enabled": false
    }
  },
  "authorization": {
    "mode": "Webhook",
    "webhook": {
      "cacheAuthorizedTTL": "5m0s",
      "cacheUnauthorizedTTL": "30s"
    }
  },
  "address": "0.0.0.0",
  "port": 10250,
  "readOnlyPort": 0,
  "cgroupDriver": "cgroupfs",
  "hairpinMode": "promiscuous-bridge",
  "serializeImagePulls": false,
  "featureGates": {
    "RotateKubeletClientCertificate": true,
    "RotateKubeletServerCertificate": true
  },
  "clusterDomain": "cluster.local",
  "clusterDNS": ["10.254.0.2"]
}
EOF

#6.拷贝到其他主机,注意，可以修改address为本机IP地址
cp kubelet.config.json /etc/kubernetes/ssl
scp kubelet.config.json 192.168.10.15:/etc/kubernetes/ssl
scp kubelet.config.json 192.168.10.16:/etc/kubernetes/ssl

#7.创建kubelet.service文件
mkdir -p /var/log/kubernetes && mkdir -p /var/lib/kubelet   #先创建目录

cat <<EOF > /etc/systemd/system/kubelet.service 
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service

[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/local/bin/kubelet \
  --bootstrap-kubeconfig=/etc/kubernetes/ssl/kubelet-bootstrap.kubeconfig \
  --cert-dir=/etc/kubernetes/ssl \
  --network-plugin=cni \
  --cni-conf-dir=/etc/cni/net.d \
  --cni-bin-dir=/usr/local/bin/ \
  --fail-swap-on=false \
  --kubeconfig=/etc/kubernetes/ssl/kubelet.kubeconfig \
  --config=/etc/kubernetes/ssl/kubelet.config.json \
  --hostname-override=192.168.10.15 \
  --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1 \
  --allow-privileged=true \
  --alsologtostderr=true \
  --logtostderr=false \
  --cgroup-driver=systemd \
  --log-dir=/var/log/kubernetes \
  --v=2
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

#拷贝到其他主机，注意修改hostname-override为本机IP地址

#8.Bootstrap Token Auth 和授予权限 ,需要先将bootstrap-token文件中的kubelet-bootstrap用户赋予system:node-bootstrapper角色，然后kubelet才有权限创建认证请求
kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --group=system:bootstrappers

#9.启动kubele服务
systemctl daemon-reload && systemctl enable kubelet && systemctl restart kubelet && systemctl status kubelet

#10.检查服务
netstat -lantp|grep kubelet
tcp        0      0 192.168.10.15:46936     192.168.10.100:8443     ESTABLISHED 15299/kubelet

#8.通过kubelet 的TLS 证书请求，kubelet 首次启动时向kube-apiserver 发送证书签名请求，必须通过后kubernetes 系统才会将该 Node 加入到集群。查看未授权的CSR 请求
kubectl get csr
NAME                                                   AGE   REQUESTOR                 CONDITION
node-csr-ZyWLfyY4nBb1GPBCCNGf2pCjbFKGHt04q50R1_3oprU   16m   system:bootstrap:rhwf4g   Pending
node-csr-hiZbOHizDYsE_n36kfuSxWTmUzobCEnCpIXfN54Lh6Y   18m   system:bootstrap:rhwf4g   Pending

approve kubelet csr请求

#1.手动approve csr请求(推荐自动的方式)
kubectl certificate approve node-csr-ZyWLfyY4nBb1GPBCCNGf2pCjbFKGHt04q50R1_3oprU #手动创建
#显示如下
certificatesigningrequest.certificates.k8s.io/node-csr-ZyWLfyY4nBb1GPBCCNGf2pCjbFKGHt04q50R1_3oprU approved

#1.1.查看Approve结果
kubectl describe csr node-csr-ZyWLfyY4nBb1GPBCCNGf2pCjbFKGHt04q50R1_3oprU   
#显示如下
Name:               node-csr-ZyWLfyY4nBb1GPBCCNGf2pCjbFKGHt04q50R1_3oprU
Labels:             <none>
Annotations:        <none>
CreationTimestamp:  Wed, 26 Jun 2019 15:12:40 +0800
Requesting User:    system:bootstrap:rhwf4g
Status:             Approved,Issued
Subject:
         Common Name:    system:node:192.168.10.16
         Serial Number:  
         Organization:   system:nodes
Events:  <none>

#1.2.特别多可以用这样的方式
kubectl get csr|grep 'Pending' | awk 'NR>0{print $1}'| xargs kubectl certificate approve
kubectl get csr|awk 'NR==3{print $1}'| xargs kubectl describe csr   #查看Approve结果

#2.自动approve csr请求(推荐),创建ClusterRoleBinding，分别用于自动 approve client、renew client、renew server 证书
cd /root/ssl
cat > csr-crb.yaml <<EOF
# Approve all CSRs for the group "system:bootstrappers"
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: auto-approve-csrs-for-group
 subjects:
 - kind: Group
   name: system:bootstrappers
   apiGroup: rbac.authorization.k8s.io
 roleRef:
   kind: ClusterRole
   name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
   apiGroup: rbac.authorization.k8s.io
---
 # To let a node of the group "system:bootstrappers" renew its own credentials
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: node-client-cert-renewal
 subjects:
 - kind: Group
   name: system:bootstrappers
   apiGroup: rbac.authorization.k8s.io
 roleRef:
   kind: ClusterRole
   name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
   apiGroup: rbac.authorization.k8s.io
---
# A ClusterRole which instructs the CSR approver to approve a node requesting a
# serving cert matching its client cert.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: approve-node-server-renewal-csr
rules:
- apiGroups: ["certificates.k8s.io"]
  resources: ["certificatesigningrequests/selfnodeserver"]
  verbs: ["create"]
---
 # To let a node of the group "system:nodes" renew its own server credentials
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: node-server-cert-renewal
 subjects:
 - kind: Group
   name: system:nodes
   apiGroup: rbac.authorization.k8s.io
 roleRef:
   kind: ClusterRole
   name: approve-node-server-renewal-csr
   apiGroup: rbac.authorization.k8s.io
EOF

#3.拷贝到其他节点
cp csr-crb.yaml /etc/kubernetes/ssl
scp csr-crb.yaml 192.168.10.13:/etc/kubernetes/ssl
scp csr-crb.yaml 192.168.10.14:/etc/kubernetes/ssl

#4.生效配置
kubectl apply -f /etc/kubernetes/ssl/csr-crb.yaml

#5.验证
kubectl get csr     #等待一段时间，查看CSR都被自动approve
#显示如下
NAME                                                   AGE   REQUESTOR                 CONDITION
node-csr-cF4D5xoTEQCkK5QCsCAmsHGItlZ2cJ43RjkGXpM4BNw   38m   system:bootstrap:8zamvk   Approved,Issued
node-csr-lUIuS1_ggYM8Q95rgsUrBawzrsAXQ4QfYcP3BbPnWl8   36m   system:bootstrap:lhxalz   Approved,Issued


kubectl get --all-namespaces -o wide nodes      #所有节点均 ready
#显示如下
NAME            STATUS   ROLES    AGE     VERSION   INTERNAL-IP     EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION                CONTAINER-RUNTIME
192.168.10.15   Ready    <none>   5m33s   v1.14.3   192.168.10.15   <none>        CentOS Linux 7 (Core)   4.4.103-1.el7.elrepo.x86_64   docker://18.9.6
192.168.10.16   Ready    <none>   54s     v1.14.3   192.168.10.16   <none>        CentOS Linux 7 (Core)   4.4.103-1.el7.elrepo.x86_64   docker://18.9.6

kubectl get nodes
NAME            STATUS   ROLES    AGE     VERSION
192.168.10.15   Ready    <none>   6m55s   v1.14.3
192.168.10.16   Ready    <none>   2m16s   v1.14.3

netstat -lnpt|grep kubelet
tcp        0      0 127.0.0.1:10248         0.0.0.0:*               LISTEN      20302/kubelet       
tcp        0      0 192.168.10.15:10250     0.0.0.0:*               LISTEN      20302/kubelet       
tcp        0      0 127.0.0.1:37706         0.0.0.0:*               LISTEN      20302/kubelet       
tcp        0      0 192.168.10.15:60332     192.168.10.100:8443     ESTABLISHED 20302/kubelet 
#10248: healthz http 服务,10250; https API 服务；注意：未开启只读端口 10255；由于关闭了匿名认证，同时开启了 webhook 授权，所有访问 10250 端口 https API 的请求都需要被认证和授权。

kublet api 认证和授权

kublet的配置文件kubelet.config.json配置了如下认证参数：

  • authentication.anonymous.enabled：设置为 false，不允许匿名访问 10250 端口；
  • authentication.x509.clientCAFile：指定签名客户端证书的 CA 证书，开启 HTTPs 证书认证；
  • authentication.webhook.enabled=true：开启 HTTPs bearer token 认证；

同时配置了如下授权参数：

  • authroization.mode=Webhook：开启 RBAC 授权；
# kubelet 收到请求后，使用 clientCAFile 对证书签名进行认证，或者查询 bearer token 是否有效。如果两者都没通过，则拒绝请求，提示 Unauthorized
curl -s --cacert /etc/kubernetes/ssl/ca.pem https://127.0.0.1:10250/metrics
curl -s --cacert /etc/kubernetes/ssl/ca.pem -H "Authorization: Bearer 123456" https://192.168.10.15:10250/metrics

#通过认证后，kubelet 使用 SubjectAccessReview API 向 kube-apiserver 发送请求，查询证书或 token 对应的 user、group 是否有操作资源的权限(RBAC)；

#1.证书认证和授权

#权限不足的证书；
curl -s --cacert /etc/kubernetes/ssl/ca.pem --cert /etc/kubernetes/ssl/kube-controller-manager.pem --key /etc/kubernetes/ssl/kube-controller-manager-key.pem https://192.168.10.15:10250/metrics

#使用部署 kubectl 命令行工具时创建的、具有最高权限的 admin 证书；
curl -s --cacert /etc/kubernetes/ssl/ca.pem --cert /etc/kubernetes/ssl/admin.pem --key /etc/kubernetes/ssl/admin-key.pem https://192.168.10.15:10250/metrics|head

#2.bear token认证和授权：

# 创建一个ServiceAccount，将它和ClusterRole system:kubelet-api-admin绑定，从而具有调用kubelet API的权限：
kubectl create sa kubelet-api-test

kubectl create clusterrolebinding kubelet-api-test --clusterrole=system:kubelet-api-admin --serviceaccount=default:kubelet-api-test 
SECRET=$(kubectl get secrets | grep kubelet-api-test | awk '{print $1}')
TOKEN=$(kubectl describe secret ${SECRET} | grep -E '^token' | awk '{print $2}')
echo ${TOKEN}

curl -s --cacert /etc/kubernetes/ssl/ca.pem -H "Authorization: Bearer ${TOKEN}" https://192.168.10.15:10250/metrics|head

# cadvisor 和 metrics
# cadvisor 统计所在节点各容器的资源(CPU、内存、磁盘、网卡)使用情况，分别在自己的 http web 页面(4194 端口)和 10250 以 promehteus metrics 的形式输出。

# 浏览器访问 http://192.168.10.15:4194/containers/ 可以查看到 cadvisor 的监控页面：
# 浏览器访问 https://192.168.10.15:10250/metrics 和 https://192.168.10.15:10250/metrics/cadvisor 分别返回 kublet 和 cadvisor 的 metrics。

注意：kublet.config.json 设置 authentication.anonymous.enabled 为 false，不允许匿名证书访问 10250 的 https 服务；参考A.浏览器访问kube-apiserver安全端口.md，创建和导入相关证书，然后访问上面的 10250 端口；

#1.需要安装jdk然后使用keytool工具
.\keytool -import -v -trustcacerts -alias appmanagement -file "E:\ca.pem" -storepass password -keystore cacerts
#2.然后在linux上执行
openssl pkcs12 -export -out admin.pfx -inkey admin-key.pem -in admin.pem -certfile ca.pem
#3.然后把证书导进去，就可以正常访问了

2.8.2、部署kube-proxy

kube-proxy 运行在所有 worker 节点上，，它监听 apiserver 中 service 和 Endpoint 的变化情况，创建路由规则来进行服务负载均衡。

#1.创建CA证书
cd /root/ssl
cat > kube-proxy-csr.json <<EOF
{
  "CN": "system:kube-proxy",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShangHai",
      "L": "ShangHai",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

#2.生成证书和私钥
cfssl gencert -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=kubernetes  kube-proxy-csr.json | cfssljson -bare kube-proxy

#3.创建kubeconfig文件

#3.1.设置集群参数
kubectl config set-cluster kubernetes \
  --certificate-authority=ca.pem \
  --embed-certs=true \
  --server=https://192.168.10.100:8443 \
  --kubeconfig=kube-proxy.kubeconfig
#3.2.设置客户端认证参数
kubectl config set-credentials kube-proxy \
  --client-certificate=kube-proxy.pem \
  --client-key=kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=kube-proxy.kubeconfig
#3.3.设置上下文参数
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-proxy \
  --kubeconfig=kube-proxy.kubeconfig
#3.4.设置默认上下文
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

#4.拷贝到其他节点
cp kube-proxy*.pem kube-proxy.kubeconfig /etc/kubernetes/ssl/
scp kube-proxy*.pem kube-proxy.kubeconfig 192.168.10.15:/etc/kubernetes/ssl/
scp kube-proxy*.pem kube-proxy.kubeconfig 192.168.10.16:/etc/kubernetes/ssl/

#5.创建kube-proxy配置文件
cd /root/ssl
cat >kube-proxy.config.yaml <<EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 192.168.10.15
clientConnection:
  kubeconfig: /etc/kubernetes/ssl/kube-proxy.kubeconfig
clusterCIDR: 172.30.0.0/16
healthzBindAddress: 192.168.10.15:10256
hostnameOverride: 192.168.10.15
kind: KubeProxyConfiguration
metricsBindAddress: 192.168.10.15:10249
mode: "ipvs"
EOF

#6.拷贝到其他节点
cp kube-proxy.config.yaml /etc/kubernetes/ssl/
scp kube-proxy.config.yaml 192.168.10.15:/etc/kubernetes/ssl/
scp kube-proxy.config.yaml 192.168.10.16:/etc/kubernetes/ssl/

#7.创建kube-proxy.service文件，然后拷贝到其他节点
cat << EOF > /etc/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/local/bin/kube-proxy \
  --config=/etc/kubernetes/ssl/kube-proxy.config.yaml \
  --alsologtostderr=true \
  --logtostderr=false \
  --log-dir=/var/log/kubernetes/kube-proxy \
  --v=2
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

#8.启动kube-proxy服务
mkdir -p /var/lib/kube-proxy && mkdir -p /var/log/kubernetes/kube-proxy
systemctl daemon-reload && systemctl enable kube-proxy && systemctl restart kube-proxy && systemctl status kube-proxy

netstat -lnpt|grep kube-proxy   #查看端口

ipvsadm -ln     #查看ipvs路由规则
#显示如下
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.254.0.1:443 rr
  -> 192.168.10.12:6443           Masq    1      0          0         
  -> 192.168.10.13:6443           Masq    1      0          0         
  -> 192.168.10.14:6443           Masq    1      0          0

2.8.3、验证集群功能

kubectl get nodes       #查看节点状态

# 1、创建nginx 测试文件
cat << EOF > nginx-web.yml 
apiVersion: v1
kind: Service
metadata:
  name: nginx-web
  labels:
    tier: frontend
spec:
  type: NodePort
  selector:
    tier: frontend
  ports:
  - name: http
    port: 80
    targetPort: 80
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: nginx-con
  labels:
    tier: frontend
spec:
  replicas: 3
  template:
    metadata:
      labels:
        tier: frontend
    spec:
      containers:
      - name: nginx-pod
        image: nginx
        ports:
        - containerPort: 80
EOF

#2.执行文件
kubectl create -f nginx-web.yml     
#显示已创建
service/nginx-web created
deployment.extensions/nginx-con created

#3.查看pod状态
kubectl get pod -o wide
#显示如下
NAME                         READY   STATUS    RESTARTS   AGE    IP            NODE            NOMINATED NODE   READINESS GATES
nginx-con-7dc84bdfb6-h6bt6   1/1     Running   0          105s   172.30.85.2   192.168.10.16   <none>           <none>
nginx-con-7dc84bdfb6-nt5qs   1/1     Running   0          105s   172.30.34.3   192.168.10.15   <none>           <none>
nginx-con-7dc84bdfb6-sfg87   1/1     Running   0          105s   172.30.34.2   192.168.10.15   <none>           <none>

#4.测试IP是否ping通
ping -c4 172.30.34.2
PING 172.30.34.2 (172.30.34.2) 56(84) bytes of data.
64 bytes from 172.30.34.2: icmp_seq=1 ttl=63 time=0.543 ms
64 bytes from 172.30.34.2: icmp_seq=2 ttl=63 time=0.684 ms
64 bytes from 172.30.34.2: icmp_seq=3 ttl=63 time=0.886 ms
64 bytes from 172.30.34.2: icmp_seq=4 ttl=63 time=0.817 ms

#5.查看server集群IP
kubectl get svc     #显示如下
NAME         TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)        AGE
kubernetes   ClusterIP   10.254.0.1       <none>        443/TCP        37h
nginx-web    NodePort    10.254.153.104   <none>        80:31808/TCP   4m19s
# 10.254.153.104是nginx的集群IP，代理前面3个pod，80是集群IP的端口31808是nodeport端口

#6.curl访问node_ip：nodeport
curl -I 192.168.10.15:31808     #状态200表示访问成功
HTTP/1.1 200 OK
Server: nginx/1.17.0
Date: Sat, 29 Jun 2019 05:03:15 GMT
Content-Type: text/html
Content-Length: 612
Last-Modified: Tue, 21 May 2019 14:23:57 GMT
Connection: keep-alive
ETag: "5ce409fd-264"
Accept-Ranges: bytes

#7.在flannel网络主机*问集群IP
ip add | grep 10.254
    inet 10.254.0.1/32 brd 10.254.0.1 scope global kube-ipvs0
    inet 10.254.153.104/32 brd 10.254.153.104 scope global kube-ipvs0

curl -I http://10.254.153.104:80    #返回如下
HTTP/1.1 200 OK
Server: nginx/1.17.0
Date: Sat, 29 Jun 2019 05:05:56 GMT
Content-Type: text/html
Content-Length: 612
Last-Modified: Tue, 21 May 2019 14:23:57 GMT
Connection: keep-alive
ETag: "5ce409fd-264"
Accept-Ranges: bytes

2.9、部署集群插件

插件是集群的附件组件，丰富和完善了集群的功能

2.9.1、部署coredns插件

#1.将kubernetes-server-linux-amd64.tar.gz解压后，再解压其中的 kubernetes-src.tar.gz 文件
tar -zxvf kubernetes-src.tar.gz -C src   #coredns对应的目录是：cluster/addons/dns

#2.修改配置文件
cd src/cluster/addons/dns/coredns
cp coredns.yaml.base /etc/kubernetes/coredns.yaml

sed -i "s/__PILLAR__DNS__DOMAIN__/cluster.local/g" /etc/kubernetes/coredns.yaml
sed -i "s/__PILLAR__DNS__SERVER__/10.254.0.2/g" /etc/kubernetes/coredns.yaml

#3.创建coredns
kubectl create -f /etc/kubernetes/coredns.yaml

#4.检查codedns功能
kubectl -n kube-system get all -o wide
#显示如下
NAME                          READY   STATUS    RESTARTS   AGE
pod/coredns-8854569d4-5vshp   1/1     Running   0          58m
#
NAME               TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                  AGE
service/kube-dns   ClusterIP   10.254.0.2   <none>        53/UDP,53/TCP,9153/TCP   81m
#
NAME                      READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/coredns   1/1     1            1           58m
#
NAME                                DESIRED   CURRENT   READY   AGE
replicaset.apps/coredns-8854569d4   1         1         1       58m
#4.1
kubectl -n kube-system describe pod coredns
#4.2
kubectl -n kube-system logs coredns-8854569d4-5vshp

#5.使用容器验证
kubectl run dns-test --rm -it --image=alpine /bin/sh
#进入容器 ping 百度正常
ping www.baidu.com
PING www.baidu.com (182.61.200.6): 56 data bytes
64 bytes from 182.61.200.6: seq=0 ttl=127 time=41.546 ms
64 bytes from 182.61.200.6: seq=1 ttl=127 time=35.043 ms
64 bytes from 182.61.200.6: seq=2 ttl=127 time=38.977 ms
64 bytes from 182.61.200.6: seq=3 ttl=127 time=40.633 ms

#查看所有集群pod
kubectl get --all-namespaces pods

#6.如果遇到镜像下载不下来，可以修改文件
sed -i "s/k8s.gcr.io/coredns/g" /etc/kubernetes/coredns.yaml

2.9.2、部署dashboard插件

参考
https://github.com/kubernetes/dashboard/wiki/Access-control
https://github.com/kubernetes/dashboard/issues/2558
https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/

#1.将kubernetes-server-linux-amd64.tar.gz 解压后，再解压其中的 kubernetes-src.tar.gz 文件。dashboard 对应的目录是：cluster/addons/dashboard ，拷贝dashboard的文件

mkdir -p /etc/kubernetes/dashboard

cp -a /opt/kubernetes/src/cluster/addons/dashboard/{dashboard-configmap.yaml,dashboard-controller.yaml,dashboard-rbac.yaml,dashboard-secret.yaml,dashboard-service.yaml} /etc/kubernetes/dashboard

#2.修改配置文件
sed -i "[email protected]:.*@image: registry.cn-hangzhou.aliyuncs.com/google_containers/kubernetes-dashboard-amd64:[email protected]" /etc/kubernetes/dashboard/dashboard-controller.yaml
sed -i "/spec/a\  type: NodePort" /etc/kubernetes/dashboard/dashboard-service.yaml
sed -i "/targetPort/a\    nodePort: 32700" /etc/kubernetes/dashboard/dashboard-service.yaml

#3.执行所有定义文件
kubectl create -f /etc/kubernetes/dashboard

#4.查看分配的NodePort
kubectl -n kube-system get all -o wide
#
NAME                                        READY   STATUS    RESTARTS   AGE
pod/coredns-8854569d4-5vshp                 1/1     Running   0          119m
pod/kubernetes-dashboard-7d5f7c58f5-mr8zn   1/1     Running   0          5m1s
#
NAME                           TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)                  AGE
service/kube-dns               ClusterIP   10.254.0.2     <none>        53/UDP,53/TCP,9153/TCP   142m
service/kubernetes-dashboard   NodePort    10.254.63.16   <none>        443:32700/TCP            51s
#
NAME                                   READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/coredns                1/1     1            1           119m
deployment.apps/kubernetes-dashboard   1/1     1            1           5m4s
#
NAME                                              DESIRED   CURRENT   READY   AGE
replicaset.apps/coredns-8854569d4                 1         1         1       119m
replicaset.apps/kubernetes-dashboard-7d5f7c58f5   1         1         1       5m4s

kubectl -n kube-system describe pod kubernetes-dashboard

#NodePort映射到dasrd pod 443端口；
#dashboard的 --authentication-mode 支持 token、basic，默认为 token。如果使用 basic，则 kube-apiserver 必须配置 '--authorization-mode=ABAC' 和 '--basic-auth-file' 参数。

#5.查看 dashboard 支持的命令行参数
kubectl exec --namespace kube-system -it kubernetes-dashboard-7d5f7c58f5-mr8zn -- /dashboard --help

#6.访问dashboard
# 为了集群安全，从1.7开始，dashboard只允许通过https访问，如果使用kube proxy则必须监听localhost或 127.0.0.1，对于NodePort没有这个限制，但是仅建议在开发环境中使用。对于不满足这些条件的登录访问，在登录成功后浏览器不跳转，始终停在登录界面。
参考1：https://github.com/kubernetes/dashboard/wiki/Accessing-Dashboard---1.7.X-and-above
参考2：https://github.com/kubernetes/dashboard/issues/2540
# 三种访问 dashboard 的方式
# 通过NodePort访问dashboard：
# 通过kubectl proxy访问dashboard：
# 通过kube-apiserver访问dashboard；

#7.通过NodePort访问dashboard
# kubernetes-dashboard服务暴露了NodePort，可以使用http://NodeIP:NodePort地址访问dashboard；

#8.通过 kubectl proxy 访问 dashboard
#启动代理：
kubectl proxy --address='localhost' --port=8086 --accept-hosts='^*$'
# --address 必须为 localhost 或 127.0.0.1；
# 需要指定 --accept-hosts 选项，否则浏览器访问 dashboard 页面时提示 “Unauthorized”；
# 浏览器访问 URL：http://127.0.0.1:8086/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy

#9.通过 kube-apiserver 访问 dashboard
# 获取集群服务地址列表：
kubectl cluster-info
# 必须通过 kube-apiserver 的安全端口(https)访问 dashbaord，访问时浏览器需要使用自定义证书，否则会被 kube-apiserver 拒绝访问。
# 创建和导入自定义证书的步骤，参考：A.浏览器访问kube-apiserver安全端口
# 浏览器访问 URL：https://192.168.10.100:8443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy

#10.创建登录 Dashboard 的 token 和 kubeconfig 配置文件
# 上面提到，Dashboard 默认只支持 token 认证，所以如果使用 KubeConfig 文件，需要在该文件中指定 token，不支持使用 client 证书认证。

# 创建登录 token，访问 dashboard时使用
kubectl create sa dashboard-admin -n kube-system
kubectl create clusterrolebinding dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin
ADMIN_SECRET=$(kubectl get secrets -n kube-system | grep dashboard-admin | awk '{print $1}')
DASHBOARD_LOGIN_TOKEN=$(kubectl describe secret -n kube-system ${ADMIN_SECRET} | grep -E '^token' | awk '{print $2}')
echo ${DASHBOARD_LOGIN_TOKEN}

#使用输出的 token 登录 Dashboard。

#创建使用 token 的 KubeConfig 文件
cd /root/ssl
#设置集群参数
kubectl config set-cluster kubernetes \
  --certificate-authority=ca.pem \
  --embed-certs=true \
  --server=https://192.168.10.100:8443 \
  --kubeconfig=dashboard.kubeconfig

#设置客户端认证参数，使用上面创建的 Token
kubectl config set-credentials dashboard_user \
  --token=${DASHBOARD_LOGIN_TOKEN} \
  --kubeconfig=dashboard.kubeconfig

#设置上下文参数
kubectl config set-context default \
  --cluster=kubernetes \
  --user=dashboard_user \
  --kubeconfig=dashboard.kubeconfig

#设置默认上下文
kubectl config use-context default --kubeconfig=dashboard.kubeconfig

#生成的 dashboard.kubeconfig 登录 Dashboard。
#由于缺少 Heapster 插件，当前 dashboard 不能展示 Pod、Nodes 的 CPU、内存等统计数据和图表；

2.9.3、部署heapster插件

Heapster是一个收集者，将每个Node上的cAdvisor的数据进行汇总，然后导到第三方工具(如InfluxDB)。Heapster 是通过调用 kubelet 的 http API 来获取 cAdvisor 的 metrics 数据的。由于 kublet 只在 10250 端口接收 https 请求，故需要修改 heapster 的 deployment 配置。同时，需要赋予 kube-system:heapster ServiceAccount 调用 kubelet API 的权限。

参考：配置 heapster：https://github.com/kubernetes/heapster/blob/master/docs/source-configuration.md

heapster下载地址：https://github.com/kubernetes-retired/heapster/releases

#1.解压heapster
mkdir /opt/heapster
tar -xzvf heapster-1.5.4.tar.gz -C /opt/heapster

#2.修改配置
mkdir -p /etc/kubernetes/heapster
cp -a /opt/heapster/deploy/kube-config/influxdb/{grafana.yaml,heapster.yaml,influxdb.yaml} /etc/kubernetes/heapster

sed -i "[email protected]:.*@image: registry.cn-hangzhou.aliyuncs.com/google_containers/heapster-grafana-amd64:[email protected]" /etc/kubernetes/heapster/grafana.yaml

sed -i "67a\  type: NodePort" /etc/kubernetes/heapster/grafana.yaml

sed -i "/targetPort/a\    nodePort: 32699" /etc/kubernetes/heapster/grafana.yaml

sed -i "[email protected]:.*@image: registry.cn-hangzhou.aliyuncs.com/google_containers/heapster-amd64:[email protected]" /etc/kubernetes/heapster/heapster.yaml

# 由于 kubelet 只在 10250 监听 https 请求，故添加相关参数；
sed -i "[email protected]=.*@source=kubernetes:https://kubernetes.default?kubeletHttps=true\&[email protected]" /etc/kubernetes/heapster/heapster.yaml

sed -i "[email protected]:.*@image: registry.cn-hangzhou.aliyuncs.com/google_containers/heapster-influxdb-amd64:[email protected]" /etc/kubernetes/heapster/influxdb.yaml

# 将 serviceAccount kube-system:heapster 与 ClusterRole system:kubelet-api-admin 绑定，授予它调用 kubelet API 的权限；
cp -a /opt/heapster/deploy/kube-config/rbac/heapster-rbac.yaml /etc/kubernetes/heapster

cat > /etc/kubernetes/heapster/heapster-rbac.yaml <<EOF
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: heapster-kubelet-api
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:kubelet-api-admin
subjects:
- kind: ServiceAccount
  name: heapster
  namespace: kube-system
EOF

#3.执行所有定义文件
kubectl create -f  /etc/kubernetes/heapster
kubectl apply -f  /etc/kubernetes/heapster/heapster-rbac.yaml

#4.检查执行结果
kubectl -n kube-system get all -o wide | grep -E 'heapster|monitoring'

kubectl -n kube-system describe pod heapster

kubectl -n kube-system describe pod monitoring

# 检查 kubernets dashboard 界面，可以正确显示各 Nodes、Pods 的 CPU、内存、负载等统计数据和图表：

kubectl -n kube-system get all -o wide

kubectl -n kube-system logs heapster-7bdc95b5cc-8h7zt

#5.访问 grafana,通过 NodePort 访问：
kubectl get svc -n kube-system|grep -E 'monitoring|heapster'
#显示如下,grafana 监听 NodePort 32699；
heapster               ClusterIP   10.254.159.62    <none>        80/TCP                   12m     k8s-app=heapster
monitoring-grafana     NodePort    10.254.167.38    <none>        80:32699/TCP             4m29s   k8s-app=grafana
monitoring-influxdb    ClusterIP   10.254.155.141   <none>        8086/TCP                 12m     k8s-app=influxdb

kubectl get pod -n kube-system -o wide |grep -E 'monitoring|heapster' 
#显示如下，然后浏览器访问 URL：http://192.168.10.16:32699/?orgId=1
heapster-7bdc95b5cc-8h7zt               1/1     Running   0          13m     172.30.34.4    192.168.10.15
monitoring-grafana-6cf5948cd4-rstxk     1/1     Running   0          5m      172.30.85.11   192.168.10.16
monitoring-influxdb-7d6c5fb944-qfd65    1/1     Running   0          13m     172.30.85.10   192.168.10.16

#6.通过 kube-apiserver 访问： 获取 monitoring-grafana 服务 URL：
kubectl cluster-info    
#查到浏览器访问URL：https://192.168.10.100:8443/api/v1/namespaces/kube-system/services/monitoring-grafana/proxy

#通过 kubectl proxy 访问：创建代理
kubectl proxy --address='192.168.10.16' --port=8086 --accept-hosts='^*$'
# 浏览器访问 URL：http://192.168.10.16:8086/api/v1/namespaces/kube-system/services/monitoring-grafana/proxy/?orgId=1

上一篇： Java并发6：CountDownLatch &CyclicBarrier &Semaphore

下一篇： Eclipse使用maven搭建spring mvc图文教程

推荐阅读