253. 安装部署Kafka

http://kafka.apache.org/
Kafka是有状态的服务，一般部署在Kubernetes之外，本次部署在 k8s7-11.host.com。由于后面需要部署的Kafka-manager只支持到 2.2.0 版本，因此这次部署采用kafka_2.12-2.2.0版本，其中2.12为Scala版本号。

1. 下载安装kafka

[root@hdss7-11 src]# wget https://archive.apache.org/dist/kafka/2.2.0/kafka_2.12-2.2.0.tgz
[root@hdss7-11 src]# tar -xf kafka_2.12-2.2.0.tgz -C /opt/
[root@hdss7-11 src]# ln -s /opt/kafka_2.12-2.2.0 /opt/kafka

配置：
[root@hdss7-11 ~]# vim /opt/kafka/config/server.properties
......
log.dirs=/data/kafka/logs
# 超过10000条日志强制刷盘，超过1000ms刷盘
log.flush.interval.messages=10000
log.flush.interval.ms=1000
# 填写需要连接的 zookeeper 集群地址，当前连接本地的 zk 集群。
zookeeper.connect=localhost:2181
# 新增以下两项
delete.topic.enable=true
host.name=k8s7-11.host.com

启动：
[root@hdss7-11 ~]# mkdir -p /data/kafka/logs
[root@hdss7-11 ~]# /opt/kafka/bin/kafka-server-start.sh -daemon /opt/kafka/config/server.properties

[root@hdss7-11 ~]# netstat -lntp|grep 9092
tcp6       0      0 10.4.7.11:9092          :::*                    LISTEN      121952/java         
tcp6       0      0 :::41211                :::*                    LISTEN      121952/java

2. 部署Kafka-manager

Kafka-manager是一款管理Kafka集群的软件，建议安装，github地址:https://github.com/yahoo/CMAK

2.1 镜像制作

# 存在的问题：
# 1. kafka-manager 改名为 CMAK，压缩包名称和内部目录名发生了变化
# 2. sbt 编译需要下载很多依赖，因为不可描述的原因，速度非常慢，个人非VPN网络大概率失败
# 3. 因本人不具备VPN条件，编译失败。又因为第一条，这个dockerfile大概率需要修改
# 4. 生产环境中一定要自己重新做一份！
FROM hseeberger/scala-sbt
ENV ZK_HOSTS=localhost:2181 \
    KM_VERSION=2.0.0.2
RUN mkdir -p /tmp && \
    cd /tmp && \
    wget https://github.com/yahoo/kafka-manager/archive/${KM_VERSION}.tar.gz && \
    tar xf ${KM_VERSION}.tar.gz && \
    cd /tmp/kafka-manager-${KM_VERSION} && \
    sbt clean dist && \
    unzip  -d / ./target/universal/kafka-manager-${KM_VERSION}.zip && \
    rm -fr /tmp/${KM_VERSION} /tmp/kafka-manager-${KM_VERSION}

WORKDIR /kafka-manager-${KM_VERSION}
EXPOSE 9000
ENTRYPOINT ["./bin/kafka-manager","-Dconfig.file=conf/application.conf"]

[root@hdss7-200 ~]# docker pull chengyinwu/kafka-manager:v2.0.0.2
[[e-mail] /data/dockerfile/kafka-manager]# docker load -i kafka-manager-v2.0.0.2.tar 
[root@hdss7-200 ~]# docker tag chengyinwu/kafka-manager:v2.0.0.2 harbor.od.com/public/kafka-manager:v2.0.0.2
[root@hdss7-200 ~]# docker push harbor.od.com/public/kafka-manager:v2.0.0.2

2.2 准备资源配置清单

[[e-mail] /data/k8s-yaml/kafka-manager]# cat deployment.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kafka-manager
  namespace: infra
  labels: 
    name: kafka-manager
spec:
  replicas: 1
  selector:
    matchLabels: 
      app: kafka-manager
  template:
    metadata:
      labels: 
        app: kafka-manager
    spec:
      containers:
      - name: kafka-manager
        image: harbor.od.com/infra/kafka-manager:v2.0.0.2
        ports:
        - containerPort: 9000
          protocol: TCP
        env:
        - name: ZK_HOSTS
          value: zk1.od.com:2181
        - name: APPLICATION_SECRET
          value: letmein

[[e-mail] /data/k8s-yaml/kafka-manager]# cat service.yaml 
apiVersion: v1
kind: Service
metadata: 
  name: kafka-manager
  namespace: infra
spec:
  ports:
  - protocol: TCP
    port: 9000
    targetPort: 9000
  selector: 
    app: kafka-manager

[[e-mail] /data/k8s-yaml/kafka-manager]# cat ingress.yaml 
apiVersion: extensions/v1beta1
kind: Ingress
metadata: 
  name: kafka-manager
  namespace: infra
spec:
  rules:
  - host: km.od.com
    http:
      paths:
      - path: /
        backend: 
          serviceName: kafka-manager
          servicePort: 9000

2.3 DNS解析

km                 A    10.4.7.10
[[e-mail] ~]# systemctl restart named
[[e-mail] ~]# dig -t A km.od.com @10.4.7.11 +short
10.4.7.10

2.4 应用资源配置清单

[[e-mail] ~]# kubectl apply -f http://k8s-yaml.od.com/kafka-manager/deployment.yaml
deployment.apps/kafka-manager created
[[e-mail] ~]# kubectl apply -f http://k8s-yaml.od.com/kafka-manager/service.yaml
service/kafka-manager created
[[e-mail] ~]# kubectl apply -f http://k8s-yaml.od.com/kafka-manager/ingress.yaml
ingress.extensions/kafka-manager created

3. 验证