部署zookeeper和kafka集群
文章目录
提供者：MappleZF
版本：1.0.0
一、部署zookeeper
1.1 准备环境
准备安装包、安装jdk
1.2 安装解压
tar xf apache-zookeeper-3.6.2-bin.tar.gz -C /opt/
cd /opt && mv apache-zookeeper-3.6.2-bin/ zookeeper
mkdir /opt/zookeeper/{data,logs}
1.3 添加DNS域名解析或修改hosts文件
[root@lb03.host.com:/root]# vim /var/named/lowan.com.zone
zk01 A 192.168.13.101
zk02 A 192.168.13.102
zk03 A 192.168.13.103
1.4 配置zoo.cfg文件
cd /opt/zookeeper/conf/
cp -p zoo_sample.cfg zoo.cfg
conf]# vim zoo.cfg
内容如下：
tickTime=2000
initLimit=10
synclimit=5
dataDir=/opt/zookeeper/data
dataLogDir=/opt/zookeeper/logs
clientPort=2181
server.1=192.168.13.101:2888:3888
server.2=192.168.13.102:2888:3888
server.3=192.168.13.103:2888:3888
autopurge.purgeInterval=12
max_session_timeout=40000
min_session_timeout=4000
分发cfg文件
scp zoo.cfg k8smaster02:/opt/zookeeper/conf/
scp zoo.cfg k8smaster03:/opt/zookeeper/conf/
1.5 创建myid文件
# 每个服务器上的myid内容都不同，且需要保证和自己的zoo.cfg配置文件中"server.id=host:port:port"的id值一致
# id的范围是1~255
[root@k8smaster01.host.com:/opt/zookeeper/conf]# echo "1" > /opt/zookeeper/data/myid
[root@k8smaster02.host.com:/opt]# echo "2" > /opt/zookeeper/data/myid
[root@k8smaster03.host.com:/opt]# echo "3" > /opt/zookeeper/data/myid
1.6 启动zookeeper
/opt/zookeeper/bin/zkServer.sh start
ps -ef | grep zookeeper
1.7 添加守护进程
vim /etc/systemd/system/zookeeper.service
[Unit]
Description=zookeeper server
Requires=network.target remote-fs.target
After=network.target remote-fs.target
[Service]
Type=forking
User=root
Environment=JAVA_HOME=/usr/local/java
ExecStart=/bin/sh -c '/opt/zookeeper/bin/zkServer.sh start /opt/zookeeper/conf/zoo.cfg'
ExecStop=/opt/zookeeper/bin/zkServer.sh stop
Restart=on-failure
RestartSec=2s
[Install]
WantedBy=multi-user.target
:x 保存退出
/opt/zookeeper/bin/zkServer.sh stop
systemctl daemon-reload
systemctl enable zookeeper.service
systemctl start zookeeper.service
systemctl status zookeeper.service
二、部署Kafka
2.1 准备安装包
wget -c https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.6.0/kafka_2.13-2.6.0.tgz
2.2 安装解压
tar -xf kafka_2.13-2.6.0.tgz -C /opt/
cd /opt/
mv kafka_2.13-2.6.0 kafka
mkdir -p /mnt/cephfs/kafka0/logs /mnt/cephfs/kafka1/logs /mnt/cephfs/kafka2/logs
2.3 添加DNS域名解析或修改hosts文件
[root@lb03.host.com:/root]# vim /var/named/lowan.com.zone
kafka0 A 192.168.13.101
kafka1 A 192.168.13.102
kafka2 A 192.168.13.103
systemctl restart named
2.4 配置server.properties文件
vim /opt/kafka/config/server.properties
---------------------------------------->
kafka0的配置如下：
broker.id=0
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://kafka0.lowan.com:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/mnt/cephfs/kafka0/logs
num.partitions=3
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=zk01.lowan.com:2181,zk02.lowan.com:2181,zk03.lowan.com:2181
zookeeper.connection.timeout.ms=12000
group.initial.rebalance.delay.ms=0
group.max.session.timeout.ms=300000
group.min.session.timeout.ms=6000
auto.create.topics.enable=true
delete.topics.enable=true
---------------------------------------->
kafka1的配置如下：
broker.id=1
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://kafka1.lowan.com:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/mnt/cephfs/kafka1/logs
num.partitions=3
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=zk01.lowan.com:2181,zk02.lowan.com:2181,zk03.lowan.com:2181
zookeeper.connection.timeout.ms=12000
group.initial.rebalance.delay.ms=0
group.max.session.timeout.ms=300000
group.min.session.timeout.ms=6000
auto.create.topics.enable=true
delete.topics.enable=true
---------------------------------------->
kafka2的配置如下：
broker.id=2
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://kafka2.lowan.com:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/mnt/cephfs/kafka2/logs
num.partitions=3
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=zk01.lowan.com:2181,zk02.lowan.com:2181,zk03.lowan.com:2181
zookeeper.connection.timeout.ms=12000
group.initial.rebalance.delay.ms=0
group.max.session.timeout.ms=300000
group.min.session.timeout.ms=6000
auto.create.topics.enable=true
delete.topics.enable=true
2.5 启动kafka
#启动
/opt/kafka/bin/kafka-server-start.sh -daemon /opt/kafka/config/server.properties
#检查java进程
jps
ps -ef | grep kafka
#如停止
/opt/kafka/bin/kafka-server-stop.sh
2.6 添加守护进程
vim /etc/systemd/system/kafka.service
[Unit]
Description=kafka server
Requires=zookeeper.service
After=zookeeper.service
[Service]
Type=simple
User=root
Environment=JAVA_HOME=/usr/local/java
ExecStart=/bin/sh -c '/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties'
ExecStop=/bin/sh -c '/opt/kafka/bin/kafka-server-stop.sh'
Restart=on-failure
RestartSec=2s
SuccessExitStatus=0 143
[Install]
WantedBy=multi-user.target
:x 保存退出
systemctl daemon-reload
systemctl enable kafka.service
systemctl start kafka.service
systemctl status kafka.service
三、Kubernetes部署kafka-manager
3.1 准备docker镜像文件
方式一：直接pull
docker pull vimagick/cmak:3.0.0.5
docker tag vimagick/cmak:3.0.0.5 harbor.iot.com/kubernetes/kubernetes-kafkamanager:3.0.5
docker push harbor.iot.com/kubernetes/kubernetes-kafkamanager:3.0.5
方式二： 编辑Dockerfile
vim Dockerfile
内容如下：
FROM debian:buster
ENV SCALA_VERSION 2.12.10
ENV SBT_VERSION 1.3.8
ENV CMAK_VERSION 3.0.0.5
RUN set -xe \
&& apt update \
&& apt install -y openjdk-11-jre-headless curl wget \
&& wget -q https://downloads.lightbend.com/scala/$SCALA_VERSION/scala-$SCALA_VERSION.deb -O scala.deb \
&& wget -q https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb -O sbt.deb \
&& dpkg -i scala.deb sbt.deb \
&& rm scala.deb sbt.deb \
&& rm -rf /var/lib/apt/lists/*
workdir /opt/cmak
RUN set -xe \
&& mkdir src \
&& curl -sSL https://github.com/yahoo/CMAK/archive/$CMAK_VERSION.tar.gz | tar xz --strip 1 -C src \
&& cd src \
&& sbt clean universal:packageZipTarball \
&& cd .. \
&& tar xzf src/target/universal/cmak-$CMAK_VERSION.tgz --strip 1 \
&& rm -rf src
VOLUME /opt/cmak/conf
EXPOSE 9000
ENTRYPOINT ["bin/cmak"]
CMD ["-Dconfig.file=conf/application.conf", "-Dhttp.port=9000", "-Dpidfile.path=/dev/null"]
HEALTHCHECK CMD curl -f http://127.0.0.1/api/health || exit 1
:x 保存退出
docker build -t harbor.iot.com/kubernetes/kubernetes-kafkamanager:3.0.5 .
3.2 创建svc
[root@k8smaster01.host.com:/data/yaml/kafka-manager]# vim svc-kafkamanager.yaml
apiVersion: v1
kind: Service
Metadata:
name: kafka-manager
spec:
ports:
- protocol: TCP
port: 9000
targetPort: 9000
selector:
name: kafka-manager
3.3 创建ingressroute
[root@k8smaster01.host.com:/data/yaml/kafka-manager]# vim ingressroute-kafkamanager.yaml
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
Metadata:
labels:
name: kafka-manager
name: kafka-manager-ingress
annotations:
traefik.ingress.kubernetes.io/router.entrypoints: web
kubernetes.io/ingress.class: "traefik"
spec:
entryPoints:
- web
routes:
- match: Host(`cmak.lowan.com`) && PathPrefix(`/`)
kind: Rule
services:
- name: kafka-manager
port: 9000
3.4 创建deployment
[root@k8smaster01.host.com:/data/yaml/kafka-manager]# vim deployment-kafkamanager.yaml
apiVersion: apps/v1
kind: Deployment
Metadata:
name: kafka-manager
labels:
name: kafka-manager
spec:
replicas: 1
selector:
matchLabels:
name: kafka-manager
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
maxSurge: 1
revisionHistoryLimit: 7
progressDeadlineseconds: 600
template:
Metadata:
labels:
name: kafka-manager
spec:
containers:
- name: kafka-manager
image: harbor.iot.com/kubernetes/kubernetes-kafkamanager:3.0.0.5
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9000
protocol: TCP
env:
- name: ZK_HOSTS
value: "zk01.lowan.com:2181,zk02.lowan.com:2181,zk03.lowan.com:2181"
- name: KAFKA_MANAGER_USERNAME
value: admin
- name: KAFKA_MANAGER_PASSWORD
value: admin
resources:
limits:
cpu: 1500m
memory: 4000Mi
requests:
cpu: 200m
memory: 500Mi
imagePullSecrets:
- name: harbor
terminationGracePeriodSeconds: 30
securityContext:
runAsUser: 0
3.5 添加DNS域名解析
[root@lb03.host.com:/root]# vim /var/named/lowan.com.zone
cmak A 192.168.13.100
systemctl restart named
3.6 应用配置清单
kubectl apply -f svc-kafkamanager.yaml
kubectl apply -f deployment-kafkamanager.yaml
kubectl apply -f ingressroute-kafkamanager.yaml
3.7 访问网页并配置
访问：http://cmak.lowan.com/
添加kafka集群
`
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 dio@foxmail.com 举报，一经查实，本站将立刻删除。
