快速部署之 cmak / kafka-manager

参考 1: 编译部署 kafka-manager

请一定要注意版本匹配，本文使用 kafka-2.12-2.5.0 + cmak-2.0.0.2(kafka-manager-2.0.0.2)，踩过的抗，如使用 cmak-3.0.0.5 连接 kafka-2.12-2.5.0/zookeeper-3.4.6 会报错如UnsupportedClassVersionError: controllers/routes has been compiled by a more recent version of the Java Runtime (class file version 55.0), this version of the Java Runtime only recognizes class file versions up to 52.0。

1. 编译安装

1.1 编译

git clone https://github.com/yahoo/CMAK.git # 新版改名了
cd CMAK
git checkout 1.3.3.16 # 注: 这个tag版本的 sbt 脚本中使用了 http://repo.typesafe.com 的地址，要改为 https，新tag版本已修复

# 解决大陆网络慢
mkdir -p ~/.sbt
cat <<-'EOF'>~/.sbt/repositories
[repositories]
aliyun: https://maven.aliyun.com/nexus/content/groups/public/
typesafe: https://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
sonatype-oss-releases
maven-central
sonatype-oss-snapshots
EOF

./sbt clean dist

过程较慢，如果重试几次还不行就得配置科学上网吧，参考 2: Linux轻松搭建Shadowsocks服务器

直接下载已编译包

# Installing
mkdir -p /opt/apps/emr
cd /opt/apps/emr
wget https://gitee.com/wl4g-collect/kafka-manager/attach_files/836364/download/kafka-manager-1.3.3.16.zip
unzip kafka-manager-1.3.3.16.zip

# Link kafka-manager home directory.
ln -snf $(pwd)/$(ls |grep kafka-manager) /usr/lib/kafka-manager-current

2. 主机部署

2.1 环境配置

cat <<-'EOF'> /etc/profile.d/profile-kafka-manager.sh
#/*
# * Copyright 2017 ~ 2025 the original author or authors. <wanglsir@gmail.com, 983708408@qq.com>
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */
export KAFKA_MANAGER_HOME=/usr/lib/kafka-manager-current
export KAFKA_MANAGER_CONF_DIR=/etc/emr/kafka-manager-conf
export KAFKA_MANAGER_DATA_DIR=/mnt/disk1/kafka-manager
export KAFKA_MANAGER_LOG_DIR=/mnt/disk1/log/kafka-manager
export KAFKA_MANAGER_OPTS="-Dhttp.port=9990"
export PATH=$PATH:$KAFKA_MANAGER_HOME/bin

2.3 运行配置

# 手动编辑修改 zookeeper 连接地址
$KAFKA_MANAGER_CONF_DIR/application.conf
# 找到 kafka-manager.zkhosts

2.4 管理脚本

cat <<-'EOF'> /etc/init.d/kafka-manager.sh
#!/bin/bash
# chkconfig: - 85 15
#/*
# * Copyright 2017 ~ 2025 the original author or authors. <Wanglsir@gmail.com, 983708408@qq.com>
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# *      http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */
#
[ -f /etc/sysconfig/network ] && . /etc/sysconfig/network
[ "$NETWORKING" = "no" ] && exit 0
#
# Load the user environment.
[ -f "/etc/bashrc" ] && . /etc/bashrc
[ -f "/etc/bash.bashrc" ] && . /etc/bash.bashrc # e.g ubuntu
if [ "$USER" == "root" ]; then
  . "/root/.bash_profile"
  . "/root/.bashrc"
else
  [ -f "/home/$USER/.bash_profile" ] && . /home/$USER/.bash_profile
  [ -f "/home/$USER/.bashrc" ] && . /home/$USER/.bashrc
  # Mac OS
  [ -f "/Users/$USER/.bash_profile" ] && . /Users/$USER/.bash_profile
  [ -f "/Users/$USER/.bashrc" ] && . /Users/$USER/.bashrc
fi
[ -f "/etc/profile.d/profile-kafka-manager.sh" ] && . /etc/profile.d/profile-kafka-manager.sh

# Initialize directory. (if not exists)
mkdir -p $KAFKA_MANAGER_CONF_DIR
mkdir -p $KAFKA_MANAGER_DATA_DIR
mkdir -p $KAFKA_MANAGER_LOG_DIR

function start() {
  local jvmOpts=${KAFKA_MANAGER_OPTS:-Dhttp.port=9000}
  local pids=$(getPids)
  if [ -z "$pids" ]; then
    # Must remove.
    rm -rf $KAFKA_MANAGER_HOME/RUNNING_PID
    rm -rf $KAFKA_MANAGER_HOME/application.home*
    if [ "$PPID" == "1" ]; then # Systemd call.
      $KAFKA_MANAGER_HOME/bin/kafka-manager -Dconfig.file=$KAFKA_MANAGER_CONF_DIR/application.conf ${jvmOpts} >"$KAFKA_MANAGER_LOG_DIR/kafka-manager.stdout" 2>&1
    else # Normal shell call.
      nohup $KAFKA_MANAGER_HOME/bin/kafka-manager -Dconfig.file=$KAFKA_MANAGER_CONF_DIR/application.conf ${jvmOpts} >"$KAFKA_MANAGER_LOG_DIR/kafka-manager.stdout" 2>&1 &
    fi

    echo -n "Starting kafka-manager ..."
    while true
    do
      pids=$(getPids)
      if [ "$pids" == "" ]; then
        echo -n ".";
        sleep 0.8;
      else
        echo $pids >"$KAFKA_MANAGER_DATA_DIR/kafka-manager.pid"
        break
      fi
    done
    echo -e "\nStarted kafka-manager on "$pids
  else
    echo "kafka-manager process is running "$pids
  fi
}

function stop() {
  local pids=$(getPids)
  if [ -z "$pids" ]; then
    echo "kafka-manager not running!"
  else
    echo -n "Stopping kafka-manager for $pids ..."
    kill -s TERM $pids
    while true
    do
      pids=$(getPids)
      if [ "$pids" == "" ]; then
        \rm -f $KAFKA_MANAGER_DATA_DIR/kafka-manager.pid
        break
      else
        echo -n ".";
        sleep 0.8;
      fi
    done
    echo -e "\nStopped kafka-manager !"
  fi
}

function status() {
  ps -ef | grep -v grep | grep $KAFKA_MANAGER_CONF_DIR
}

function getPids() {
  local pids=$(ps ax | grep -i "$KAFKA_MANAGER_CONF_DIR" | grep -v grep | awk '{print $1}')
  echo $pids # Output execution result value.
  return 0 # Return the execution result code.
}

# --- Main call. ---
case $1 in
  status)
    status
    ;;
  start)
    start
    ;;
  stop)
    stop
    ;;
  restart)
    stop
    start
    ;;
    *|--help|help)
  echo $"Usage: {start|stop|restart|status}"
  exit 2
esac
EOF
chmod +x /etc/init.d/kafka-manager.sh

2.5 服务配置

cat <<-'EOF'> /etc/systemd/system/kafka-manager.service
# Copyright (c) 2017 ~ 2025, the original author wangl.sir individual Inc,
# All rights reserved. Contact us <wanglsir@gmail.com, 983708408@qq.com>
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# see: https://github.com/yahoo/cmak or https://gitee.com/wl4g-collect/kafka-manager/releases/1.3.3.16
[Unit]
Description=Yahoo CMAK for Apache Kafka
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=simple
ExecStart=/bin/bash -c "/etc/init.d/kafka-manager.sh start"
ExecReload=/bin/bash -c "/etc/init.d/kafka-manager.sh restart"
ExecStop=/bin/bash -c "/etc/init.d/kafka-manager.sh stop"
StandardOutput=null
StandardError=journal+console
LimitNOFILE=1048576
LimitNPROC=1048576
LimitCORE=infinity
TimeoutStartSec=60
Restart=always
KillMode=process
User=root
Group=root
SuccessExitStatus=143

[Install]
WantedBy=multi-user.target
EOF

# 启动服务
systemctl daemon-reload
systemctl enable kafka-manager
systemctl start kafka-manager
# 或 /etc/init.d/kafka-manager.sh start

2.6 访问 UI

http://kafka-manger-ip:9000

3. docker 部署

如果容器引擎是containerd，则：

export baseUrl='https://github.com/containerd/nerdctl/releases/download/v0.21.0/nerdctl-0.21.0-linux-amd64.tar.gz'

curl -OL $baseUrl
# 或使用代理加速
curl -OL "https://ghproxy.com/$baseUrl"

# 解压安装
tar -xf nerdctl-*
mv nerdctl /bin/

# 设置别名
alias docker='nerdctl' # if the engine is containerd.

3.1 部署老版 (最高仅支持 <=2.2.0)

docker run -d \
--name cmak \
--network host \
--restart always \
-e ZK_HOSTS=10.0.0.162:2181 \
registry.cn-shenzhen.aliyuncs.com/wl4g/kafka-manager:2.0.0.2 -Dhttp.port=9000

3.2 （推荐）部署新版 (>=0.8.1.1 支持 <=3.1.0+)

官方说明：hub.docker.com/r/deltaprojects/kafka-manager

docker run \
--name=cmak \
--network=bridge \
--restart=always \
--detach=true \
-e="ZK_HOSTS=172.24.47.4:2181,172.24.47.5:2181,172.24.47.6:2181" \
-e="KAFKA_MANAGER_AUTH_ENABLED=true" \
-e="KAFKA_MANAGER_USERNAME=admin" \
-e="KAFKA_MANAGER_PASSWORD=admin" \
-e="KAFKA_MANAGER_LDAP_ENABLED=false" \
-e="KAFKA_MANAGER_LDAP_SERVER=ldap.example.com" \
-e="KAFKA_MANAGER_LDAP_PORT=636" \
-e="KAFKA_MANAGER_LDAP_USERNAME=cn=rouser,dc=example,dc=com" \
-e="KAFKA_MANAGER_LDAP_PASSWORD=rouserpassword" \
-e="KAFKA_MANAGER_LDAP_SEARCH_BASE_DN=ou=users,dc=example,dc=com" \
-e="KAFKA_MANAGER_LDAP_SEARCH_FILTER=(&(objectClass=inetOrgPerson)(&(uid=\$capturedLogin\$)(|(memberof=cn=Operations,ou=Groups,dc=example,dc=com)(memberof=cn=Development,ou=Groups,dc=example,dc=com))))" \
-e="KAFKA_MANAGER_LDAP_CONNECTION_POOL_SIZE=10" \
-e="KAFKA_MANAGER_LDAP_SSL=false" \
-e="KAFKA_MANAGER_EXTRA_PLAY_OPTS=-Dhttp.port=9000 -Dhttps.port=9443" \
-e="APPLICATION_SECRET=abcdefghijklmnopqrstyvwxyz" \
-e="JAVA_OPTS=-Djava.net.preferIPv4Stack=true -Xmx2G" \
-p 0.0.0.0:1443:9443 \
-p 0.0.0.0:19000:9000 \
-v /opt/cloudera/parcels/CDH/jars/zookeeper-3.4.5-cdh6.3.1.jar:/opt/cmak/lib/org.apache.zookeeper.zookeeper-3.5.7.jar \
-v /opt/cloudera/parcels/CDH/jars/zookeeper-3.4.5-cdh6.3.1.jar:/opt/cmak/lib/org.apache.zookeeper.zookeeper-jute-3.5.7.jar \
deltaprojects/kafka-manager:v3.0.0.6-2

# 或使用克隆镜像
registry.cn-shenzhen.aliyuncs.com/wl4g/kafka-manager:v3.0.0.6-2

注：如果您要连接的 zookeeper server < 3.5.7，则必须按照如上命令示例(这是cdh环境，其他同理)，在运行容器时挂载替换为匹配的版本，否则在 kafka-manager 上 Add Cluster 会报错：KeeperErrorCode = Unimplemented for /kafka-manager/mutex Try again. ，参见官方 issue: https://github.com/yahoo/CMAK/issues/748

一	二	三	四	五	六	日
		1	2	3	4	5
6	7	8	9	10	11	12
13	14	15	16	17	18	19
20	21	22	23	24	25	26
27	28	29	30	31