From 06b8970fba7c52afbec223ac66cb7aa0c633a442 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 23 May 2022 08:38:16 +0800 Subject: [PATCH] Deployment and benchmark scripts for Kafka. topic, producer and consumer test work. add benchmark script on client add launcher on server side. modify from .py to .sh add prepare.sh 1. lowercase var to uppercase. 2. change way to detect if release tarball is already exist. modify hint, dot to colon, means ask for user choice. add sleep and create-topic after start server. remove single quotes in replace KAFKA_SEVER_IP add 5s sleep after start server move sleep part after start server into manager.sh change method of modify listen config. fix incorrect varname add kafka client yaml add READEME.md correct word spelling --- examples/tuning/kafka/README.md | 26 ++++ examples/tuning/kafka/benchmark_on_client.sh | 56 ++++++++ examples/tuning/kafka/kafka_client.yaml | 10 ++ examples/tuning/kafka/launcher_on_server.sh | 32 +++++ examples/tuning/kafka/manager.sh | 127 +++++++++++++++++++ examples/tuning/kafka/prepare.sh | 76 +++++++++++ 6 files changed, 327 insertions(+) create mode 100644 examples/tuning/kafka/README.md create mode 100644 examples/tuning/kafka/benchmark_on_client.sh create mode 100644 examples/tuning/kafka/kafka_client.yaml create mode 100644 examples/tuning/kafka/launcher_on_server.sh create mode 100755 examples/tuning/kafka/manager.sh create mode 100644 examples/tuning/kafka/prepare.sh diff --git a/examples/tuning/kafka/README.md b/examples/tuning/kafka/README.md new file mode 100644 index 00000000..a4105026 --- /dev/null +++ b/examples/tuning/kafka/README.md @@ -0,0 +1,26 @@ +# Kafka Tuning and Deployment + +## Prerequisites + +- To simulate a production environment, we need at least two hosts, a server and a client. + +- The /tmp/ on the server must be more significant than 2GB because the server will receive the massive amount of data sent by the client and write it to /tmp/.You can follow the steps below to increase the size. + + ```bash + mount -o remount,size=10G /tmp/ + ``` + +## Deployment ⚙️ + +Before tuning, we need to deploy Kafka first. + +You can download the Kafka release tarball() and put it under this directory so that the deployment script will skip the download. This tarball will be sent to the client by scp cause we will use the benchmark tool included in it. + +Running `prepare.sh` to deploy, follow the hint. The script will create a new ssh-key from server to client if not set up yet. + +## About Scrpits + +- `prepare.sh`, deploy the whole environment for tuning. +- `launcher_on_server.sh`, launch the `benchmark_on_client.sh` on the client, and copy the result log back to the server. +- `benchmark_on_client.sh`, do Kafka producer and consumer test, generate result log. +- `manager.sh`, control the Kafka service. diff --git a/examples/tuning/kafka/benchmark_on_client.sh b/examples/tuning/kafka/benchmark_on_client.sh new file mode 100644 index 00000000..98a42424 --- /dev/null +++ b/examples/tuning/kafka/benchmark_on_client.sh @@ -0,0 +1,56 @@ +#!/usr/bin/bash + +# Copyright (c) 2022 Huawei Technologies Co., Ltd. +# A-Tune is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +# ############################################# +# @Author : shangyingjie +# @Contact : yingjie@isrc.iscas.ac.cn +# @Date : 2022/5/23 +# @License : Mulan PSL v2 +# @Desc : kafka benchmark script +# ############################################# + +KAFKA_DIR=/root/kafka_2.13-3.2.0 +KAFKA_PRODUCER_TEST=$KAFKA_DIR/bin/kafka-producer-perf-test.sh +KAFKA_CONSUMER_TEST=$KAFKA_DIR/bin/kafka-consumer-perf-test.sh +KAFKA_SERVER_IP='will be replaced after running prepare.sh' +KAFKA_SERVER_PORT=9092 + +function get_producer_test_result() { + output=$( + $KAFKA_PRODUCER_TEST \ + --topic kafka-benchmark \ + --throughput -1 \ + --num-records 1000000 \ + --record-size 1024 \ + --producer-props acks=all bootstrap.servers="$KAFKA_SERVER_IP":"$KAFKA_SERVER_PORT" + ) + result_line=$(grep '1000000' <<<"$output") + arr=($result_line) + echo "${arr[3]}" +} + +function get_consumer_test_result() { + output=$( + $KAFKA_CONSUMER_TEST \ + --topic kafka-benchmark \ + --broker-list "$KAFKA_SERVER_IP":"$KAFKA_SERVER_PORT" \ + --messages 1000000 + ) + result_line=$(awk 'FNR == 2' <<<"$output") + IFS=',' read -r -a arr <<<"$result_line" + echo "${arr[5]}" +} + +producer_test_result=$(get_producer_test_result) +consumer_test_result=$(get_consumer_test_result) +sum=$(echo "$producer_test_result + $consumer_test_result" | bc) +echo "$sum" >/root/kafka_benchmark.log diff --git a/examples/tuning/kafka/kafka_client.yaml b/examples/tuning/kafka/kafka_client.yaml new file mode 100644 index 00000000..2c373783 --- /dev/null +++ b/examples/tuning/kafka/kafka_client.yaml @@ -0,0 +1,10 @@ +project: "kafka" +engine: "gbrt" +iterations: 100 +benchmark: "bash ./launcher_on_server.sh" +evaluations: + - name: "QPS" + info: + get: cat ./kafka_benchmark.log + type: "negative" + weight: 100 diff --git a/examples/tuning/kafka/launcher_on_server.sh b/examples/tuning/kafka/launcher_on_server.sh new file mode 100644 index 00000000..85190c9a --- /dev/null +++ b/examples/tuning/kafka/launcher_on_server.sh @@ -0,0 +1,32 @@ +#!/usr/bin/bash +# Copyright (c) 2020 Huawei Technologies Co., Ltd. +# A-Tune is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +############################################# +# @Author : shangyingjie +# @Contact : yingjie@isrc.iscas.ac.cn +# @Date : 2022/5/21 +# @License : Mulan PSL v2 +# @Desc : kafka benchmark trigger script +# ############################################# + +echo 'launch benchmark' + +cd "$(dirname "$0")" + +KAFKA_CLIENT_IP='will be replaced after running prepare.sh' + +if [ -f "./client_ssh_key" ]; then + ssh -i client_ssh_key root@"$KAFKA_CLIENT_IP" -t "/usr/bin/bash /root/benchmark_on_client.sh" + scp -i client_ssh_key root@"$KAFKA_CLIENT_IP":/root/kafka_benchmark.log ./ +else + ssh root@"$KAFKA_CLIENT_IP" -t "/usr/bin/bash /root/benchmark_on_client.sh" + scp root@"$KAFKA_CLIENT_IP":/root/kafka_benchmark.log ./ +fi diff --git a/examples/tuning/kafka/manager.sh b/examples/tuning/kafka/manager.sh new file mode 100755 index 00000000..7ccff7ee --- /dev/null +++ b/examples/tuning/kafka/manager.sh @@ -0,0 +1,127 @@ +#!/usr/bin/bash + +# Copyright (c) 2022 Huawei Technologies Co., Ltd. +# A-Tune is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +############################################# +# @Author : shangyingjie +# @Contact : yingjie@isrc.iscas.ac.cn +# @Date : 2022/5/21 +# @License : Mulan PSL v2 +# @Desc : Kafka management script +# ############################################# + +USAGE="Usage: task_manager [OPTION] +--start start kafka and zookeeper instance +--stop stop kafka and zookeeper instance +--restart delete benchmark topic and create new one, stop kafka and zookeeper instance, then start them. +--list-topic list existing topic +--create-topic create topic: kafka-benchmark +--delete-topic delete topic: kafka-benchmark +--help output this usage" + +KAFKA_DIR=./kafka_2.13-3.2.0 +ZOOKEEPER_SERVER_START_SCRPIT=$KAFKA_DIR/bin/zookeeper-server-start.sh +ZOOKEEPER_SERVER_STOP_SCRPIT=$KAFKA_DIR/bin/zookeeper-server-stop.sh +ZOOKEEPER_CONFIG=$KAFKA_DIR/config/zookeeper.properties +KAFKA_SERVER_START_SCRPIT=$KAFKA_DIR/bin/kafka-server-start.sh +KAFKA_SERVER_STOP_SCRPIT=$KAFKA_DIR/bin/kafka-server-stop.sh +KAFKA_SERVER_CONFIG=$KAFKA_DIR/config/server.properties +KAFKA_TOPIC_SCRIPT=$KAFKA_DIR/bin/kafka-topics.sh + +cd "$(dirname "$0")" + +if [[ $# -gt 0 ]]; then + case $1 in + --start) + echo "start zookeeper" + $ZOOKEEPER_SERVER_START_SCRPIT $ZOOKEEPER_CONFIG >/dev/null 2>&1 & + zookeeper_pid=$! + + echo "start kafka" + $KAFKA_SERVER_START_SCRPIT $KAFKA_SERVER_CONFIG >/dev/null 2>&1 & + kafka_pid=$! + + echo "zookeeper_pid: $zookeeper_pid + kafka_pid: $kafka_pid" >./process_info + + sleep 5s + + cat ./process_info + exit 0 + ;; + --stop) + echo "stop kafka" + $KAFKA_SERVER_STOP_SCRPIT + + sleep 5s + + echo "stop zookeeper" + $ZOOKEEPER_SERVER_STOP_SCRPIT + + cat ./process_info + exit 0 + ;; + --clean) + echo "remove /tmp/kafka-logs" + rm -rf /tmp/kafka-logs/ + exit 0 + ;; + --delete-topic) + echo "delete topic: kafka-benchmark" + $KAFKA_TOPIC_SCRIPT \ + --bootstrap-server=localhost:9092 \ + --delete \ + --topic kafka-benchmark + exit 0 + ;; + --create-topic) + echo "create topic for benchmark, topic: kafka-benchmark" + $KAFKA_TOPIC_SCRIPT \ + --bootstrap-server 127.0.0.1:9092 \ + --create \ + --topic kafka-benchmark \ + --partitions 6 \ + --replication-factor 1 \ + --config retention.ms=3600000 + exit 0 + ;; + --list-topic) + echo "existing topics:" + $KAFKA_TOPIC_SCRIPT \ + --bootstrap-server=localhost:9092 \ + --list + exit 0 + ;; + --restart) + if [ ! -e "./process_info" ]; then + echo "kafka is not running!" + exit 1 + fi + + echo "restart kafka service" + bash ./manager.sh --delete-topic + bash ./manager.sh --create-topic + bash ./manager.sh --stop + bash ./manager.sh --start + sleep 5s + exit 0 + ;; + --help) + echo "${USAGE}" + exit 0 + ;; + --*) + echo "Unknown option $1" + echo "${USAGE}" + exit 1 + ;; + esac +fi diff --git a/examples/tuning/kafka/prepare.sh b/examples/tuning/kafka/prepare.sh new file mode 100644 index 00000000..d67818e4 --- /dev/null +++ b/examples/tuning/kafka/prepare.sh @@ -0,0 +1,76 @@ +#!/usr/bin/bash + +# Copyright (c) 2022 Huawei Technologies Co., Ltd. +# A-Tune is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. + +############################################# +# @Author : shangyingjie +# @Contact : yingjie@isrc.iscas.ac.cn +# @Date : 2022/5/21 +# @License : Mulan PSL v2 +# @Desc : Kafka deployment script +# ############################################# + +cd "$(dirname "$0")" + +while [[ -z "$kafka_server_ip" ]]; do + read -p "please input kafka-server(localhost) IP:" kafka_server_ip +done + +while [[ -z "$kafka_client_ip" ]]; do + read -p "please input kafka-client IP:" kafka_client_ip +done +read -p "enter y to set up new ssh keys for the kafka-client or skip if already deployed:" setup_new_ssh_key + +if [ "$setup_new_ssh_key" = "y" ]; then + ssh-keygen -t rsa -f client_ssh_key -N "" + ssh-copy-id -i client_ssh_key root@"$kafka_client_ip" +fi + +echo "update launcher_on_server.sh and benchmark_on_client.sh with value just entered..." +sed -i "s#KAFKA_SERVER_IP=.*#KAFKA_SERVER_IP=$kafka_server_ip#g" benchmark_on_client.sh +sed -i "s#KAFKA_CLIENT_IP=.*#KAFKA_CLIENT_IP=$kafka_client_ip#g" launcher_on_server.sh + +echo "copy benchmark script to kafka-client" "$kafka_client_ip" +if [ "$setup_new_ssh_key" = "y" ]; then + scp -i client_ssh_key benchmark_on_client.sh root@"$kafka_client_ip":/root/ +else + scp benchmark_on_client.sh root@"$kafka_client_ip":/root/ +fi + +if [ -f "./kafka_2.13-3.2.0.tgz" ]; then + echo "Kafka release exist on local." +else + echo "download Kafka release on kafka-server." + wget https://dlcdn.apache.org/kafka/3.2.0/kafka_2.13-3.2.0.tgz +fi + +echo "copy Kafka release to kafka-client." +scp ./kafka_2.13-3.2.0.tgz root@"$kafka_client_ip":/root/ + +echo "install tar and jdk runtime on kafka-server" +yum install -y tar java +echo "install tar and jdk runtime on kafka-client" +ssh -i client_ssh_key -t root@"$kafka_client_ip" "yum install -y tar java;" + +echo "extract Kafka release tarball on kafka-server." +tar -xzf ./kafka_2.13-3.2.0.tgz +echo "extract Kafka release tarball on kafka-client." +ssh -i client_ssh_key -t root@"$kafka_client_ip" "tar -xzf /root/kafka_2.13-3.2.0.tgz" + +echo "modify kafka server configuration" +echo "listeners=PLAINTEXT://0.0.0.0:9092 \ +advertised.listeners=PLAINTEXT://$KAFKA_SERVER_IP:9092" >>./kafka_2.13-3.2.0/config/server.properties + +echo "start kafka service" +bash ./manager.sh --start + +echo "create topic for benchmark" +bash ./manager.sh --create-topic -- Gitee