zookeeper+kafka集群一键安装脚本

1. zookeeper_kafka_install.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
1#!/bin/bash

2

3read -p "is need to deploy cluster environment[yes|no]: " booleanResult

4if [ $booleanResult = "yes" ] || [ $booleanResult = "no" ];then

5   echo "input is: "$booleanResult

6else

7   echo "only input [yes|no]"

8   exit

9fi

10## root path

11INSTALL_BESE_URL=/home/bigdata/app

12## deployment template path

13TGZ_PACKEGER_URL=/home/bigdata/app/base/dev

14## install path

15INSTALL_PACKEGER_URL=/home/bigdata/app/base/install

16## zookeeper tgz name

17ZOOKEEPER_TGZ=Auto.BigData.SystemBase.zookeeper.tgz

18## kafka tgz name

19KAFKA_TGZ=Auto.BigData.SystemBase.kafka.tgz

20## zookeeper name

21ZOOKEEPER_NAME=zookeeper-3.4.14

22## kafka name

23KAFKA_NAME=kafka_2.11-1.1.1

24## kafka config file path

25KAFKA_CONFIG_FILE_PATH=$TGZ_PACKEGER_URL/$KAFKA_NAME/config/server.properties

26## remove old file

27rm -rf /home/bigdata/app/base

28PID=$(ps aux | grep kafka | grep -v grep | awk '{print $2}' )

29echo "kafka runnig id is "$PID

30## get local ip 

31LOCAL_IP=$(ifconfig | grep  inet | grep netmask | grep broadcast | awk '{print $2}' | sed -n '1p')

32echo "local ip is "$LOCAL_IP    

33LOCAL_PATH=$(pwd)

34echo "local path pwd:"$LOCAL_PATH

35if [ ! -d $TGZ_PACKEGER_URL ];then

36  echo "$TGZ_PACKEGER_URL is not create"

37  mkdir -p $TGZ_PACKEGER_URL

38else

39  echo "$TGZ_PACKEGER_URL is exist"

40fi

41if [ -e $LOCAL_PATH/$ZOOKEEPER_TGZ ];then

42  tar -zxvf $LOCAL_PATH/$ZOOKEEPER_TGZ -C $TGZ_PACKEGER_URL

43else 

44  echo "zookeeper tgz is not exist,path=$INSTALL_BESE_URL/$ZOOKEEPER_TGZ"

45  exit

46fi

47

48if [ -e $LOCAL_PATH/$KAFKA_TGZ ];then

49  tar -zxvf $LOCAL_PATH/$KAFKA_TGZ -C $TGZ_PACKEGER_URL

50else 

51  echo "kafka tgz is not exist,path=$INSTALL_BESE_URL/$KAFKA_TGZ"

52  exit

53fi

54

55if [ ! -d $TGZ_PACKEGER_URL/$ZOOKEEPER_NAME/data ];then

56  mkdir $TGZ_PACKEGER_URL/$ZOOKEEPER_NAME/data

57fi

58START_ZOOKEEPER_FILE_NAME=zookeeper_start.sh

59START_KAFKA_FILE_NAME=kafka_start.sh

60KILL_ZK_KF_PID_FILE_NAME=kill_zk_kf_pid.sh

61cp $LOCAL_PATH/$START_ZOOKEEPER_FILE_NAME $TGZ_PACKEGER_URL

62cp $LOCAL_PATH/$START_KAFKA_FILE_NAME $TGZ_PACKEGER_URL

63cp $LOCAL_PATH/$KILL_ZK_KF_PID_FILE_NAME $TGZ_PACKEGER_URL

64chmod -R 777 $TGZ_PACKEGER_URL/*

65echo "end tar tgz!!!!!"

66

67

68## check user input ip

69input_ip_list=""

70function check_input_ip_info {

71  echo "========start vail ip is it repeated========"

72  for ip in $input_ip_list;do

73      echo "-------for ip="$ip",inputIpInfo="$inputIpInfo"-----------"

74      if [ $ip = $inputIpInfo ]; then

75          echo "input ip is already exist: "$input_ip_list

76          return 1

77      fi

78  done

79  echo "========end vail ip is it repeated========"

80  ping -c 1  -w 100 $inputIpInfo

81  if [ $? -eq 0 ];then

82      echo "input ip is right"

83      input_ip_list=$input_ip_list" $inputIpInfo"

84  else

85      echo "input ip is error,please reinput"     

86  fi

87}

88## loop set zookeeper property

89total_num=0

90function loop_set_zookeeper_server_config {

91  echo "start set zookeeeper config"

92  for ip_obj in $input_ip_list;

93  do

94      echo $ip_obj

95      let total_num++

96      server_ip="server.$total_num=$ip_obj:2888:3888"

97      echo $server_ip >> $TGZ_PACKEGER_URL/$ZOOKEEPER_NAME/conf/zoo.cfg

98      echo "zookeeper server ip $total_num=$server_ip"

99  done

100 echo "end set zookeeeper config"

101}

102

103

104## set kafka server.properties property

105function loop_set_kafka_config {

106 echo "start set kafka config"

107 echo 'num.partitions='$total_num >> $KAFKA_CONFIG_FILE_PATH

108 echo 'offsets.topic.replication.factor='$total_num >> $KAFKA_CONFIG_FILE_PATH

109 echo 'transaction.state.log.replication.factor='$total_num >> $KAFKA_CONFIG_FILE_PATH

110 echo 'transaction.state.log.min.isr='$total_num >> $KAFKA_CONFIG_FILE_PATH

111 echo 'default.replication.factor='$total_num >> $KAFKA_CONFIG_FILE_PATH

112

113 echo 'num.network.threads=3' >> $KAFKA_CONFIG_FILE_PATH

114 echo 'num.io.threads=8' >> $KAFKA_CONFIG_FILE_PATH

115 echo 'socket.send.buffer.bytes=102400' >> $KAFKA_CONFIG_FILE_PATH

116 echo 'socket.receive.buffer.bytes=102400' >> $KAFKA_CONFIG_FILE_PATH

117 echo 'socket.request.max.bytes=104857600' >> $KAFKA_CONFIG_FILE_PATH

118 echo 'log.dirs='$INSTALL_PACKEGER_URL'/'$KAFKA_NAME'/logs' >> $KAFKA_CONFIG_FILE_PATH

119 echo 'num.recovery.threads.per.data.dir=1' >> $KAFKA_CONFIG_FILE_PATH

120 echo 'log.retention.hours=48' >> $KAFKA_CONFIG_FILE_PATH

121 echo 'log.retention.bytes=214748364800' >> $KAFKA_CONFIG_FILE_PATH

122 echo 'log.segment.bytes=1073741824' >> $KAFKA_CONFIG_FILE_PATH

123 echo 'log.retention.check.interval.ms=300000' >> $KAFKA_CONFIG_FILE_PATH

124 echo 'zookeeper.connection.timeout.ms=6000' >> $KAFKA_CONFIG_FILE_PATH

125 echo 'group.initial.rebalance.delay.ms=0' >> $KAFKA_CONFIG_FILE_PATH

126 zookeeepr_ip=""

127 zookeeper_flag_num=0

128 for ip_info in $input_ip_list;

129 do

130     let zookeeper_flag_num++

131     if [ $zookeeper_flag_num -eq 1 ];then

132         zookeeepr_ip=$ip_info":2181"

133     else

134         zookeeepr_ip=$zookeeepr_ip",$ip_info:2181"

135     fi

136 done

137 echo 'zookeeper.connect='$zookeeepr_ip >> $KAFKA_CONFIG_FILE_PATH

138 echo "end set kafka config"

139}

140

141## local install funtion

142function generate_local_install {

143 if [ -d $INSTALL_PACKEGER_URL ];then

144     rm -rf $INSTALL_PACKEGER_URL/*

145 else

146     mkdir -p $INSTALL_PACKEGER_URL

147 fi

148 cp $TGZ_PACKEGER_URL/* $INSTALL_PACKEGER_URL/

149 $INSTALL_PACKEGER_URL/$KILL_ZK_KF_PID_FILE_NAME

150}

151## remote install function

152function generate_remote_install {

153 echo "#####################check install package path is exist ip:"$ip_adr"############################"

154 ssh bigdata@$ip_adr "[ -d "$INSTALL_PACKEGER_URL" ] && rm -rf "$INSTALL_PACKEGER_URL"/* || mkdir -p" $INSTALL_PACKEGER_URL

155 echo "#####################send remote install package ip:"$ip_adr"############################"

156 scp -r $TGZ_PACKEGER_URL/* bigdata@$ip_adr:$INSTALL_PACKEGER_URL

157 echo "#####################kill old kafka zookeeper process id ip:"$ip_adr"############################"

158 ssh bigdata@$ip_adr $INSTALL_PACKEGER_URL/$KILL_ZK_KF_PID_FILE_NAME

159}

160## start zookeeper function

161function start_zookeeper_fun {

162 for ip_adr_zookeeper in $input_ip_list;do

163     if [ $ip_adr_zookeeper = $LOCAL_IP ];then

164         echo "install kafka is local ip:"$ip_adr_zookeeper

165         $INSTALL_PACKEGER_URL/$START_ZOOKEEPER_FILE_NAME

166     else

167         echo "install zookeeper is remote ip:"$ip_adr_zookeeper

168         ssh bigdata@$ip_adr_zookeeper $INSTALL_PACKEGER_URL/$START_ZOOKEEPER_FILE_NAME

169     fi

170 done

171}

172## start kafka function

173function start_kafka_fun {

174 for ip_adr_kafka in $input_ip_list;do

175     if [ $ip_adr_kafka = $LOCAL_IP ];then

176         echo "install kafka is local ip:"$ip_adr_kafka

177         $INSTALL_PACKEGER_URL/$START_KAFKA_FILE_NAME

178     else

179         echo "install kafka is remote ip:"$ip_adr_kafka

180         ssh bigdata@$ip_adr_kafka $INSTALL_PACKEGER_URL/$START_KAFKA_FILE_NAME

181     fi

182 done

183}

184## loop send zookee+kafka install packages to remote servers

185send_flag_num=0

186function loop_send_kafka_zookeeper {

187 echo "====================start send remote zookeeper========================"

188 echo "ips="$input_ip_list

189 for ip_adr in $input_ip_list;

190 do

191     echo "delete kafka server.properties and zookeeper myid file"

192     rm -rf $KAFKA_CONFIG_FILE_PATH

193     rm -rf $TGZ_PACKEGER_URL/$ZOOKEEPER_NAME/data/myid

194     echo "recreate kafka server.properties and zookeeper myid file"

195     touch $KAFKA_CONFIG_FILE_PATH

196     touch $TGZ_PACKEGER_URL/$ZOOKEEPER_NAME/data/myid

197     echo "set kafka attr"

198     echo 'broker.id='$send_flag_num >> $KAFKA_CONFIG_FILE_PATH

199     echo 'port=9092' >> $KAFKA_CONFIG_FILE_PATH

200     echo 'host.name='$ip_adr >> $KAFKA_CONFIG_FILE_PATH

201     loop_set_kafka_config

202     

203     let send_flag_num++

204     echo $send_flag_num >> $TGZ_PACKEGER_URL/$ZOOKEEPER_NAME/data/myid

205     

206     if [ $ip_adr = $LOCAL_IP ];then

207         echo "start local install ip: $ip_adr"

208         generate_local_install

209         echo "end local install ip: $ip_adr"

210     else

211         echo "start remote install ip: $ip_adr"

212         generate_remote_install

213         echo "end remote install ip: $ip_adr"

214     fi

215     echo "send ip: $ip_adr"

216 done

217 echo "====================end send remote ips=$input_ip_list,start execute path /home/bigdata/app/base/install/zookeeper_kafka_start.sh========================"

218 echo "======================start install zookeeepr=========================="

219 start_zookeeper_fun

220 echo "======================end install zookeeepr=========================="

221 

222 echo "======================start install kafka=========================="

223 start_kafka_fun

224 echo "======================end install kafka=========================="

225}

226

227

228if [ $booleanResult = "yes" ];then

229## need to deploy cluster version

230 echo "start colony set"

231 while true

232 do

233        read -p "Please enter the cluster IP address,exit input[exit]: " inputIpInfo

234     if [ $inputIpInfo = "exit" ];then

235         echo "break ip input loop"    

236         break

237     else

238         check_input_ip_info

239     fi 

240 done

241 echo "user input finish"

242 if [ -z $input_ip_list ];then

243     echo "ips is empty"

244     exit

245 fi

246 loop_set_zookeeper_server_config

247 loop_send_kafka_zookeeper

248 echo "end colony ips is: "$input_ip_list

249else

250## simply deploy the stand-alone version

251 echo "start not colony set"

252 inputIpInfo=$LOCAL_IP

253 echo 'broker.id='$send_flag_num >> $KAFKA_CONFIG_FILE_PATH

254 echo 'port=9092' >> $KAFKA_CONFIG_FILE_PATH

255 echo 'host.name='$inputIpInfo >> $KAFKA_CONFIG_FILE_PATH

256 total_num=1

257 loop_set_kafka_config

258 if [ ! -d $INSTALL_PACKEGER_URL ];then

259     mkdir -p $INSTALL_PACKEGER_URL

260 fi

261 rm -rf $INSTALL_PACKEGER_URL/*

262 mv $TGZ_PACKEGER_URL/* $INSTALL_PACKEGER_URL/

263##  execute start cmd

264 $INSTALL_PACKEGER_URL/$KILL_ZK_KF_PID_FILE_NAME

265 $INSTALL_PACKEGER_URL/$START_ZOOKEEPER_FILE_NAME

266 $INSTALL_PACKEGER_URL/$START_KAFKA_FILE_NAME

267    echo "end not colony set"

268fi

269rm -rf $TGZ_PACKEGER_URL

270

271

272

2. kill_zk_kf_pid.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
1#!/bin/bash

2

3## kafka process id

4KAFKA_PID=$( ps aux | grep kafka_2.11-1.1.1 | grep -v grep | awk '{print $2}' )

5## zookeeper process id

6ZOOKEEPER_PID=$( ps aux | grep zookeeper-3.4.14 | grep -v grep | awk '{print $2}' )

7

8## kill old zookeeper process id

9for zk_pid in $ZOOKEEPER_PID;

10do

11  echo "kill process zookeeper pid:"$zk_pid

12  kill -9 $zk_pid

13done

14## kill old kafka process id

15for ka_pid in $KAFKA_PID;

16do

17  echo "kill process kafka pid:"$ka_pid

18  kill -9 $ka_pid

19done

20

21

22

3. zookeeper_start.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
1#!/bin/bash

2

3## install path

4INSTALL_PACKEGER_URL=/home/bigdata/app/base/install

5## zookeeper name

6ZOOKEEPER_NAME=zookeeper-3.4.14

7## kafka name

8KAFKA_NAME=kafka_2.11-1.1.1

9## kafka process id

10KAFKA_PID=$( ps aux | grep kafka_2.11-1.1.1 | grep -v grep | awk '{print $2}' )

11## zookeeper process id

12ZOOKEEPER_PID=$( ps aux | grep zookeeper-3.4.14 | grep -v grep | awk '{print $2}' )

13## zookeeper service path

14ZKFILE="/usr/lib/systemd/system/zookeeper.service"

15echo "delete zookeeper service file"

16sudo rm -rf $ZKFILE

17## determine whether or not JDK is installed

18which java > /dev/null 2>&1

19if [ $? -eq 0 ];then

20    echo "java was installed"

21else

22    echo "java isn't install,please install java first"

23    exit

24fi

25

26## zookeeper environment variable path

27ZKBIN_PATH="export PATH=${PATH}:/home/bigdata/app/base/install/"$ZOOKEEPER_NAME"/bin"

28## judge zookeeper environment variable is exist

29sudo grep -i "/home/bigdata/app/base/install/"$ZOOKEEPER_NAME"/bin" /etc/profile

30## set zookeeper environment variable

31if [ $? -eq 0 ];then

32    echo $ZKBIN_PATH" is exist,skip"

33else

34    sudo chmod 745 /etc/profile

35    sudo chmod 777 /etc/profile

36    sudo echo $ZKBIN_PATH >> /etc/profile

37  source /etc/profile

38fi

39echo "chmod kafka and zookeeper bin file power"

40chmod -R 777 /home/bigdata/app/base/install/kafka_2.11-1.1.1/bin/*

41chmod -R 777 /home/bigdata/app/base/install/kafka_2.11-1.1.1/config/*

42chmod -R 777 /home/bigdata/app/base/install/kafka_2.11-1.1.1/libs/*

43chmod -R 777 /home/bigdata/app/base/install/kafka_2.11-1.1.1/*

44chmod -R 777 /home/bigdata/app/base/install/zookeeper-3.4.14/bin/*

45chmod -R 777 /home/bigdata/app/base/install/zookeeper-3.4.14/data/*

46chmod -R 777 /home/bigdata/app/base/install/zookeeper-3.4.14/lib/*

47chmod -R 777 /home/bigdata/app/base/install/zookeeper-3.4.14/conf/*

48chmod -R 777 /home/bigdata/app/base/install/zookeeper-3.4.14/*

49

50## set up zookeeper boot start

51function set_zookeeper_system {

52  echo "start set zookeeper service config"

53  sudo sh -c 'echo "[Unit]" > '$ZKFILE

54  sudo sh -c 'echo "Description=zookeeper.service" >> '$ZKFILE

55  sudo sh -c 'echo "After=network.target" >> '$ZKFILE

56  sudo sh -c 'echo "[Service]" >> '$ZKFILE

57  sudo sh -c 'echo "User=bigdata" >> '$ZKFILE

58  sudo sh -c 'echo "Type=forking" >> '$ZKFILE

59  sudo sh -c 'echo "Environment=ZOO_LOG_DIR=/home/bigdata/app/base/install/zookeeper-3.4.14/log" >> '$ZKFILE

60  sudo sh -c 'echo "ExecStart=/home/bigdata/app/base/install/zookeeper-3.4.14/bin/zkServer.sh start" >> '$ZKFILE

61  sudo sh -c 'echo "ExecStop=/home/bigdata/app/base/install/zookeeper-3.4.14/bin/zkServer.sh stop" >> '$ZKFILE

62  sudo sh -c 'echo "ExecReload=/home/bigdata/app/base/install/zookeeper-3.4.14/bin/zkServer.sh restart" >> '$ZKFILE

63  sudo sh -c 'echo "[Install]" >> '$ZKFILE

64  sudo sh -c 'echo "WantedBy=multi-user.target" >> '$ZKFILE

65

66  sudo systemctl daemon-reload

67  echo "end set zookeeper service config"   

68  echo "==========start zookeeper process=========="

69  sudo systemctl start zookeeper

70  if [ $? -eq 0 ];then

71      echo "zookeeper was installed success"

72  else

73      echo "zookeeper was installed false,please check your install file"

74  fi

75}

76

77sudo touch $ZKFILE

78set_zookeeper_system

79

80

4. kafka_start.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
1#!/bin/bash

2

3KAFKAFILE="/usr/lib/systemd/system/kafka.service"

4

5echo "delete kafka service file"

6sudo rm -rf $KAFKAFILE

7

8## set up kafka boot start

9function set_kafka_system {

10  echo "start set kafka service config"

11  sudo sh -c 'echo "[Unit]"  > '$KAFKAFILE

12  sudo sh -c 'echo "Description=kafka.service"  >> '$KAFKAFILE

13  sudo sh -c 'echo "[Service]"  >> '$KAFKAFILE

14  sudo sh -c 'echo "Type=forking" >> '$KAFKAFILE

15  sudo sh -c 'echo "User=bigdata"  >> '$KAFKAFILE

16  sudo sh -c 'echo "Environment=JMX_PORT=9988" >> '$KAFKAFILE

17  sudo sh -c 'echo "ExecStart=/home/bigdata/app/base/install/kafka_2.11-1.1.1/bin/kafka-server-start.sh -daemon /home/bigdata/app/base/install/kafka_2.11-1.1.1/config/server.properties"  >> '$KAFKAFILE

18  sudo sh -c 'echo "[Install]"  >> '$KAFKAFILE

19  sudo sh -c 'echo "WantedBy=multi-user.target"  >> '$KAFKAFILE

20  sudo systemctl daemon-reload

21  echo "end set kafka service config"

22  echo "==========start kafka process=========="

23  sudo systemctl start kafka

24}

25sudo touch $KAFKAFILE

26set_kafka_system

27

28