释放双眼，带上耳机，听听看~！
（
3）引擎服务器（以甘肃引擎为例,脚本解释参照甘肃web）
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
2
3
4
5 #!/bin/bash
6
7 #ip
8
9 yq_ip=$(/sbin/ifconfig eth0|grep "inet addr"|cut -d : -f 2|awk '{print $1}')
10
11 #name
12
13 yq_name=$(hostname)
14
15 monitor_name=yinqing
16
17 here='gansu'
18
19 #mysql info
20
21 mysql_ip='1.1.1.1'
22
23 mysql_username='root'
24
25 mysql_passwd='****'
26
27 mysql_database='monitor'
28
29 memory_table=''$here'_memory'
30
31 load_table=''$here'_load'
32
33 io_table=''$here'_io'
34
35 hardware_table=''$here'_hardware'
36
37 message_table=''$here'_message'
38
39 user_table=''$here'_user'
40
41 disk_table=''$here'_disk'
42
43 cpu_table=''$here'_cpu'
44
45 service_table=''$here'_service'
46
47 #date and log
48
49 day="$(date +%Y%m%d)"
50
51 worklog='/usr/local/monitor/logs/all_work_log'
52
53 downlog='/usr/local/monitor/logs/all_down_log'
54
55 now="$(date +%Y-%m-%d-%T)"
56
57 loghere='/usr/local/monitor/logs'
58
59 #yinqing service
60
61 memcache="$(ps -ef|grep 11211|grep -v grep|wc -l)"
62
63 datastorageservice="$(netstat -ant|grep 52010|wc -l)"
64
65 dbstatserver="$(ps -ef|grep dbstatserver|grep -v grep|wc -l)"
66
67 yq_http="$(netstat -antl|grep 80|wc -l)"
68
69 #cpu service
70
71 alert_cpu='80'
72
73 yq_cpu_idle="$(top -b -n 1 | grep Cpu | awk '{print $5}' | cut -f 1 -d .)"
74
75 yq_cpu="$(echo 100 - $yq_cpu_idle|/usr/bin/bc)"
76
77 #memory service
78
79 alert_mem='100'
80
81 yq_mem="$(/usr/bin/free -m|grep Mem|awk '{print $4}')"
82
83 yq_swap="$(/usr/bin/free -m|grep Swap|awk '{print $3}')"
84
85 alert_swap='0'
86
87 #memory log
88
89 memory_worklog='/usr/local/monitor/logs/mem_work_log'
90
91 memory_downlog='/usr/local/monitor/logs/mem_down_log'
92
93 #load service
94
95 cpu_count="$(grep -c 'model name' /proc/cpuinfo)"
96
97 alert_load="$(echo $cpu_count/2|/usr/bin/bc)"
98
99 yq_load="$(uptime|awk '{print $NF}'|cut -f 1 -d .)"
100
101 yq_load_15="$(uptime|awk '{print $NF}')"
102
103 #load log
104
105 load_worklog='/usr/local/monitor/logs/load_work_log'
106
107 load_downlog='/usr/local/monitor/logs/load_down_log'
108
109 #io service
110
111 alert_io='80'
112
113 yq_io_idle_back="$(/usr/bin/iostat|awk 'NR==4{print $NF}'|cut -f 1 -d .)"
114
115 yq_io_idle="$(echo 100 - $yq_io_idle_back|/usr/bin/bc)"
116
117 #io log
118
119 io_worklog='/usr/local/monitor/logs/io_work_log'
120
121 io_downlog='/usr/local/monitor/logs/io_down_log'
122
123 #hardware service
124
125 yq_hardware_error="$(dmesg|grep -i error|wc -l)"
126
127 yq_info_error="$(dmesg|grep -i error)"
128
129 #hardware error log
130
131 hard_worklog='/usr/local/monitor/logs/hard_work_log'
132
133 hard_downlog='/usr/local/monitor/logs/hard_down_log'
134
135 #message service
136
137 yq_message_error_count="$(awk '/"$(date +%Y%m%d)"/' /var/log/messages|grep -i error|wc -l)"
138
139 yq_message_error="$(awk '/"$(date +%Y%m%d)"/' /var/log/messages|grep -i error)"
140
141 #message error log
142
143 message_worklog='/usr/local/monitor/logs/message_work_log'
144
145 message_downlog='/usr/local/monitor/logs/message_down_log'
146
147 #user service
148
149 yq_user_count="$(/usr/bin/who|wc -l)"
150
151 yq_user_info="$(/usr/bin/who)"
152
153 #user log
154
155 user_worklog='/usr/local/monitor/logs/user_work_log'
156
157 user_downlog='/usr/local/monitor/logs/user_down_log'
158
159 #disk service
160
161 alert_disk='80'
162
163 yq_disk="$(/bin/df -H|sort -k5nr|grep -vE 'Filesystem|tmpfs|cdrom'|awk '{print $5 " " $1}')"
164
165 yq_disk_use=$(echo $yq_disk|awk '{print $1}'|cut -d '%' -f1)
166
167 yq_disk_partition=$(echo $yq_disk|awk '{print $2}')
168
169 #disk log
170
171 disk_worklog='/usr/local/monitor/logs/disk_work_log'
172
173 disk_downlog='/usr/local/monitor/logs/disk_down_log'
174
175 #cpu log
176
177 cpu_worklog='/usr/local/monitor/logs/cpu_work_log'
178
179 cpu_downlog='/usr/local/monitor/logs/cpu_down_log'
180
181 #notification mail
182
183 email='denglei@ctfo.com'
184
185 #monitor
186
187 if [ ! -d "$loghere" ];then
188
189 mkdir $loghere
190
191 fi
192
193 #yinqing check
194
195 if [ $memcache -ge 1 ];then
196
197 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: memcache Monitor_Server: $monitor_name is working" >> $worklog-$day
198
199 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','memcache','working',now())";
200
201 else
202
203 /opt/memcached-1.4.5/memcached -d -m 1024 -p 11211 -u root
204
205 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: memcache Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: memcache was a problem" $email
206
207 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: memcache Monitor_Server: $montior_name is down" >> $downlog-$day
208
209 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','memcache','downing',now())";
210
211 fi
212
213 if [ $datastorageservice -ge 1 ];then
214
215 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: datastorageservice Monitor_Server: $monitor_name is working" >> $worklog-$day
216
217 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','datastorageservice','working',now())";
218
219 else
220
221 /usr/local/lbs/bin/DataStorageService_0.sh restart >>/dev/null
222
223 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: datastorageservice Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: datastorageservice was a problem" $email
224
225 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: datastorageservice Monitor_Server: $montior_name is down" >> $downlog-$day
226
227 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','datastorageservice','downing',now())";
228
229 fi
230
231 if [ $dbstatserver -ge 1 ];then
232
233 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: dbstatserver Monitor_Server: $monitor_name is working" >> $worklog-$day
234
235 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','dbstatserver','working',now())";
236
237 else
238
239 /bin/sh /usr/local/lbs/java/lib/dbstatserver/dbstatserver.sh
240
241 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: dbstatserver Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: dbstatserver was a problem" $email
242
243 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: dbstatserver Monitor_Server: $montior_name is down" >> $downlog-$day
244
245 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','dbstatserver','downing',now())";
246
247 fi
248
249 if [ $yq_http -ge 1 ];then
250
251 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: http Monitor_Server: $monitor_name is working" >> $worklog-$day
252
253 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','yq_http','working',now())";
254
255 else
256
257 service httpd restart
258
259 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: http Monitor_Server: $monitor_name is down"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: http was a problem" $email
260
261 echo "$now ShengFen: $here Server: $yq_name Service: http Monitor_Server: $montior_name is down" >> $downlog-$day
262
263 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $service_table values ('','$here','$yq_name','$yq_ip','$monitor_name','yq_http','downing',now())";
264
265 fi
266
267 #check cpu_idle
268
269 if [ $yq_cpu -ge $alert_cpu ];then
270
271 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $yq_cpu"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: cpu_use was Exceed Threshold value: 80%" $email
272
273 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $yq_cpu" >> $cpu_downlog-$day
274
275 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $cpu_table values ('','$here','$yq_name','$yq_ip','$monitor_name','cpu_use','$alert_cpu','$yq_cpu','abnormal',now())";
276
277 else
278
279 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $yq_cpu" >> $cpu_worklog-$day
280
281 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $cpu_table values ('','$here','$yq_name','$yq_ip','$monitor_name','cpu_use','$alert_cpu','$yq_cpu','normal',now())";
282
283 fi
284
285 #check memory
286
287 if [ $yq_mem -le $alert_mem ];then
288
289 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: memory Monitor_Server: $monitor_name Free_mem: $yq_mem"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: memory was Exceed Threshold value" $email
290
291 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: memory Monitor_Server: $monitor_name Free_mem: $yq_mem" >> $memory_downlog-$day
292
293 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$yq_name','$yq_ip','$monitor_name','memory','$alert_mem','$yq_mem','abnormal',now())";
294
295 else
296
297 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: memory Monitor_Server: $monitor_name Free_mem: $yq_mem" >> $memory_worklog-$day
298
299 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$yq_name','$yq_ip','$monitor_name','memory','$alert_mem','$yq_mem','normal',now())";
300
301 fi
302
303 #check swap
304
305 if [ $yq_swap -gt $alert_swap ];then
306
307 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: swap Monitor_Server: $monitor_name Swap_yq: $yq_swap"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: swap Ip: $yq_ip was Exceed Threshold value" $email
308
309 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: swap Monitor_Server: $monitor_name Swap_yq: $yq_swap" >> $memory_downlog-$day
310
311 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$yq_name','$yq_ip','$monitor_name','swap','$alert_swap','$yq_swap','abnormal',now())";
312
313 else
314
315 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: swap Monitor_Server: $monitor_name Swap_yq: $yq_swap" >> $memory_worklog-$day
316
317 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$yq_name','$yq_ip','$monitor_name','swap','$alert_swap','$yq_swap','normal',now())";
318
319 fi
320
321 #check load_15
322
323 if [ $yq_load -ge $alert_load ];then
324
325 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: load_15 Monitor_Server: $monitor_name Load_use: $yq_load_15"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: load_15 Ip: $yq_ip was Exceed Threshold value" $email
326
327 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: load_15 Monitor_Server: $monitor_name Load_use: $yq_load_15" >> $load_downlog-$day
328
329 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $load_table values ('','$here','$yq_name','$yq_ip','$monitor_name','load_15','$alert_load','$yq_load_15','abnormal',now())";
330
331 else
332
333 echo "$now ShengFen: $here Server: $server Ip: $ip Service: load Monitor_Server: $monitor_name Load_use: $yq_load_15" >> $load_worklog-$day
334
335 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $load_table values ('','$here','$yq_name','$yq_ip','$monitor_name','load_15','$alert_load','$yq_load_15','normal',now())";
336
337 fi
338
339 #check io_idle
340
341 if [ $yq_io_idle -ge $alert_io ];then
342
343 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $yq_io_idle"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: io_use was Exceed Threshold value: 80%" $email
344
345 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $yq_io_idle" >> $io_downlog-$day
346
347 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $io_table values ('','$here','$yq_name','$yq_ip','$monitor_name','io_use','$alert_io','$yq_io_idle','abnormal',now())";
348
349 else
350
351 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $yq_io_idle" >> $io_worklog-$day
352
353 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $io_table values ('','$here','$yq_name','$yq_ip','$monitor_name','io_use','$alert_io','$yq_io_idle','normal',now())";
354
355 fi
356
357 #check hareware error info
358
359 if [ $yq_hardware_error -gt 0 ];then
360
361 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: hardware_error Monitor_Server: $monitor_name Error: $yq_info_error"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: hardware_error were some hardware imformation error" $email
362
363 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: hardware_error Monitor_Server: $monitor_name Error: $yq_info_error" >> $hard_downlog-$day
364
365 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $hardware_table values ('','$here','$yq_name','$yq_ip','$monitor_name','hardware_error','0','$yq_hardware_error','abnormal',now())";
366
367 else
368
369 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: hardware_error Monitor_Server: $monitor_name Error: Nothing" >> $hard_worklog-$day
370
371 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $hardware_table values ('','$here','$yq_name','$yq_ip','$monitor_name','hardware_error','0','$yq_hardware_error','normal',now())";
372
373 fi
374
375 #check message error
376
377 if [ $yq_message_error_count -ge 1 ];then
378
379 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: message_error Monitor_Server: $monitor_name Message_error: $yq_message_error"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: message_error were some message imformation error" $email
380
381 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: message_error Monitor_Server: $monitor_name Message_error: $yq_message_error" >> $message_downlog-$day
382
383 /usr/bin/mysql-h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $message_table values ('','$here','$yq_name','$yq_ip','$monitor_name','message_error','1','$yq_message_error_count','abnormal',now())";
384
385 else
386
387 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: message_error Monitor_Server: $monitor_name Message_error: Nothing" >> $message_worklog-$day
388
389 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $message_table values ('','$here','$yq_name','$yq_ip','$monitor_name','message_error','1','$yq_message_error_count','normal',now())";
390
391 fi
392
393 #check user
394
395 if [ $yq_user_count -ge 3 ];then
396
397 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: user Monitor_Server: $monitor_name User: $yq_user_info"|/bin/mail -s "Notification ShengFen: $here Server: $yq_name Service: user was Exceed Threshold value: 3" $email
398
399 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: user Monitor_Server: $monitor_name User: $yq_user_info" >> $user_downlog-$day
400
401 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $user_table values ('','$here','$yq_name','$yq_ip','$monitor_name','user','3','$yq_user_count','abnormal',now())";
402
403 else
404
405 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: user Monitor_Server: $monitor_name User: normal" >> $user_worklog-$day
406
407 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $user_table values ('','$here','$yq_name','$yq_ip','$monitor_name','user','3','$yq_user_count','normal',now())";
408
409 fi
410
411 #check disk
412
413 if [ $yq_disk_use -ge $alert_disk ];then
414
415 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $yq_disk_partition ($yq_disk_use%)"|/bin/mail -s "Warning!!! $here Server: $yq_name Service: disk_use was Exceed Threshold value : $alert_disk% " $email
416
417 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $yq_disk_partion ($yq_disk_use%)" >> $disk_downlog-$day
418
419 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $disk_table values ('','$here','$yq_name','$yq_ip','$monitor_name','disk_use','$alert_disk','$yq_disk_partition','$yq_disk_use','abnormal',now())";
420
421 else
422
423 echo "$now ShengFen: $here Server: $yq_name Ip: $yq_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $yq_disk_partition ($yq_disk_use%)" >> $disk_worklog-$day
424
425 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $disk_table values ('','$here','$yq_name','$yq_ip','$monitor_name','disk_use','$alert_disk','$yq_disk_partition','$yq_disk_use','normal',now())";
426
427 fi
428
429
430
（
4）备用服务器（以甘肃备用为例，脚本解释参照甘肃web)
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
2
3
4
5 #!/bin/bash
6
7 #ip
8
9 by_ip=$(/sbin/ifconfig eth0|grep "inet addr"|cut -d : -f 2|awk '{print $1}')
10
11 #name
12
13 by_name=$(hostname)
14
15 monitor_name=beiyong
16
17 here='gansu'
18
19 #mysql info
20
21 mysql_ip='1.1.1.1'
22
23 mysql_username='root'
24
25 mysql_passwd='****'
26
27 mysql_database='monitor'
28
29 memory_table=''$here'_memory'
30
31 load_table=''$here'_load'
32
33 io_table=''$here'_io'
34
35 hardware_table=''$here'_hardware'
36
37 message_table=''$here'_message'
38
39 user_table=''$here'_user'
40
41 disk_table=''$here'_disk'
42
43 cpu_table=''$here'_cpu'
44
45 service_table=''$here'_service'
46
47 #date and log
48
49 day="$(date +%Y%m%d)"
50
51 worklog='/usr/local/monitor/logs/all_work_log'
52
53 downlog='/usr/local/monitor/logs/all_down_log'
54
55 now="$(date +%Y-%m-%d-%T)"
56
57 loghere='/usr/local/monitor/logs'
58
59 #cpu service
60
61 alert_cpu='80'
62
63 by_cpu_idle="$(top -b -n 1 | grep Cpu | awk '{print $5}' | cut -f 1 -d .)"
64
65 by_cpu="$(echo 100 - $by_cpu_idle|/usr/bin/bc)"
66
67 #memory service
68
69 alert_mem='100'
70
71 by_mem="$(/usr/bin/free -m|grep Mem|awk '{print $4}')"
72
73 by_swap="$(/usr/bin/free -m|grep Swap|awk '{print $3}')"
74
75 alert_swap='0'
76
77 #memory log
78
79 memory_worklog='/usr/local/monitor/logs/mem_work_log'
80
81 memory_downlog='/usr/local/monitor/logs/mem_down_log'
82
83 #load service
84
85 cpu_count="$(grep -c 'model name' /proc/cpuinfo)"
86
87 alert_load="$(echo $cpu_count/2|/usr/bin/bc)"
88
89 by_load="$(uptime|awk '{print $NF}'|cut -f 1 -d .)"
90
91 by_load_15="$(uptime|awk '{print $NF}')"
92
93 #load log
94
95 load_worklog='/usr/local/monitor/logs/load_work_log'
96
97 load_downlog='/usr/local/monitor/logs/load_down_log'
98
99 #io service
100
101 alert_io='80'
102
103 by_io_idle_back="$(/usr/bin/iostat|awk 'NR==4{print $NF}'|cut -f 1 -d .)"
104
105 by_io_idle="$(echo 100 - $by_io_idle_back|/usr/bin/bc)"
106
107 #io log
108
109 io_worklog='/usr/local/monitor/logs/io_work_log'
110
111 io_downlog='/usr/local/monitor/logs/io_down_log'
112
113 #hardware service
114
115 by_hardware_error="$(dmesg|grep -i error|wc -l)"
116
117 by_info_error="$(dmesg|grep -i error)"
118
119 #hardware error log
120
121 hard_worklog='/usr/local/monitor/logs/hard_work_log'
122
123 hard_downlog='/usr/local/monitor/logs/hard_down_log'
124
125 #message service
126
127 by_message_error_count="$(awk '/"$(date +%Y%m%d)"/' /var/log/messages|grep -i error|wc -l)"
128
129 by_message_error="$(awk '/"$(date +%Y%m%d)"/' /var/log/messages|grep -i error)"
130
131 #message error log
132
133 message_worklog='/usr/local/monitor/logs/message_work_log'
134
135 message_downlog='/usr/local/monitor/logs/message_down_log'
136
137 #user service
138
139 by_user_count="$(/usr/bin/who|wc -l)"
140
141 by_user_info="$(/usr/bin/who)"
142
143 #user log
144
145 user_worklog='/usr/local/monitor/logs/user_work_log'
146
147 user_downlog='/usr/local/monitor/logs/user_down_log'
148
149 #disk service
150
151 alert_disk='80'
152
153 by_disk="$(/bin/df -H|sort -k5nr|grep -vE 'Filesystem|tmpfs|cdrom'|awk '{print $5 " " $1}')"
154
155 by_disk_use=$(echo $by_disk|awk '{print $1}'|cut -d '%' -f1)
156
157 by_disk_partition=$(echo $by_disk|awk '{print $2}')
158
159 #disk log
160
161 disk_worklog='/usr/local/monitor/logs/disk_work_log'
162
163 disk_downlog='/usr/local/monitor/logs/disk_down_log'
164
165 #cpu log
166
167 cpu_worklog='/usr/local/monitor/logs/cpu_work_log'
168
169 cpu_downlog='/usr/local/monitor/logs/cpu_down_log'
170
171 #notification mail
172
173 email='denglei@ctfo.com'
174
175 #monitor
176
177 if [ ! -d "$loghere" ];then
178
179 mkdir $loghere
180
181 fi
182
183 #check cpu_idle
184
185 #check cpu_idle
186
187 if [ $by_cpu -ge $alert_cpu ];then
188
189 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $by_cpu"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: cpu_use was Exceed Threshold value: 80%" $email
190
191 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $by_cpu" >> $cpu_downlog-$day
192
193 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $cpu_table values ('','$here','$by_name','$by_ip','$monitor_name','cpu_use','$alert_cpu','$by_cpu','abnormal',now())";
194
195 else
196
197 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: cpu_idle Monitor_Server: $monitor_name Cpu_use: $by_cpu" >> $cpu_worklog-$day
198
199 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $cpu_table values ('','$here','$by_name','$by_ip','$monitor_name','cpu_use','$alert_cpu','$by_cpu','normal',now())";
200
201 fi
202
203 #check memory
204
205 if [ $by_mem -le $alert_mem ];then
206
207 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: memory Monitor_Server: $monitor_name Free_mem: $by_mem"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: memory was Exceed Threshold value" $email
208
209 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: memory Monitor_Server: $monitor_name Free_mem: $by_mem" >> $memory_downlog-$day
210
211 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$by_name','$by_ip','$monitor_name','memory','$alert_mem','$by_mem','abnormal',now())";
212
213 else
214
215 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: memory Monitor_Server: $monitor_name Free_mem: $by_mem" >> $memory_worklog-$day
216
217 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$by_name','$by_ip','$monitor_name','memory','$alert_mem','$by_mem','normal',now())";
218
219 fi
220
221 #check swap
222
223 if [ $by_swap -gt $alert_swap ];then
224
225 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: swap Monitor_Server: $monitor_name Swap_by: $by_swap"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: swap Ip: $by_ip was Exceed Threshold value" $email
226
227 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: swap Monitor_Server: $monitor_name Swap_by: $by_swap" >> $memory_downlog-$day
228
229 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$by_name','$by_ip','$monitor_name','swap','$alert_swap','$by_swap','abnormal',now())";
230
231 else
232
233 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: swap Monitor_Server: $monitor_name Swap_by: $by_swap" >> $memory_worklog-$day
234
235 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $memory_table values ('','$here','$by_name','$by_ip','$monitor_name','swap','$alert_swap','$by_swap','normal',now())";
236
237 fi
238
239 #check load_15
240
241 if [ $by_load -ge $alert_load ];then
242
243 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: load_15 Monitor_Server: $monitor_name Load_use: $by_load_15"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: load_15 Ip: $by_ip was Exceed Threshold value" $email
244
245 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: load_15 Monitor_Server: $monitor_name Load_use: $by_load_15" >> $load_downlog-$day
246
247 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $load_table values ('','$here','$by_name','$by_ip','$monitor_name','load_15','$alert_load','$by_load_15','abnormal',now())";
248
249 else
250
251 echo "$now ShengFen: $here Server: $server Ip: $ip Service: load Monitor_Server: $monitor_name Load_use: $by_load_15" >> $load_worklog-$day
252
253 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $load_table values ('','$here','$by_name','$by_ip','$monitor_name','load_15','$alert_load','$by_load_15','normal',now())";
254
255 fi
256
257 #check io_idle
258
259 if [ $by_io_idle -ge $alert_io ];then
260
261 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $by_io_idle"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: io_use was Exceed Threshold value: 80%" $email
262
263 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $by_io_idle" >> $io_downlog-$day
264
265 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $io_table values ('','$here','$by_name','$by_ip','$monitor_name','io_use','$alert_io','$by_io_idle','abnormal',now())";
266
267 else
268
269 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: io_idle Monitor_Server: $monitor_name Io_use: $by_io_idle" >> $io_worklog-$day
270
271 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $io_table values ('','$here','$by_name','$by_ip','$monitor_name','io_use','$alert_io','$by_io_idle','normal',now())";
272
273 fi
274
275 #check hareware error info
276
277 if [ $by_hardware_error -gt 0 ];then
278
279 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: hardware_error Monitor_Server: $monitor_name Error: $by_info_error"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: hardware_error were some hardware imformation error" $email
280
281 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: hardware_error Monitor_Server: $monitor_name Error: $by_info_error" >> $hard_downlog-$day
282
283 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $hardware_table values ('','$here','$by_name','$by_ip','$monitor_name','hardware_error','0','$by_hardware_error','abnormal',now())";
284
285 else
286
287 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: hardware_error Monitor_Server: $monitor_name Error: Nothing" >> $hard_worklog-$day
288
289 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $hardware_table values ('','$here','$by_name','$by_ip','$monitor_name','hardware_error','0','$by_hardware_error','normal',now())";
290
291 fi
292
293 #check message error
294
295 if [ $by_message_error_count -ge 1 ];then
296
297 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: message_error Monitor_Server: $monitor_name Message_error: $by_message_error"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: message_error were some message imformation error" $email
298
299 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: message_error Monitor_Server: $monitor_name Message_error: $by_message_error" >> $message_downlog-$day
300
301 /usr/bin/mysql-h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $message_table values ('','$here','$by_name','$by_ip','$monitor_name','message_error','1','$by_message_error_count','abnormal',now())";
302
303 else
304
305 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: message_error Monitor_Server: $monitor_name Message_error: Nothing" >> $message_worklog-$day
306
307 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $message_table values ('','$here','$by_name','$by_ip','$monitor_name','message_error','1','$by_message_error_count','normal',now())";
308
309 fi
310
311 #check user
312
313 if [ $by_user_count -ge 3 ];then
314
315 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: user Monitor_Server: $monitor_name User: $by_user_info"|/bin/mail -s "Notification ShengFen: $here Server: $by_name Service: user was Exceed Threshold value: 3" $email
316
317 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: user Monitor_Server: $monitor_name User: $by_user_info" >> $user_downlog-$day
318
319 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $user_table values ('','$here','$by_name','$by_ip','$monitor_name','user','3','$by_user_count','abnormal',now())";
320
321 else
322
323 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: user Monitor_Server: $monitor_name User: normal" >> $user_worklog-$day
324
325 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $user_table values ('','$here','$by_name','$by_ip','$monitor_name','user','3','$by_user_count','normal',now())";
326
327 fi
328
329 #check disk
330
331 if [ $by_disk_use -ge $alert_disk ];then
332
333 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $by_disk_partition ($by_disk_use%)"|/bin/mail -s "Warning!!! $here Server: $by_name Service: disk_use was Exceed Threshold value : $alert_disk% " $email
334
335 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $by_disk_partion ($by_disk_use%)" >> $disk_downlog-$day
336
337 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $disk_table values ('','$here','$by_name','$by_ip','$monitor_name','disk_use','$alert_disk','$by_disk_partition','$by_disk_use','abnormal',now())";
338
339 else
340
341 echo "$now ShengFen: $here Server: $by_name Ip: $by_ip Service: disk_use Monitor_Server: $monitor_name Disk_use: $by_disk_partition ($by_disk_use%)" >> $disk_worklog-$day
342
343 /usr/bin/mysql -h $mysql_ip -u$mysql_username -p$mysql_passwd $mysql_database -e "insert into $disk_table values ('','$here','$by_name','$by_ip','$monitor_name','disk_use','$alert_disk','$by_disk_partition','$by_disk_use','normal',now())";
344
345 fi
346
347
348
下一篇文章地址：
运维自动化之使用PHP+MYSQL+SHELL打造私有监控系统（四）
