要在多台服务器上搭建hadoop、hbase各种配置很繁琐，自动地安装既省时又省力。
参考了这篇博客：[url=http://www.cnblogs.com/tlm1992/p/tlm_hadoop_auto.html]http://www.cnblogs.com/tlm1992/p/tlm_hadoop_auto.html[/url]，并对其进行了一些修改。
前提：安装了ssh
[b]处理流程：[/b]
1.修改配置文件
（1）修改hadoop的配置文件，修改hadoop/hadoop-1.2.1/conf/下的
hdfs-site.xml，mapred-site.xml，core-site.xml 这三个文件,不需要修改hadoop/hadoop-1.2.1/conf/里的slaves和masters文件，脚本会根据hosts的设置自动修改。
（2）修改hbase的配置文件，修改hadoop/hbase-0.94.16/conf下的hbase-site.xml，不需要修改hadoop/hbase-0.94.16/conf/regionservers文件，脚本会根据hosts的设置自动修改。
2.修改hadoop/setup文件夹里的hosts文件来设置所有节点的名字和ip
3.编辑setHadoopOnce.sh的loginName变量（设置登录账号），pw变量(设置节点的密码)。账号、密码各节点必须都是同一个。此外还要修改slaveNum变量。
4.利用终端cd到setup这个文件夹，执行setHadoopOnce.sh脚本。
[b]全部代码：[/b]
1.setHadoopOnce.sh，该文件是文件脚本执行的起点
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
| 1#!/bin/bash
2#修改密码
3PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:.
4export PATH
5pw=123456
6loginName=hadoop
7master=master
8slave=slave
9slaveNum=1
10set timeout 100000
11> ../hadoop-1.2.1/conf/masters
12> ../hadoop-1.2.1/conf/slaves
13#update local file
14while read line
15do
16 echo $line
17 ip=`echo $line | cut -d" " -f1`
18 name=`echo $line | cut -d" " -f2`
19 if [ ! -z $ip ]; then
20 echo $name
21 if [[ $name == maste* ]]; then
22 echo "$name" >> ../hadoop-1.2.1/conf/masters
23 elif [[ $name == slave* ]]; then
24 echo "$name" >> ../hadoop-1.2.1/conf/slaves
25 echo "$name" >> ../hbase-0.94.16/conf/regionservers
26 fi
27 fi
28done < hosts
29#upload file to all nodes
30while read line
31do
32 ip=`echo $line | cut -d" " -f1`
33 name=`echo $line | cut -d" " -f2`
34 if [ ! -z $ip ]; then
35 expect copyDataToAll.exp $ip $loginName $pw
36 expect setForAll.exp $ip $loginName $pw
37 fi
38done < hosts
39while read line
40do
41 ip=`echo $line | cut -d" " -f1`
42 name=`echo $line | cut -d" " -f2`
43 if [ ! -z $ip ]; then
44 if [[ $name == maste* ]]; then
45 expect setForMaster.exp $ip $loginName $pw
46 fi
47 fi
48done < hosts
49while read line
50do
51 ip=`echo $line | cut -d" " -f1`
52 name=`echo $line | cut -d" " -f2`
53 if [ ! -z $ip ]; then
54 expect setForSSH.exp $ip $loginName $pw
55 fi
56done < hosts
57
58 |
1
2
| 1 2.copyDataToAll.exp在setHadoopOnce.sh中的36行被调用，以复制文件到所有节点。
2 |
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
| 1#!/usr/bin/expect
2proc usage {} {
3 puts stderr "usage: $::argv0 ip usrname password"
4 exit 1
5}
6if {$argc != 3} { usage }
7set hostip [lindex $argv 0]
8set username [lindex $argv 1]
9set password [lindex $argv 2]
10set timeout 100000
11spawn scp -r ../../hadoop ${username}@${hostip}:~
12expect {
13 "*assword:" {
14 send "$password\n"
15 expect eof
16 }
17 expect eof
18}
19
20 |
1
2
| 1 3.setForAll.exp为所有节点进行进一步的配置工作，在setHadoopOnce.sh中的37行被调用。
2 |
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
| 1#!/usr/bin/expect
2proc usage {} {
3 puts stderr "usage: $::argv0 ip usrname password"
4 exit 1
5}
6proc connect {pwd} {
7 expect {
8 "*(yes/no)?" {
9 send "yes\n"
10 expect "*assword:" {
11 send "$pwd\n"
12 expect {
13 "*Last login:*" {
14 return 0
15 }
16 }
17 }
18 }
19 "*assword:" {
20 send "$pwd\n"
21 expect {
22 "*Last login:*" {
23 return 0
24 }
25 }
26 }
27 "*Last login:*" {
28 return 0
29 }
30 }
31 return 1
32}
33if {$argc != 3} { usage }
34set hostip [lindex $argv 0]
35set username [lindex $argv 1]
36set password [lindex $argv 2]
37set timeout 100000
38spawn ssh ${username}@${hostip}
39if {[connect $password]} {
40 exit 1
41}
42#set host
43send "sudo bash ~/hadoop/setup/addHosts.sh\r"
44expect "*assword*"
45send "$password\r"
46expect "*ddhostsucces*"
47sleep 1
48send "ssh-agent bash ~/hadoop/setup/sshGen.sh\n"
49expect {
50 "*(yes/no)?" {
51 send "yes\n"
52 exp_continue
53 }
54 "*verwrite (y/n)?" {
55 send "n\n"
56 exp_continue
57 }
58 "*nter file in which to save the key*" {
59 send "\n"
60 exp_continue
61 }
62 "*nter passphrase*" {
63 send "\n"
64 exp_continue
65 }
66 "*nter same passphrase again*" {
67 send "\n"
68 exp_continue
69 }
70 "*our public key has been saved*" {
71 exp_continue
72 }
73 "*etsshGenSucces*" {
74 sleep 1
75 }
76}
77send "bash ~/hadoop/setup/setEnvironment.sh\n"
78expect "*etEnvironmentSucces*"
79sleep 1
80send "exit\n"
81expect eof
82
83 |
1
2
| 1 3.1 addHosts.sh在setForAll.exp中被调用，用于设置节点的hosts文件。
2 |
1
2
3
4
5
6
| 1#!/bin/bash
2hadoopRoot=~/hadoop
3cat $hadoopRoot/setup/hosts >> /etc/hosts
4echo "addhostsuccess"
5
6 |
1
2
| 13.2 sshGen.sh在setForAll.sh中被调用，用于生成sshkey.
2 |
1
2
3
4
5
6
7
8
9
10
11
| 1#!/bin/bash
2sshPath=~/.ssh
3setupPath=~/hadoop/setup
4rm "$sshPath"/authorized_keys
5sleep 1
6ssh-keygen -t rsa
7cat "$sshPath"/id_rsa.pub >> "$sshPath"/authorized_keys
8ssh-add
9echo "setsshGenSuccess"
10
11 |
1
2
| 13.3 setEnviroment.sh 在setForAll.sh中被调用，用于设置环境变量
2 |
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
| 1#!/bin/bash
2hadoopRoot=~/hadoop
3hadoopPath=$hadoopRoot/hadoop-1.2.1
4hbasePath=$hadoopRoot/hbase-0.94.16
5setupPath=$hadoopRoot/setup
6JAVA_VERSION=`java -version 2>&1 | awk '/java version/ {print $3}'|sed 's/"//g'|awk '{if ($1>=1.6) print "ok"}'`
7if [ "$JAVA_VERSION"x != "okx" ]; then
8 cat "$setupPath"/jdkenv >> ~/.bashrc
9 sleep 1
10 source ~/.bashrc
11 sleep 1
12fi
13 echo "export JAVA_HOME=~/hadoop/jdk1.7.0" >> "$hadoopPath"/conf/hadoop-env.sh
14 echo "export JAVA_HOME=~/hadoop/jdk1.7.0" >> "$hbasePath"/conf/hbase-env.sh
15Hadoop_Version=`hadoop version|awk '/Hadoop/ {print $2}'|awk '{if ($1>=1.0) print "ok"}'`
16if [ "$Hadoop_Version"x != "okx" ]; then
17 cat "$setupPath"/hadoopenv >> ~/.bashrc
18 sleep 1
19 source ~/.bashrc
20 sleep 1
21fi
22echo "setEnvironmentSuccess"
23
24 |
1
2
| 14 setForMaster.exp 远程sshsetForMaster.sh，配置无密码登录
2 |
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
| 1#!/usr/bin/expect
2proc usage {} {
3 puts stderr "usage: $::argv0 ip usrname password"
4 exit 1
5}
6proc connect {pwd} {
7 expect {
8 "*(yes/no)?" {
9 send "yes\n"
10 expect "*assword:" {
11 send "$pwd\n"
12 expect {
13 "*Last login:*" {
14 return 0
15 }
16 }
17 }
18 }
19 "*assword:" {
20 send "$pwd\n"
21 expect {
22 "*Last login:*" {
23 return 0
24 }
25 }
26 }
27 "*Last login:*" {
28 return 0
29 }
30 }
31 return 1
32}
33if {$argc != 3} { usage }
34set hostip [lindex $argv 0]
35set username [lindex $argv 1]
36set password [lindex $argv 2]
37set timeout 100000
38spawn ssh ${username}@${hostip}
39if {[connect $password]} {
40 exit 1
41}
42send "bash ~/hadoop/setup/setForMaster.sh\n"
43expect {
44 "*etForMasterSucces*" {
45 sleep 1
46 }
47 "*assword*" {
48 send "$password\n"
49 exp_continue
50 }
51 "*(yes/no)?" {
52 send "yes\n"
53 exp_continue
54 }
55}
56
57 |
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
| 1#!/bin/bash
2while read line
3do
4 ip=`echo $line | cut -d" " -f1`
5 name=`echo $line | cut -d" " -f2`
6 if [ ! -z $ip ]; then
7 if [[ $name == slave* ]]; then
8 scp $ip:~/.ssh/authorized_keys ~/tmpkey
9 cat ~/tmpkey >> ~/.ssh/authorized_keys
10 fi
11 fi
12done < ~/hadoop/setup/hosts
13sleep 1
14rm -f ~/tmpkey
15while read line
16do
17 ip=`echo $line | cut -d" " -f1`
18 name=`echo $line | cut -d" " -f2`
19 if [ ! -z $ip ]; then
20 if [[ $name == slave* ]]; then
21 scp ~/.ssh/authorized_keys $ip:~/.ssh/authorized_keys
22 fi
23 fi
24done < ~/hadoop/setup/hosts
25echo "setForMasterSuccess"
26
27 |
1
2
| 1 5.setForSSH.exp配置了SSH，可能由于.ssh 目录和~/.ssh/authorized_keys权限不对，还是需要输入密码。因此修改权限。
2 |
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
| 1#!/usr/bin/expect
2proc usage {} {
3 puts stderr "usage: $::argv0 ip usrname password"
4 exit 1
5}
6proc connect {pwd} {
7 expect {
8 "*(yes/no)?" {
9 send "yes\n"
10 expect "*assword:" {
11 send "$pwd\n"
12 expect {
13 "*Last login:*" {
14 return 0
15 }
16 }
17 }
18 }
19 "*assword:" {
20 send "$pwd\n"
21 expect {
22 "*Last login:*" {
23 return 0
24 }
25 }
26 }
27 "*Last login:*" {
28 return 0
29 }
30 }
31 return 1
32}
33if {$argc != 3} { usage }
34set hostip [lindex $argv 0]
35set username [lindex $argv 1]
36set password [lindex $argv 2]
37set timeout 100000
38spawn ssh ${username}@${hostip}
39if {[connect $password]} {
40 exit 1
41}
42sleep 1
43send "bash ~/hadoop/setup/setForSSH.sh\n"
44expect {
45 "*ForSSHSuccess*" {
46 sleep 1
47}
48}
49send "exit\n"
50expect eof
51
52 |
1
2
3
4
5
6
7
| 1#!/bin/bash
2chmod 700 ~/.ssh
3chmod 644 ~/.ssh/authorized_keys
4sleep 1
5echo "setForSSHSuccess"
6
7 |
本想上传源代码，发现文件不能超过10M，无法上传。
附件中是文件的目录，上面的linux脚本在setup目录下，其他的hbase,hadoop,jdk安装文件请自己下载吧。
