释放双眼，带上耳机，听听看~！
本文所用到的localHTML.html等练习文件
详见此处下载：链接地址
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
2
3import java.io.File;
4import java.io.FileInputStream;
5import java.io.FileOutputStream;
6
7import org.apache.commons.io.FileUtils;
8import org.apache.commons.io.FilenameUtils;
9import org.apache.commons.io.IOUtils;
10import org.apache.commons.lang.StringUtils;
11import org.apache.http.HttpEntity;
12import org.apache.http.HttpResponse;
13import org.apache.http.client.HttpClient;
14import org.apache.http.client.methods.HttpGet;
15import org.apache.http.impl.client.DefaultHttpClient;
16import org.apache.http.util.EntityUtils;
17import org.htmlparser.Node;
18import org.htmlparser.NodeFilter;
19import org.htmlparser.Parser;
20import org.htmlparser.filters.NodeClassFilter;
21import org.htmlparser.nodes.TagNode;
22import org.htmlparser.nodes.TextNode;
23import org.htmlparser.tags.ImageTag;
24import org.htmlparser.util.NodeList;
25
26/**
27 * HTMLParser入门_01_网络爬虫的雏形_解析文章和处理文章中的图片
28 * @see ---------------------------------------------------------------------------------------------------------
29 * @see 所有jar如下
30 * @see commons-io-2.3.jar
31 * @see commons-land-2.3.jar
32 * @see commons-codec-1.6.jar(以下7个jar取自HttpClient官网下载的httpcomponents-client-4.2.1-bin.zip)
33 * @see commons-logging-1.1.1.jar
34 * @see fluent-hc-4.2.1.jar
35 * @see httpclient-4.2.1.jar
36 * @see httpclient-cache-4.2.1.jar
37 * @see httpcore-4.2.1.jar
38 * @see httpmime-4.2.1.jar
39 * @see filterbuilder.jar(以下5个jar取自HTMLParser官网下载的HTMLParser-2.0-SNAPSHOT-bin.zip)
40 * @see htmllexer.jar
41 * @see htmlparser.jar
42 * @see sitecapturer.jar
43 * @see thumbelina.jar
44 * @see ---------------------------------------------------------------------------------------------------------
45 * @see 网络爬虫的雏形
46 * @see 通过本文的三个方法,就可以初步实现一个网络爬虫
47 * @see 比如有个后台管理系统,可添加待爬虫的技术文章,输入参数为文章URL,输出参数为文章中的实际内容
48 * @see 并且它还会下载文章内容中的图片,并修改内容中的图片URL为本地的绝对路径,使得文章图片正常显示
49 * @see 接下来就是在前台显示文章了,至于怎么显示,随心所欲喽..
50 * @see 至于如何获取文章标题,作者,关键字,总体描述等信息,可参考我的下面这一篇文章
51 * @see http://blog.csdn.net/jadyer/article/details/8656477
52 * @see ---------------------------------------------------------------------------------------------------------
53 * @create Mar 10, 2013 4:05:55 PM
54 * @author 玄玉<http://blog.csdn/net/jadyer>
55 */
56public class SpiderDemo {
57 private static final String articleURI = "http://www.ibm.com/developerworks/cn/java/j-javaroundtable/index.html";
58 private static final String localHTML = "D:/Download/localHTML.html";
59
60 /**
61 * 下载文章
62 */
63 private static void downloadArticle() throws Exception {
64 HttpClient httpClient = new DefaultHttpClient();
65 HttpGet httpGet = new HttpGet(articleURI);
66 try {
67 HttpResponse response = httpClient.execute(httpGet);
68 HttpEntity entity = response.getEntity();
69 if(null != entity){
70 String responseContent = EntityUtils.toString(entity, "UTF-8");
71 EntityUtils.consume(entity);
72 //文章内容写到本地(IOUtils干完活儿会自动关闭IO流)
73 IOUtils.write(responseContent, new FileOutputStream(localHTML), "UTF-8");
74 }
75 }finally{
76 httpClient.getConnectionManager().shutdown();
77 }
78 }
79
80
81 /**
82 * 下载文章内容中的图片到本地
83 */
84 private static void downloadArticleImage() throws Exception {
85 //加载文章整理
86 String html = IOUtils.toString(new FileInputStream(localHTML), "UTF-8");
87 //获取文章内容,这里是通过分析文章<body>中的特征来获取内容的
88 String article = StringUtils.substringBetween(html, "<!-- MAIN_COLUMN_CONTENT_BEGIN -->", "<!-- CMA");
89 System.out.println("文章内容:" + article);
90 Parser parser = new Parser();
91 parser.setInputHTML(article);
92 //提取所有的<img>标签
93 //这里使用了内置的NodeClassFilter,它会根据节点类型来过滤,这里过滤的是ImageTag节点类型
94 NodeList imageTags = parser.parse(new NodeClassFilter(ImageTag.class));
95 for(int i=0; i<imageTags.size(); i++){
96 ImageTag it = (ImageTag)imageTags.elementAt(i);
97 //it.getImageURL()即<img>中的src属性值
98 System.out.println("图片链接:" + it.getImageURL());
99 //合成图片的绝对路径http://www.ibm.com/developerworks/cn/java/j-javaroundtable/ + it.getImageURL()
100 String absoluteURL = articleURI.substring(0, articleURI.lastIndexOf("/")+1) + it.getImageURL();
101 System.out.println("图片地址:" + absoluteURL);
102 //下载文章内容中的图片
103 HttpClient httpClient = new DefaultHttpClient();
104 HttpGet httpGet = new HttpGet(absoluteURL);
105 HttpResponse response = httpClient.execute(httpGet);
106 HttpEntity entity = response.getEntity();
107 if(null != entity){
108 byte[] images = EntityUtils.toByteArray(entity);
109 IOUtils.write(images, new FileOutputStream("D:/Download/" + FilenameUtils.getName(absoluteURL)));
110 System.out.println("图片[" + absoluteURL + "]下载完毕");
111 }
112 }
113 }
114
115
116 /**
117 * 修改文章内容中的图片链接为本地链接
118 */
119 @SuppressWarnings("serial")
120 private static void modifyImageURL() throws Exception{
121 StringBuilder sb = new StringBuilder();
122 String html = IOUtils.toString(new FileInputStream(localHTML), "UTF-8");
123 String article = StringUtils.substringBetween(html, "<!-- MAIN_COLUMN_CONTENT_BEGIN -->", "<!-- CMA");
124 Parser parser = new Parser();
125 parser.setInputHTML(article);
126 //这里nodeList将包含网页中的所有内容
127 NodeList nodeList = parser.parse(
128 new NodeFilter(){
129 @Override
130 public boolean accept(Node node) {
131 //接受所有的标签
132 return true;
133 }
134 }
135 );
136 //遍历文章内容中的所有标签
137 for(int i=0; i<nodeList.size(); i++){
138 Node node = nodeList.elementAt(i);
139 if(node instanceof ImageTag){
140 ImageTag it = (ImageTag)node;
141 //修改文章内容中<img>图片地址为本地硬盘上的绝对路径
142 it.setImageURL("D:/Download/" + it.getImageURL());
143 //把<img>标签所有的内容放到StringBuilder
144 sb.append(it.toHtml());
145 }else if(node instanceof TextNode){
146 TextNode tn = (TextNode)node;
147 //对于文本节点,直接将其文本放到StringBuilder
148 sb.append(tn.getText());
149 }else{
150 TagNode tn = (TagNode)node;
151 //将标签原样放到StringBuilder
152 sb.append("<").append(tn.getText()).append(">");
153 }
154 }
155 System.out.println("图片链接修改后的文章内容为:" + sb);
156 //FileUtils的好处是,当文件所在目录不存在时,它会自动创建,省去我们判断的步骤了
157 FileUtils.writeStringToFile(new File("D:/Download/newHTML.html"), sb.toString());
158 }
159}
160
