释放双眼，带上耳机，听听看~！
本文所用到的localHTML.html等练习文件
详见此处下载：链接地址
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
2
3import java.io.FileInputStream;
4import java.io.FileOutputStream;
5import java.util.List;
6
7import org.apache.commons.io.IOUtils;
8import org.apache.http.HttpEntity;
9import org.apache.http.HttpResponse;
10import org.apache.http.client.HttpClient;
11import org.apache.http.client.methods.HttpGet;
12import org.apache.http.impl.client.DefaultHttpClient;
13import org.apache.http.util.EntityUtils;
14import org.htmlparser.Node;
15import org.htmlparser.NodeFilter;
16import org.htmlparser.Parser;
17import org.htmlparser.tags.Div;
18import org.htmlparser.tags.LinkTag;
19import org.htmlparser.tags.MetaTag;
20import org.htmlparser.util.NodeList;
21
22/**
23 * HTMLParser入门_02_网络爬虫的雏形_解析文章的主题和作者及关键字等信息
24 * @see ---------------------------------------------------------------------------------------------------------
25 * @see 所有jar如下
26 * @see commons-io-2.3.jar
27 * @see commons-codec-1.6.jar(以下7个jar取自HttpClient官网下载的httpcomponents-client-4.2.1-bin.zip)
28 * @see commons-logging-1.1.1.jar
29 * @see fluent-hc-4.2.1.jar
30 * @see httpclient-4.2.1.jar
31 * @see httpclient-cache-4.2.1.jar
32 * @see httpcore-4.2.1.jar
33 * @see httpmime-4.2.1.jar
34 * @see filterbuilder.jar(以下5个jar取自HTMLParser官网下载的HTMLParser-2.0-SNAPSHOT-bin.zip)
35 * @see htmllexer.jar
36 * @see htmlparser.jar
37 * @see sitecapturer.jar
38 * @see thumbelina.jar
39 * @see ---------------------------------------------------------------------------------------------------------
40 * @see 本文所用的HTMLParser工具类,详见我的下面的这一篇文章
41 * @see http://blog.csdn.net/jadyer/article/details/8656479
42 * @see ---------------------------------------------------------------------------------------------------------
43 * @create Mar 10, 2013 5:05:55 PM
44 * @author 玄玉<http://blog.csdn/net/jadyer>
45 */
46public class SpiderDemo {
47 private static final String articleURI = "http://www.ibm.com/developerworks/cn/java/j-javaroundtable/index.html";
48 private static final String localHTML = "D:/Download/localHTML.html";
49
50 /**
51 * 下载文章
52 */
53 private static void downloadArticle() throws Exception {
54 HttpClient httpClient = new DefaultHttpClient();
55 HttpGet httpGet = new HttpGet(articleURI);
56 try {
57 HttpResponse response = httpClient.execute(httpGet);
58 HttpEntity entity = response.getEntity();
59 if(null != entity){
60 String responseContent = EntityUtils.toString(entity, "UTF-8");
61 EntityUtils.consume(entity);
62 //文章内容写到本地(IOUtils干完活儿会自动关闭IO流)
63 IOUtils.write(responseContent, new FileOutputStream(localHTML), "UTF-8");
64 }
65 }finally{
66 httpClient.getConnectionManager().shutdown();
67 }
68 }
69
70
71 /**
72 * 解析具有某种特征的标签
73 */
74 @SuppressWarnings("serial")
75 private static void parseSpecifiedTag() throws Exception{
76 String inputHTML = IOUtils.toString(new FileInputStream(localHTML), "UTF-8");
77 Parser parser = new Parser();
78 parser.setInputHTML(inputHTML);
79 //提取name="title"的<meta>标签,符合条件的应该只有一个<meta>标签,所以这里用单数
80 //这里我们自己写一个Filter,并且用内部类的方式
81 NodeList metaTag = parser.parse(
82 new NodeFilter(){
83 @Override
84 public boolean accept(Node node) {
85 //找到<meta>标签
86 if(node instanceof MetaTag){
87 MetaTag mt = (MetaTag)node;
88 //找到<meta name="title">的标签
89 if(null!=mt.getMetaTagName() && "title".equals(mt.getMetaTagName())){
90 return true;
91 }
92 }
93 return false;
94 }
95 }
96 );
97 //提取<meta name="title" content="2010 年春 Java 平台圆桌会议"/>标签中的content属性值
98 System.out.println("name=title,content=" + ((MetaTag)metaTag.elementAt(0)).getMetaContent());
99 }
100
101
102 /**
103 * 解析文章的简介,关键字,作者姓名等信息
104 */
105 private static void parseAbstractAndKeywords() throws Exception{
106 String html = IOUtils.toString(new FileInputStream(localHTML), "UTF-8");
107 List<MetaTag> metaTags = HTMLParseUtil.parseTags(html, MetaTag.class, "name", "Abstract");
108 for(MetaTag mt : metaTags){
109 System.out.println("文章的简介:" + mt.getMetaContent());
110 }
111 MetaTag mt = HTMLParseUtil.parseTag(html, MetaTag.class, "name", "Keywords");
112 System.out.println("文章关键字:" + mt.getMetaContent());
113 List<Div> divTags = HTMLParseUtil.parseTags(html, Div.class, "class", "author");
114 for(Div div : divTags){
115 //div.getStringText()可以得到<div></div>所嵌套的内容
116 LinkTag aTag = HTMLParseUtil.parseTag(div.getStringText(), LinkTag.class, "class", "dwauthor");
117 System.out.println("作者姓名:" + aTag.getStringText());
118 }
119 }
120}
121
