1 # -*- coding:UTF-8 -*-
 2 from urllib import request
 3 from bs4 import BeautifulSoup
 4 import re
 5 import sys
 6 
 7 if __name__ == "__main__":
 8     #创建txt文件
 9     file = open(\'一念永恒.txt\', \'w\', encoding=\'utf-8\')
10     #一念永恒小说目录地址
11     target_url = \'http://www.biqukan.com/1_1094/\'
12     #User-Agent
13     head = {}
14     head[\'User-Agent\'] = \'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166  Safari/535.19\'
15     target_req = request.Request(url = target_url, headers = head)
16     target_response = request.urlopen(target_req)
17     target_html = target_response.read().decode(\'gbk\',\'ignore\')
18     #创建BeautifulSoup对象
19     listmain_soup = BeautifulSoup(target_html,\'lxml\')
20 
21     #搜索文档树,找出div标签中class为listmain的所有子标签
22     chapters = listmain_soup.find_all(\'div\',class_ = \'listmain\')
23     #使用查询结果再创建一个BeautifulSoup对象,对其继续进行解析
24     download_soup = BeautifulSoup(str(chapters), \'lxml\')
25     #计算章节个数
26     numbers = (len(download_soup.dl.contents) - 1) / 2 - 8
27     index = 1
28     #开始记录内容标志位,只要正文卷下面的链接,最新章节列表链接剔除
29     begin_flag = False
30     #遍历dl标签下所有子节点
31     for child in download_soup.dl.children:
32         #滤除回车
33         if child != \'\n\':
34             #找到《一念永恒》正文卷,使能标志位
35             if child.string == u"《一念永恒》正文卷":
36                 begin_flag = True
37             #爬取链接并下载链接内容
38             if begin_flag == True and child.a != None:
39                 download_url = "http://www.biqukan.com" + child.a.get(\'href\')
40                 download_req = request.Request(url = download_url, headers = head)
41                 download_response = request.urlopen(download_req)
42                 download_html = download_response.read().decode(\'gbk\',\'ignore\')
43                 download_name = child.string
44                 soup_texts = BeautifulSoup(download_html, \'lxml\')
45                 texts = soup_texts.find_all(id = \'content\', class_ = \'showtxt\')
46                 soup_text = BeautifulSoup(str(texts), \'lxml\')                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               
47                 write_flag = True
48                 file.write(download_name + \'\n\n\')
49                 #将爬取内容写入文件
50                 for each in soup_text.div.text.replace(\'\xa0\',\'\'):
51                     if each == \'h\':
52                         write_flag = False
53                     if write_flag == True and each != \' \':
54                         file.write(each)
55                     if write_flag == True and each == \'\r\':
56                         file.write(\'\n\')
57                 file.write(\'\n\n\')
58                 #打印爬取进度
59                 sys.stdout.write("已下载:%.3f%%" % float(index/numbers) + \'\r\')
60                 sys.stdout.flush()
61                 index += 1
62     file.close()

 1 #!/usr/bin/python
 2 # -*- coding: UTF-8 -*-
 3 import requests
 4 from bs4 import BeautifulSoup
 5 # get_url_list 获取所有章节的URL，在一个list里
 6 def get_url_list(url):
 7     content = requests.get(url).content           #获取页面内容
 8     soup = BeautifulSoup(content,\'lxml\')          #Beautifulsoup 实例化对象
 9     url_list = []                                #空的url_list 数组
10     # urls = soup.find(\'div\',{\'id\':\'list\'}).find(\'dl\').find_all(\'dd\')
11     urls = soup.select(\'#list > dl > dd > a\')    # 根据页面选择到URL ，还可以urls = soup.find(\'div\',{\'id\':\'list\'}).find(\'dl\').find_all(\'dd\')
12     for i in urls:          #遍历里面的每一章的URL
13         i = i.get(\'href\')   #获取URL
14         # print(i)      
15         i = \'http://www.biquge.com.tw\' + i   #分析文章组成，形成最终的URL
16         url_list.append(i)    #添加到url_list 里面去
17     # print (url_list)
18     return url_list
19 # 获取这一章的内容
20 def get_data(url):
21     content = requests.get(url).content
22     soup = BeautifulSoup(content, \'lxml\')
23     f = open(r\'C:\Users\HBX\Documents\staudy\HMXX.txt\',\'a+\',encoding=\'utf-8\')  #不加utf-8 会有编码报错
24     text_name = soup.find(\'div\',{\'class\':\'bookname\'}).find(\'h1\').text     #获得章节名字
25     # text_content = soup.select(\'#content\')
26     text_content = soup.find(\'div\',{\'id\':\'content\'}).get_text()    #获得章节内容  ，还有一种select css 选择的获取章节内容的方式
27     book =text_name+ \'\r\n\' + text_content    #整体的一章
28     # print(book)
29     f.write((book)+\'\r\n\')   #换行写入
30     f.close()   #关闭文件
31     # for x in text_content:
32     #     a = x.text.replace(\'readx();\', \'\')
33     #     print(a)
34 
35 
36 
37 if __name__ ==\'__main__\':
38     url = \'http://www.biquge.com.tw/18_18049/\'   #笔趣阁的小说目录页面
39     url_list = get_url_list(url)   #获取了所有的url
40     for i in url_list:    # 循环一章url
41         get_data(i)      #获取文章内容