1 from urllib import request
 2 import random
 3 import time
 4 import csv
 5 import re
 6 import xlwt
 7 
 8 class catEyesMovie:
 9 
10     def __init__(self):
11         self.url = 'https://maoyan.com/board/4?offset={}'
12         self.ua_list = [
13             'Win7:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1'
14         ]
15         self.line = 1;
16             #'User-Agent': 'Win7:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1'
17 
18 
19     #获取网页内容
20     def get_page(self,url):
21         #使用随机的user-agent
22         headers = {'User-Agent':random.choice(self.ua_list)}
23         #创建请求对象
24         req = request.Request(url=url,headers=headers)
25         #发送请求
26         res = request.urlopen(req)
27         #获取相应对象
28         html = res.read().decode('utf-8','ignore')
29         return html
30 
31     #清洗数据
32     def clean_page(self,html,xwlt):
33         pattern = re.compile("""<div class="movie-item-info">.*?title="(.*?)".*?class="star">(.*?)</p>.*?releasetime">(.*?)</p>""", re.S)
34         r_list = pattern.findall(html)
35         self.write_page(r_list,xwlt)
36 
37 
38     #保存内容
39     def write_page(self,r_list,xwlt):
40         one_film_dict = {}
41         for rt in r_list:
42             xwlt.write(self.line,0,rt[0].strip())
43             xwlt.write(self.line,1,rt[1].strip())
44             xwlt.write(self.line,2,rt[2].strip())
45             self.line+=1
46 
47 
48 
49     def main(self,xwlt):
50         #凭借字符串
51         res = []
52         for i in range(1,11):
53             #拼接url地址 https://maoyan.com/board/4?offset={}
54             #获取当前页数
55             offset = (i-1)*10
56             url = self.url.format(offset)
57             html = self.get_page(url)
58             self.clean_page(html,xwlt)
59 
60 
61 
62 
63 
64 if __name__ == '__main__':
65     start = time.time()
66     spider = catEyesMovie()
67     #创建一个xlwt对象
68     book = xlwt.Workbook(encoding='utf-8')
69     #创建sheet,Sheet1为表的名字,cell_overwirite_ok为是否覆盖单元格
70     sheet1 = book.add_sheet(u'Sheet1',cell_overwrite_ok=True)
71     #进行第一行标题定义
72     sheet1.write(0,0,'电影名称')
73     sheet1.write(0,1,'主演')
74     sheet1.write(0,2,'上映时间')
75     #进行爬取
76     spider.main(sheet1)
77     book.save('D:\\write.xls')
78     end = time.time()
79     print('执行时间为: %.2f' % (end-start))

1 {'电影名称': '速度与激情5', '主演': '速度与激情5', '上映时间': '速度与激情5'}
2 {'电影名称': '驯龙高手', '主演': '驯龙高手', '上映时间': '驯龙高手'}
3 {'电影名称': '勇敢的心', '主演': '勇敢的心', '上映时间': '勇敢的心'}
4 {'电影名称': '闻香识女人', '主演': '闻香识女人', '上映时间': '闻香识女人'}
5 {'电影名称': '神偷奶爸', '主演': '神偷奶爸', '上映时间': '神偷奶爸'}