网址 https://movie.douban.com/top250

一共250部电影，有分页，获取每一部的详细信息

不采用框架，使用 urilib读取网页，re进行正则表达式匹配，lxml进行xpath查找

 1 from film import *
 2 from urllib import request
 3 import time,re
 4 url=r\'https://movie.douban.com/top250?start=\'
 5 for i in range(10):
 6     url=url+str(i*25)
 7     print(url)
 8     
 9     headers = {
10         \'User-Agent\': r\'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) \'
11                       r\'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3\',
12         \'Connection\': \'keep-alive\'
13     }
14     req=request.Request(url,headers=headers)
15     page=request.urlopen(req).read()
16     page=page.decode(\'utf-8\')
17     #fp=open("page.txt",mode="w",encoding="UTF-8")
18     #fp.writelines(page)
19     p=re.compile(r\'\<em\sclass=\"\"\>\d+\</em\>\s*\<a\shref=\"https://movie.douban.com/subject/\d+/\"\>\')
20     result=p.findall(page)
21     for item in result:
22         #print(item)
23         p=re.compile(r\'\d+\')
24         no=p.findall(item)
25         #print(no[0])
26         p=re.compile(r\'https://movie.douban.com/subject/\d+/\')
27         rurl=p.findall(item)
28         #print(rurl[0])
29         filma=film(no[0],rurl[0],\'\',\'\',\'\',\'\',\'\',\'\')
30         filma.getall()
31         filma.detail()
32         time.sleep(3)
33     #print (result)
34     time.sleep(3)
35     #print(i)

 

film.py 如果要做数据的持久化，在这里实现

 1 from urllib import request
 2 from lxml import etree
 3 class film:
 4     def __init__(self,no,url,name,year,score,director,classification,actor):
 5         self.name=name
 6         self.year=year
 7         self.score=score
 8         self.director=director
 9         self.classification=classification
10         self.actor=actor
11         self.url=url
12         self.no=no
13     
14     def detail(self):
15         temp = "No：%s;url：%s;片名：%s;年份：%s;分数：%s;导演：%s;分级：%s;演员：%s;"   %(self.no,self.url,self.name,self.year,self.score,self.director,self.classification,self.actor)  
16         print(temp)
17     def getall(self):
18         headers={
19         \'User-Agent\': r\'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) \'
20                       r\'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3\',
21         \'Connection\': \'keep-alive\'
22         }
23         req=request.Request(self.url,headers=headers)
24         page=request.urlopen(req).read()
25         page=page.decode(\'utf-8\')
26         selector=etree.HTML(page)
27         print (page)
28         self.name=selector.xpath(\'/html/body/div[3]/div[1]/h1/span[1]/text()\')
29         self.year=selector.xpath(\'//*[@id="content"]/h1/span[2]/text()\')
30         self.score=selector.xpath(\'//*[@id="interest_sectl"]/div[1]/div[2]/strong/text()\')
31         self.director=selector.xpath(\'//*[@id="info"]/span[1]/span[2]/a/text()\')
32         self.classification=selector.xpath(\'//*[@id="info"]/span[5]/text()\')
33         self.actor=selector.xpath(\'//*[@id="info"]/span[3]/span[2]/a/text()\')
34         
35

 

版权声明：本文为newvoyage原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://www.cnblogs.com/newvoyage/p/7043682.html