配置scrapy-splash+python爬取医院信息（利用了scrapy-splash）
北京艾丽斯妇科医院（http://fuke.fuke120.com/）
首先先说一下配置splash
1.利用pip安装scrapy-splash库
pip install scrapy-splash
2.现在就要用到另一个神器（Docker）
Docker下载地址：https://www.docker.com/community-edition#/windows
3.安装好Docker后启动Docker拉取镜像
docker pull scrapinghub/splash
4.利用Docker运行splash
docker run -p 8050:8050 scrapinghub/splash（运行之后大家可以去浏览器输入http://192.168.99.100:8050检查Docker是否正确）
5settings.py配置
SPLASH_URL = 'http://192.168.99.100:8050'（重中之重，一个大坑，一定要注意这个IP就是192.168.99.100，我就一直用的自己IP一直没运行成功）
DOWNLOADER_MIDDLEWARES = {
'scrapy_splash.SplashCookiesMiddleware': 723,
'scrapy_splash.SplashMiddleware': 725,
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
}
SPIDER_MIDDLEWARES = {
'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
}
DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'
HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'
ROBOTSTXT_OBEY = True（此处注意，有的网站是True，而有的网站需要把它改成False）
爬虫的py文件1.py
# -*- coding: utf-8 -*-
import re
from urllib.request import urlopen
from scrapy.http import Request
# from urllib.request import Request
from bs4 import BeautifulSoup
from lxml import etree
import pymongo
import scrapy
from scrapy.selector import HtmlXPathSelector
client = pymongo.MongoClient(host="127.0.0.1")
db = client.Health
collection = db.Healthclass # 表名classification
import redis # 导入redis数据库
r = redis.Redis(host='127.0.0.1', port=6379, db=0)
ii = 0
class healthcareClassSpider(scrapy.Spider):
name = "HealthCare"
allowed_domains = ["fuke120.com"] # 允许访问的域
start_urls = [
"http://fuke.fuke120.com/",
]
# 每爬完一个网页会回调parse方法
def parse(self, response):
global ii
hxs = HtmlXPathSelector(response)
hx = hxs.select('//div[@id="allsort"]/div[@class="item"]/span/a')
hx1 = hxs.select('//div[@id="allsort"]/div[@class="item born"]/span/a')
# hx2 = hxs.select('//div[@id="allsort"]/div[@class="item"]/div[@class="i-mc"]/div[@class="i-mc01"]/ul[@class="w_ul01"]/li/a')
for secItem in hx:
ii+=1
url = secItem.select("@href").extract()
c = "http://fuke.fuke120.com"+url[0]
name = secItem.select("text()").extract()
print(c)
print(name)
classid = collection.insert({'healthclass': name, 'pid': None})
healthurl = '%s,%s,%s' % (classid, c, ii)
r.lpush('healthclassurl',healthurl)
for secItem1 in hx1:
url = secItem1.select("@href").extract()
c1 = "http://fuke.fuke120.com"+url[0]
name1 = secItem1.select("text()").extract()
print(c1)
print(name1)
classid = collection.insert({'healthclass': name1, 'pid': None})
healthurl = '%s,%s,%s' % (classid, c1, 0)
r.lpush('healthclassurl', healthurl)
2.py
# -*- coding: utf-8 -*-
import re
from urllib.request import urlopen
from urllib.request import Request
from bs4 import BeautifulSoup
from lxml import etree
import pymongo
import scrapy
from scrapy.selector import HtmlXPathSelector
from bson.objectid import ObjectId
# from scrapy.http import Request
# from urllib.request import urlopen
from scrapy.http import Request
# from hello.items import ZhaopinItem
# from scrapy.spiders import CrawlSpider, Rule
# from scrapy.linkextractors import LinkExtractor
from urllib.request import Request,ProxyHandler
from urllib.request import build_opener
client = pymongo.MongoClient(host="127.0.0.1")
db = client.Health #库名dianping
collection = db.Diseaseclass #表名classification
import redis #导入redis数据库
r = redis.Redis(host='192.168.60.112', port=6379, db=0, charset='utf-8')
class healthcareClassSpider(scrapy.Spider):
name = "HealthCare1"
allowed_domains = ["fuke120.com"] # 允许访问的域
dict = {}
start_urls = []
def __init__(self):
a = r.lrange('healthclassurl', 0,-1)
for item in a:
healthurl = bytes.decode(item)
arr = healthurl.split(',')
healthcareClassSpider.start_urls.append(arr[1])
num = arr[2]
pid = arr[0]
url = arr[1]
self.dict[url] = {"pid": pid, "num": num}
def parse(self, response):
nameInfo = self.dict[response.url]
pid1 = nameInfo['pid']
pid = ObjectId(pid1)
num = nameInfo['num']
hxs = HtmlXPathSelector(response)
hx = hxs.select('//div[@class="x_con02_2"]/div[@class="x_con02_3"]/ul/li/p/a')
for secItem in hx:
url = secItem.select("@href").extract()
url = "http://fuke.fuke120.com"+url[0]
name = secItem.select("text()").extract()
print(url)
print(name)
classid = collection.insert({'Diseaseclass': name, 'pid': pid})
diseaseclassurl = '%s,%s,%s' % (classid, url, pid)
r.lpush('diseaseclassurl', diseaseclassurl)
3.py
# -*- coding: utf-8 -*-
import re
from urllib.request import urlopen
from urllib.request import Request
from bs4 import BeautifulSoup
from lxml import etree
import pymongo
import scrapy
from scrapy_splash import SplashMiddleware
from scrapy.http import Request, HtmlResponse
from scrapy_splash import SplashRequest
from scrapy.selector import Selector
from scrapy.selector import HtmlXPathSelector
from bson.objectid import ObjectId
# from diseaseHealth.diseaseHealth.spiders.SpiderJsDynamic import phantomjs1
# from scrapy.http import Request
# from urllib.request import urlopen
from scrapy.http import Request
client = pymongo.MongoClient(host="127.0.0.1")
db = client.Health # 库名dianping
collection = db.Treatclass # 表名classification
#
import redis # 导入redis数据库
#
r = redis.Redis(host='192.168.60.112', port=6379, db=0, charset='utf-8')
class healthcareClassSpider(scrapy.Spider):
name = "HealthCare2"
allowed_domains = ["fuke120.com"] # 允许访问的域
dict = {}
start_urls = []
def __init__(self):
a = r.lrange('diseaseclassurl', 0,-1)
for item in a:
healthurl = bytes.decode(item)
arr = healthurl.split(',')
healthcareClassSpider.start_urls.append(arr[1])
num = arr[2]
pid = arr[0]
url = arr[1]
self.dict[url] = {"pid": pid, "num": num}
def start_requests(self):
for url in self.start_urls:
yield SplashRequest(url, self.parse, args={'wait': 0.5})
def parse(self, response):
# a = response.body.decode('utf-8')
# print(a)
nameInfo = self.dict[response.url]
pid1 = nameInfo['pid']
pid = ObjectId(pid1)
num = nameInfo['num']
print(num)
print(pid)
hxs = HtmlXPathSelector(response)
hx = hxs.select('//div[@class="dh01"]/ul[@class="ul_bg01"]/li/a')
for secItem in hx:
url = secItem.select("@href").extract()
c = "http://fuke.fuke120.com" + url[0]
name = secItem.select("text()").extract()
print(c)
print(name)
classid = collection.insert({'Treatclass': name, 'pid': pid})
treatclassurl = '%s,%s,%s' % (classid, c, pid)
r.lpush('treatclassurl', treatclassurl)
大功告成，主要还是为了使用scrapy-splash。
版权声明：本文为wangyuhangboke原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。