from scrapy.http import HtmlResponse
from scrapy.http import Request

class Md1(object):
	@classmethod
	def from_crawler(cls, crawler):
		# This method is used by Scrapy to create your spiders.
		s = cls()
		return s

	def process_request(self, request, spider):
		# Called for each request that goes through the downloader
		# middleware.

		# Must either:
		# - return None: continue processing this request
		# - or return a Response object
		# - or return a Request object
		# - or raise IgnoreRequest: process_exception() methods of
		#   installed downloader middleware will be called
		print('md1.process_request',request)
		# 1. 返回Response
		# import requests
		# result = requests.get(request.url)
		# return HtmlResponse(url=request.url, status=200, headers=None, body=result.content)
		# 2. 返回Request
		# return Request('https://dig.chouti.com/r/tec/hot/1')

		# 3. 抛出异常
		# from scrapy.exceptions import IgnoreRequest
		# raise IgnoreRequest

		# 4. 对请求进行加工(*)
		# request.headers['user-agent'] = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"


	def process_response(self, request, response, spider):
		# Called with the response returned from the downloader.

		# Must either;
		# - return a Response object
		# - return a Request object
		# - or raise IgnoreRequest
		print('m1.process_response',request,response)
		return response

	def process_exception(self, request, exception, spider):
		# Called when a download handler or a process_request()
		# (from other downloader middleware) raises an exception.

		# Must either:
		# - return None: continue processing this exception
		# - return a Response object: stops process_exception() chain
		# - return a Request object: stops process_exception() chain
		pass

DOWNLOADER_MIDDLEWARES = {
   #'xdb.middlewares.XdbDownloaderMiddleware': 543,
	# 'xdb.proxy.XdbProxyMiddleware':751,
	'xdb.md.Md1':666,
	'xdb.md.Md2':667,
}

class Sd1(object):
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.

@classmethod
def from_crawler(cls, crawler):
	# This method is used by Scrapy to create your spiders.
	s = cls()
	return s

def process_spider_input(self, response, spider):
	# Called for each response that goes through the spider
	# middleware and into the spider.

	# Should return None or raise an exception.
	return None

def process_spider_output(self, response, result, spider):
	# Called with the results returned from the Spider, after
	# it has processed the response.

	# Must return an iterable of Request, dict or Item objects.
	for i in result:
		yield i

def process_spider_exception(self, response, exception, spider):
	# Called when a spider or process_spider_input() method
	# (from other spider middleware) raises an exception.

	# Should return either None or an iterable of Response, dict
	# or Item objects.
	pass

# 只在爬虫启动时，执行一次。
def process_start_requests(self, start_requests, spider):
	# Called with the start requests of the spider, and works
	# similarly to the process_spider_output() method, except
	# that it doesn’t have a response associated.

	# Must return only requests (not items).
	for r in start_requests:
		yield r

SPIDER_MIDDLEWARES = {
   # 'xdb.middlewares.XdbSpiderMiddleware': 543,
	'xdb.sd.Sd1': 666,
	'xdb.sd.Sd2': 667,
}