首先需要在ip代理的网站爬取有用的ip，保存到数据库中

  1. import requests
  2. from scrapy.selector import Selector
  3. import pymysql
  5. conn = pymysql.connect(host = \'127.0.0.1\', user = \'root\' ,passwd = \'root\',db = \'mysql18_text\',charset = \'utf8\')
  6. cursor = conn.cursor()
  8. def crawl_ips():
  9.     #爬取xici的免费ip代理
  10.     agent = \'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0\'
  11.     header = {
  12.         \'User-Agent\':agent
  13.     }
  15.     for i in range(1,3458):
  16.         reas = requests.get(\'http://www.xicidaili.com/nn/\',headers = header)
  17.         Selectora = Selector(reas)
  18.         all_trs = Selectora.xpath(\'//table[@id="ip_list"]/tr\')
  19.         ip_list = []
  20.         for tr in all_trs[1:]:
  21.             spend_str = tr.xpath(\'./td/div[@class="bar"]/@title\').extract()[0] ##提取速度
  22.             if spend_str:
  23.                 speed = float(spend_str.split(\'秒\')[0])
  24.                 all_text = tr.xpath(\'./td/text()\').extract()
  25.                 ip = all_text[0]
  26.                 port = all_text[1]
  27.                 proxy_type = all_text[5]
  28.                 ip_list.append((ip,port,speed,proxy_type))
  29.         for ip_info in ip_list:
  30.             cursor.execute(
  31.                 """insert project_ip(ip,port,speed,proxy_type) VALUES(\'{0}\',\'{1}\',\'{2}\',\'HTTP\')""".format(
  32.                     ip_info[0],ip_info[1],ip_info[2]
  33.                 )
  34.             )
  35.             conn.commit()
  36.         print(ip_list)
  37.         
  38.         
  39. crawl_ips()
  41. conn.close()
  42. cursor.close()

随机在数据库中获取一个ip的代码

  1. class GetIP(object):
  2.     
  3.     def delete_ip(self,ip):
  4.         #从数据库中删除无效的ip
  5.         delete_sql = """
  6.             delete from project_ip where ip=\'{0}\'
  7.         """.format(ip)
  8.         cursor.execute(delete_sql)
  9.         conn.commit()
  10.         return True
  11.     
  12.     
  13.     def judge_ip(self,ip,port):
  14.         #判断一个ip是否可用
  15.         http_url = \'http://www.baidu.com\'
  16.         proxy_url = \'https://{0}:{1}\'.format(ip,port)
  17.         
  18.         try:
  19.             proxy_dict = {
  20.                 \'http\':proxy_url,
  21.             }
  22.             requests.get(http_url,proxies = proxy_dict)
  23.             return True
  24.         except Exception as e:
  25.             print("ip出现异常")
  26.             #出现异常后就把这个ip给删除掉
  27.             self.delete_ip(ip)
  28.             return False
  29.         else:
  30.             code = response.status_code
  31.             if code>=200 and code<300:
  32.                 print(\'effective ip\')
  33.                 return True
  34.             else:
  35.                 print(\'invalid\')
  36.                 self.delete_ip(ip)
  37.                 return False
  40.     
  41.     def get_random_ip(self):
  42.         #从数据库中随机获取到一个可用的ip
  43.         random_sql = """
  44.             SELECT ip,port FROM project_ip
  45.             ORDER BY RAND()
  46.             LIMIT 1
  47.         """
  48.         result = cursor.execute(random_sql)
  49.         
  50.         for ip_info in cursor.fetchall():
  51.             ip = ip_info[0]
  52.             port = ip_info[1]
  53.             judge_re = self.judge_ip(ip,port)
    　　　　　　　if judge_re:#如果返回True
    　　　　　　　　  return "http://\'{0}\':\'{1}\'".format(ip,port)
    　　　　　　　else:
    　　　　　　　　　 return get_random_ip()

Middleware动态设置ip代理

  1. class RandomProxyMiddleware(object):
  2.     def process_request(self,request,spider):
  3.         get_ip = GetIP()#这里需要导入那个函数
  4.         request.meta[\'proxy\'] = get_ip.get_random_ip()

 

版权声明：本文为ArtisticMonk原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://www.cnblogs.com/ArtisticMonk/p/9738921.html