1 ITEM_PIPELINES = {
2     \'air_quality.pipelines.AirQualityPipeline\': 300,
3 }

 1 import scrapy
 2 
 3 
 4 class AirQualityItem(scrapy.Item):
 5     # define the fields for your item here like:
 6     # name = scrapy.Field()
 7     city_name = scrapy.Field()  # 城市名称
 8     record_date = scrapy.Field()  # 检测日期
 9     aqi_val = scrapy.Field()  # AQI
10     range_val = scrapy.Field()  # 范围
11     quality_level = scrapy.Field()  # 质量等级
12     pm2_5_val = scrapy.Field()  # PM2.5
13     pm10_val = scrapy.Field()  # PM10
14     so2_val = scrapy.Field()  # SO2
15     co_val = scrapy.Field()  # CO
16     no2_val = scrapy.Field()  # NO2
17     o3_val = scrapy.Field()  # O3
18     rank = scrapy.Field()  # 排名

 1 from scrapy.exporters import CsvItemExporter
 2 
 3 class AirQualityPipeline(object):
 4 
 5     def open_spider(self,spider):
 6         self.file = open(\'air_quality.csv\', \'wb\')
 7         self.exporter = CsvItemExporter(self.file)
 8         self.exporter.start_exporting()
 9 
10     def close_spider(self,spider):
11         self.exporter.finish_exporting()
12         self.file.close()
13 
14     def process_item(self, item,spider):
15         self.exporter.export_item(item)
16         return item

 1 # -*- coding: utf-8 -*-
 2 import scrapy
 3 from urllib import parse
 4 from air_quality.items import AirQualityItem
 5 
 6 base_url = \'https://www.aqistudy.cn/historydata/\'
 7 
 8 class ApiHistorySpiderSpider(scrapy.Spider):
 9     name = \'api_history_spider\'
10     allowed_domains = ["aqistudy.cn"]
11     start_urls = [\'https://www.aqistudy.cn/historydata/\']
12 
13     def parse(self, response):
14         """
15             解析初始页面
16         """
17         # 获取所有城市的URL
18         city_url_list = response.xpath(\'//div[@class="all"]//div[@class="bottom"]//a//@href\')
19 
20         for city_url in city_url_list:
21             # 依次遍历城市URL
22             city_month_url = base_url + city_url.extract()
23             # 解析每个城市的月份数据
24             request = scrapy.Request(city_month_url, callback=self.parse_city_month)
25             yield request
26 
27     def parse_city_month(self, response):
28         """
29             解析该城市的月份数据
30         """
31         # 获取该城市的所有月份URL
32         month_url_list = response.xpath(\'//table[@class="table table-condensed \'
33                                         \'table-bordered table-striped table-hover \'
34                                         \'table-responsive"]//a//@href\')
35 
36         for month_url in month_url_list:
37             # 依次遍历月份URL
38             city_day_url = base_url + month_url.extract()
39             # 解析该城市的每日数据
40             request = scrapy.Request(city_day_url, callback=self.parse_city_day)
41             yield request
42 
43     def parse_city_day(self, response):
44         """
45             解析该城市的每日数据
46         """
47         url = response.url
48         item = AirQualityItem()
49         city_url_name = url[url.find(\'=\') + 1:url.find(\'&\')]
50 
51         # 解析url中文
52         # item[\'city_name\'] = city_url_name
53         item[\'city_name\'] = parse.unquote(city_url_name)
54 
55         # 获取每日记录
56         day_record_list = response.xpath(\'//table[@class="table table-condensed \'
57                                          \'table-bordered table-striped table-hover \'
58                                          \'table-responsive"]//tr\')
59         for i, day_record in enumerate(day_record_list):
60             if i == 0:
61                 # 跳过表头
62                 continue
63             td_list = day_record.xpath(\'.//td\')
64 
65             item[\'record_date\'] = td_list[0].xpath(\'text()\').extract_first()  # 检测日期
66             item[\'aqi_val\'] = td_list[1].xpath(\'text()\').extract_first()  # AQI
67             item[\'range_val\'] = td_list[2].xpath(\'text()\').extract_first()  # 范围
68             item[\'quality_level\'] = td_list[3].xpath(\'.//div/text()\').extract_first()  # 质量等级
69             item[\'pm2_5_val\'] = td_list[4].xpath(\'text()\').extract_first()  # PM2.5
70             item[\'pm10_val\'] = td_list[5].xpath(\'text()\').extract_first()  # PM10
71             item[\'so2_val\'] = td_list[6].xpath(\'text()\').extract_first()  # SO2
72             item[\'co_val\'] = td_list[7].xpath(\'text()\').extract_first()  # CO
73             item[\'no2_val\'] = td_list[8].xpath(\'text()\').extract_first()  # NO2
74             item[\'o3_val\'] = td_list[9].xpath(\'text()\').extract_first()  # O3
75             item[\'rank\'] = td_list[10].xpath(\'text()\').extract_first()  # 排名
76 
77             yield item