1. # coding:utf-8
  2. # 爬取58同城二手电脑信息
  3. # 进入http://bj.58.com/pbdn/0/pn2/页面
  4. # 爬取列表中除转转、推广商品外的正常商品
  6. from bs4 import BeautifulSoup
  7. import requests
  8. import time
  11. def get_links_from(who_sells):  # 爬取列表中除转转、推广商品外的正常商品爬取列表中除转转、推广商品外的正常商品的连接
  12.     urls = []
  13.     list_view = \'http://bj.58.com/pbdn/{}/pn2/\'.format(str(who_sells))
  14.     wb_data = requests.get(list_view)
  15.     soup = BeautifulSoup(wb_data.text, \'lxml\')
  17.     # 通过对页面分析 发现商品链接在 tr > td.t > a.t 中
  18.     for link in soup.select(\'tr td.t a.t\'):
  19.         if len(link.get(\'href\').split(\'?\')[0]) == 53: # 因为转转商品也符合 tr > td.t > a.t,要排除，观察发现正常商品链接
  20.             # 的长度为53, 可通过字符串长度筛选去正常的连接
  21.             urls.append(link.get(\'href\').split(\'?\')[0])
  22.     return urls
  25. def get_views(url):
  26.     id = url.split(\'/\')[-1].strip(\'x.shtml\')
  27.     api = \'http://jst1.58.com/counter?infoid={}\'.format(id)
  28.     js = requests.get(api)
  29.     views = js.text.split(\'=\')[-1]
  30.     return views
  33. def get_item_info(who_sells=0): #
  34.     urls = get_links_from(who_sells)
  36.     for url in urls:
  38.         time.sleep(2)
  39.         web_data = requests.get(url)
  40.         soup = BeautifulSoup(web_data.text, \'lxml\')
  41.         data = {
  42.             \'title\': soup.title.text,
  43.             \'price\': soup.find_all(\'span\', \'price c_f50\')[0].text,
  44.             \'area\': list(soup.select(\'.c_25d\')[0].stripped_strings) if soup.find_all(\'span\',\'c_25d\') else None,
  45.             \'date\': soup.select(\'.time\')[0].text,
  46.             \'cate\': \'个人\' if who_sells == 0 else \'商家\',
  47.             \'views\': get_views(url)
  48.         }
  49.         print(data)
  51. get_item_info()

版权声明：本文为november1943原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://www.cnblogs.com/november1943/p/5242490.html