1 import requests
 2 import re
 3 import bs4
 4 import traceback
 5 
 6 def getHTMLText(url, code = "utf-8"):
 7     # 获得股票页面
 8     try:
 9         r = requests.get(url)
10         r.raise_for_status()
11         r.encoding = code
12         # r.encoding = r.apparent_encoding
13         # 直接用"utf-8"编码节省时间
14         return r.text
15     except:
16         return ""
17 
18 def getStockList(lst, stockURL):
19     # 获取股票列表
20     html = getHTMLText(stockURL, "GB2312")
21     # 东方财富网用"GB2312"方式编码
22     soup = bs4.BeautifulSoup(html, "html.parser")
23     a = soup. find_all("a")
24     for i in a:
25         try:
26             href = i.attrs["href"]
27             lst.append(re.findall(r"[s][hz]\d{6}", href)[0])
28         except:
29             continue
30 
31 def getStockInfo(lst, stockURL, fpath):
32 
33     count = 0
34     # 增加进度条
35 
36     # 获取个股信息
37     for stock in lst:
38         url = stockURL + stock + ".html"
39         html = getHTMLText(url)
40         try:
41             if html == "":
42             # 判断页面是否为空
43                 continue
44             infoDict = { }
45             # 定义一个字典用来储存股票信息
46             soup = bs4.BeautifulSoup(html, "html.parser")
47             stockInfo = soup.find("div", attrs={"class":"stock-bets"})
48             # 获得股票信息标签
49 
50             name = stockInfo.find_all(attrs={"class":"bets-name"})[0]
51             # 在标签中查找股票名称
52             infoDict.update({"股票名称":name.text.split()[0]})
53             # 将股票名称增加到字典中
54 
55             keyList = stockInfo.find_all("dt")
56             # "dt"标签是股票信息键的域
57             valueList = stockInfo.find_all("dd")
58             # "dd"标签是股票信息值的域
59 
60             for i in range(len(keyList)):
61             # 还原键值对并存储到列表中
62                 key = keyList[i].text
63                 val = valueList[i].text
64                 infoDict[key] = val
65 
66             with open(fpath, "a", encoding="utf-8") as f:
67                 f.write(str(infoDict) + "\n")
68 
69                 count += 1
70                 # 增加进度条
71                 print("\r当前进度:{:.2f}%".format(count*100/len(lst)),end = "")
72 
73         except:
74             count += 1
75             # 增加进度条
76             print("\r当前进度：{:.2f}%".format(count * 100 / len(lst)), end="")
77 
78             # 用traceback获得异常信息
79             #traceback.print_exc()
80             continue
81     return ""
82 
83 if __name__ == \'__main__\':
84     stock_list_url = "http://quote.eastmoney.com/stocklist.html"
85     # 获得个股链接
86     stock_info_url = "https://gupiao.baidu.com/stock/"
87     # 获取股票信息的主题部分
88     output_file = "C:\\Users\\W419L\\Desktop\\股票爬取.txt"
89     # 文件保存地址
90     slist = []
91     # 存储股票信息
92     getStockList(slist, stock_list_url)
93     getStockInfo(slist, stock_info_url, output_file)