python爬虫实战：爬取股票信息，对上交所和深交所所有的股票信息进行搜集

python爬虫实战：爬取股票信息，对上交所和深交所所有的股票信息进行搜集

要用到两个网站：
1.获取所有股票的名称的网址（这里指上交所和深交所的股票）
.html2.获取单个股票的各类信息
.html

'''
要用到两个网站：
1.获取所有股票的名称的网址（这里指上交所和深交所的股票）
.html2.获取单个股票的各类信息
.html
'''import requests
from bs4 import BeautifulSoup
import traceback
import re#获取网页内容
def getHTMLText(url, code="utf-8"):try:r = (url)r.raise_for_status()r.encoding = cept:return ""#获取所有的股票名称，将其放在一个列表中
def getStockList(lst, stockURL):html = getHTMLText(stockURL, "GB2312")soup = BeautifulSoup(html, 'html.parser')a = soup.find_all('a')for i in a:try:href = i.attrs['href']lst.append(re.findall(r"d{6}", href)[0])except:continuedef getStockInfo(lst, stockURL, fpath):count = 0for stock in lst:url = stockURL + "sz" + stock + ".html"#对应的每只股票的网址html = getHTMLText(url)try:if html == "":continueinfoDict = {}soup = BeautifulSoup(html, 'html.parser')stockInfo = soup.find('div', attrs={'class': 'stock-bets'})name = stockInfo.find_all(attrs={'class': 'bets-name'})[0]infoDict.update({'股票名称': split()[0]})keyList = stockInfo.find_all('dt')valueList = stockInfo.find_all('dd')for i in range(len(keyList)):key = keyList[i].textval = valueList[i].textinfoDict[key] = val#保存到本地，并加载进度条with open(fpath, 'a', encoding='utf-8') as f:f.write(str(infoDict) + 'n')count = count + 1print("r当前进度: {:.2f}%".format(count * 100 / len(lst)), end="")except:count = count + 1print("r当前进度: {:.2f}%".format(count * 100 / len(lst)), end="")continuedef main():stock_list_url = '.html'stock_info_url = '/'output_file = 'D:/'slist = []getStockList(slist, stock_list_url)getStockInfo(slist, stock_info_url, output_file)main()