导入深交所年报代码

导入深交所年报代码

# -*- coding: utf-8 -*-
"""
Created on Sat Sep 14 15:34:11 2019@author: gogogo
"""import requests
import time
import pandas as pd
import random
import os
import json
# 定义爬取函数
def get_pdf_address(pageNum,start_date,end_date):url = '=%s' % random.random()headers = {'Accept': 'application/json, text/javascript, */*; q=0.01','Accept-Encoding': 'gzip, deflate','Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8','Content-Type': 'application/json','Host': 'www.szse','Origin': '','Proxy-Connection': 'keep-alive','Referer': '.html','User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36','X-Request-Type': 'ajax','X-Requested-With': 'XMLHttpRequest'}pagenum = int(pageNum)payload = {"seDate":["{}-12-31".format(start_date),"{}-12-31".format(end_date)],"channelCode":["fixed_disc"],"bigCategoryId":["010301"],"pageSize":30,"pageNum":pagenum}r = requests.post(url,headers = headers,data = json.dumps(payload))result = r.json()return result#创建一个DataFrame储存爬取信息
data_download_pdf = pd.DataFrame(columns =['secCode','secName','url','title','publishTime'])
count = 0
url_head = '/'
end = 2018
while end > 2017:start = end  - 2result = get_pdf_address(1,start,end)print("开始爬取{}-12-31至{}-12-31的年报".format(start,end))pages = int(result['announceCount']/30 + 1)print("共%s页" % pages)for i in range(1,pages):print("爬取深交所年报下载地址第{}页".format(i))print("完成{}%".format(round(i/pages,4) * 100))result = get_pdf_address(i,start,end)num = len(result['data'])for each in range(num):secCode = result['data'][each]['secCode'][0]secName = result['data'][each]['secName'][0]url = url_head + result['data'][each]['attachPath']title = result['data'][each]['title']publishTime = result['data'][each]['publishTime']dict1 = {'secCode':secCode,'Name':secName,'url':url,'title':title,'publishTime':publishTime}data_download_pdf = data_download_pdf.append(dict1, ignore_index=True )print('获取完成')print("========================================")time.sleep(random.uniform(1,2)) # 控制访问速度end = end - 2# 提取title中字符串获取年份
data_download_pdf['Year'] = data_download_pdf['title'].act('([0-9]{4})')
file_path = "F:\深交所年报\"
headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}
# 文件路径要以\结尾，如放入F盘年报文件夹，现在F盘创建文件夹，将路径写成   F:\年报\
for each in range(data_download_pdf.shape[0]):# each = 1# pdf_url = "//disc/disk02/finalpage/2019-07-05/dde0ce5e-e2c7-4c09-b6f4-a03ad9d593ee.PDF"code = data_download_pdf.at[each,'secCode']name = data_download_pdf.at[each,'secName'].replace("*","")year = data_download_pdf.at[each,'Year']print("开始下载{}，股票代码{}的{}年报".format(name,code,year))file_name = "{}{}{}.pdf".format(code,name,year)file_full_name = os.path.join(file_path, file_name)file_full_name  = 'F:\1.pdf'# print(file_full_name)pdf_url = data_download_pdf.at[each,'url']rs = (pdf_url,headers= headers, stream=True)with open(file_full_name, "wb") as fp:for chunk in rs.iter_content(chunk_size=10240):if chunk:fp.write(chunk)time.sleep(random.uniform(1,2)) # 控制访问速度print("===================下载完成==========================")