python爬网易新闻

print(model_url)

(2)解析每个板块

由于从下载器下载下来的五大板块的详情页数据为静态数据，而目标的实际页面为动态加载的新闻数据。因此需要在引擎和下载器之间的中间件中设置方法来实现获取动态加载新闻数据。

​

wangyi.py文件

#每一个板块对应的新闻标题相关的内容都是动态加载

def parse_model(self,response): #解析每一个板块页面中对应新闻的标题和新闻详情页的url

# response.xpath()

div_list = response.xpath('/html/body/div/div[3]/div[4]/div[1]/div/div/ul/li/div/div')

for div in div_list:

title = div.xpath('./div/div[1]/h3/a/text()').extract_first()

new_detail_url = div.xpath('./div/div[1]/h3/a/@href').extract_first()

item = WangyiproItem()

item['title'] = title

#对新闻详情页的url发起请求

yield scrapy.Request(url=new_detail_url,callback=self.parse_detail,meta={'item':item})

在wangyi.py中添加selenium模块，用于进行动态加载页面的处理。

middlewares.py文件

from scrapy import signals

from scrapy.http import HtmlResponse

from time import sleep

class WangyiproDownloaderMiddleware:

# Not all methods need to be defined. If a method is not defined,

# scrapy acts as if the downloader middleware does not modify the

# passed objects.

def process_request(self, request, spider):

# Called for each request that goes through the downloader

# middleware.

# Must either:

# - return None: continue processing this request

# - or return a Response object

# - or return a Request object

# - or raise IgnoreRequest: process_exception() methods of

# installed downloader middleware will be called

return None

#通过该方法拦截五大板块对应的响应对象，进行篡改

def process_response(self, request, response, spider):

#获取在爬虫类定义的浏览器对象

bro = spider.bro

#挑选出指定的响应对象进行篡改

#通过url指定request

#通过request指定response

#spider表示的是爬虫对象

if request.url dels_urls:

#五个模板对应的url进行请求

sleep(2)

page_text = bro.page_source #包含了动态加载的新闻数据

#response #五大板块对应的响应对象

#针对定位到的这些response进行篡改

#实例化一个新的响应对象(符合需求：包含动态加载出的新闻数据)，替代原来旧的响应对象

#如何获取动态加载出的新闻数据？

#基于selenium可便捷的获取动态加载数据

new_response = HtmlResponse(url=request.url,body=page_text,encoding='utf-8',request=request)

return new_response

else:

#response #其他请求对应的响应对象

return response

def process_exception(self, request, exception, spider):

# Called when a download handler or a process_request()

# (from other downloader middleware) raises an exception.

# Must either:

# - return None: continue processing this exception

# - return a Response object: stops process_exception() chain

# - return a Request object: stops process_exception() chain

pass

在process_response(self,request,response,spider)方法中拦截五大板块的响应对象，对其进行篡改。方法是通过url指定request，再通过request指定response。从spider中获取bro浏览器对象，再从中获取动态加载的页面数据。从而构建出动态加载的页面的新的请求对象。

(3)解析每个模块里的标题中详情页信息

wangyi.py

import scrapy

from selenium import webdriver

from wangyiPro.items import WangyiproItem

class WangyiSpider(scrapy.Spider):

name = 'wangyi'

# allowed_domains = ['wwwcom']

start_urls = ['/']

models_urls = [] #存储五个板块对应详情页的url

#解析五大板块对应详情页的url

#实例化一个浏览器对象

def __init__(self):

self.bro = webdriver.Chrome(executable_path='D:WebCrawlerscrapy_programwangyiProwangyiProspiderschromedriver')

def parse(self, response):

li_list = response.xpath('//*[@id="index2016_wrap"]/div[1]/div[2]/div[2]/div[2]/div[2]/div/ul/li')

alist = [3,4,6,7,8]

for index in alist:

model_url = li_list[index].xpath('./a/@href').extract_first()