python爬虫微信公众号留言

self.fail_time = fail_time

self.timeout = timeout

def prepare(self):

p = PreparedRequest()

p.prepare(

method&#hod,

url=self.url,

headers=self.headers,

)

return p

class WeixinResponse():

def __init__(self, text):

def set_status_code(self, status_code):

self.status_code = status_code

import pickle

from redis import StrictRedis

class RedisQueue():

def __init__(self):

"""

初始化redis

"""

self.db = StrictRedis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=3)

def add(self, request):

"""

向队列添加序列化后的Request

:param request:请求对象

:return:添加结果

"""

if isinstance(request, WeixinRequest):

return self.db.rpush(REDIS_KEY, pickle.dumps(request))

return False

def pop(self):

"""

取出下一个request并反序列化

:return: Request 或者 None

"""

if self.db.llen(REDIS_KEY):

return pickle.loads(self.db.lpop(REDIS_KEY))

return False

def empty(self):

return self.db.llen(REDIS_KEY) == 0

def del_all(self):

return self.db.delete(REDIS_KEY)

def get_proxy(self):

"""

从代理池获取代理IP

:return:

"""

try:

response = (PROXY_POOL_URL)

if response.status_code == 200:

print('get Proxy', )

except requests.ConnectionError:

return None

from urllib.parse import urlencode

from requests import ReadTimeout, ConnectionError

from pyquery import PyQuery as pq

VALD_STATUES = [200]

class Spider():

base_url = '?'

# 这里的page可以修改，即第几页，我本来想获取所有的个数再除以10 这样就能爬完了，但是我只是测试所以这里并没有做

# 但如果需要做可以加到schedule方法的while循环内的最下面 即self.params['page']+=1

params = {'type': 2, 's_from': 'input', 'query': keyword, 'page': 1, 'ie': 'utf8', '_sug_': 'n',

'_sug_type_': ''}

headers = {'Host': 'weixin.sogou',

'Connection': 'keep-alive',

'Cache-Control': 'max-age=0',

'Upgrade-Insecure-Requests': '1',

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',

'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',

'Accept-Encoding': 'gzip, deflate',

'Accept-Language': 'zh-CN,zh;q=0.9',

'Referer': 'http: // weixin.sogou /',

'Cookie': '你的cookie'} # TODO 不可能把我的给你撒

session = Session()

queue = RedisQueue()

queue.del_all()

mysql = mysqlConn()

def start(self):

"""

初始化工作

:return:

"""

# 全局更新headers

# 如果你试过用这个方法修改headers，那么就知道这个在这里好像没什么用，我在这里浪费了至少两个小时

self.session.headers.update(self.headers)

start_url = self.base_url + urlencode(self.params)

# 这里我将need_proxy=False设为了False 即并没有使用代理 ps：我也就是测试一下

# 真正修改了headers是在这里

weixin_request = WeixinRequest(url=start_url, callback=self.parse_index, headers=self.headers, need_proxy=False)

# 调度第一个请求

self.queue.add(weixin_request)

def schedule(self):

"""

调度请求

:return:

"""

while not pty():

weixin_request = self.queue.pop()

callback = weixin_request.callback

print('Schedule', weixin_request.url)

response = quest(weixin_request)

if response and response.status_code in VALD_STATUES:

results = list(callback(response))

if results:

for result in results:

print('New Result', result)

if isinstance(result, WeixinRequest):

# 将新的文章详情的url也加入队列

self.queue.add(result)

if isinstance(result, dict):

# 储存到mysql