python爬取58同城北京租房信息

python爬取58同城北京租房信息

第一次写 有很多毛病，没防封ip。。。没多线程 不过数据量不大凑活着用吧，代码如下：

from bs4 import BeautifulSoup

import requests

import time

import re

urls = ["{}/?PGTID=0d100000-000f-12db-512a-5598dd1c3104&ClickID=1".format(str(i)) for i in range(1,20)]

for url in urls:

headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537."}

response = (url,headers = headers)

web_data = ,"lxml")

allurl = web_data.select("body > wrap.clearfix > t.clearfix > div.listcon > div > dl > dd > h3 > a")

for i in allurl:

x = []

y = []

z = []

v = []

url = i.get("href")

#headers = {

# "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537."}

response1 = (url, headers=headers)

web_data1 = , "lxml")

prices = web_data1.select("body > div.main > div.house-info.white-block.clearfix > div.house-primary-content-wrap.fr > ul > li.house-primary-content-li.house-primary-content-fir.clearfix > div > i > em")

payformants = web_data1.select("body > div.main > div.house-info.white-block.clearfix > div.house-primary-content-wrap.fr > ul > li.house-primary-content-li.house-primary-content-fir.clearfix > div > span")

rooms = web_data1.select("body > div.main > div.house-info.white-block.clearfix > div.house-primary-content-wrap.fr > ul > li:nth-of-type(2) > div")

address = web_data1.select("body > div.main > div.house-info.white-block.clearfix > div.house-primary-content-wrap.fr > ul > li:nth-of-type(4) > div")

peizhis = web_data1.select("body > div.main > div.house-info.white-block.clearfix > div.house-primary-content-wrap.fr > ul > li.house-primary-content-li.clearfix.broker-config > div > span")

pat = repile("s")

for peizhi in peizhis:

x.append(re.sub(pat, "", _text()))

l = " ".join(x)

for price in prices:

y.append(re.sub(pat, "", _text()))

for room in rooms:

z.append(re.sub(pat, "", _text()))

for addres in address:

v.append(re.sub(pat, "", _text()))

p = " ".join(v)

data = {"配置": l, "价格": y[0], "房型": z[0], "地址": p}

print(data)