-
下面是抓取信息的函数
def get_item_info(url):
web_data = requests.get(url)
soup = BeautifulSoup(web_data.text, 'lxml')
title = soup.select('h1.title-name')[0].text if soup.find_all('h1',{'class':'title-name'}) else None
if title == None:
pass
else:
data = {'time':list(soup.select('i.pr-5')[0].stripped_strings)[0].split()[0] if soup.find('i',{'class':'pr-5'}) else None,
'type':soup.select('#wrapper > div.content.clearfix > div.leftBox > div:nth-of-type(3) > div > ul > li:nth-of-type(1) > span > a')[0].text if soup.find_all('ul',{'class':'det-infor'}) else None,
'price':soup.select('i.f22.fc-orange.f-type')[0].text if soup.find_all('i',{'class':'f22 fc-orange f-type'}) else None,
'address':list(map(lambda x:x.text,soup.select('#wrapper > div.content.clearfix > div.leftBox > div > div > ul > li:nth-of-type(3) > a'))) if soup.find_all('li') else None,
'old_new':list(soup.select('#wrapper > div.content.clearfix > div.leftBox > div:nth-of-type(4) > div.det-summary > div > div.second-dt-bewrite > ul > li:nth-of-type(1)')[0].stripped_strings) if soup.select('#wrapper > div.content.clearfix > div.leftBox > div:nth-of-type(4) > div.det-summary > div > div.second-dt-bewrite > ul > li:nth-of-type(1)') else None }
#ul.det-infor > li:nth-of-type(1) > span 这个selector好像不能取下新旧程度
item_info.insert_one(data)
print(data)
-
下面是调用上面函数的代码
from get_third_url import get_item_info
from get_third_url import whole_third_url#whole_third_url是存储链接的
from multiprocessing import Pool
import requests
if __name__ == '__main__':
pool = Pool()
try:
pool.map(get_item_info,whole_third_url.find(['url']))
except requests.exceptions.InvalidSchema:
pass
然后跑出来这么一个错误
Traceback (most recent call last):
File "/Users/wangpegnfei/Desktop/Plan-for-combating-master/week2/week2_homework/myself/action2.py", line 10, in <module>
pool.map(get_item_info,whole_third_url.find(['url']))
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/pymongo/collection.py", line 1137, in find
return Cursor(self, *args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/pymongo/cursor.py", line 121, in __init__
validate_is_mapping("filter", spec)
File "/Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages/pymongo/common.py", line 375, in validate_is_mapping
"collections.Mapping" % (option,))
TypeError: filter must be an instance of dict, bson.son.SON, or other type that inherits from collections.Mapping
不知道是哪里出错了，错误类型：过滤器必须是字典的一个实例
2016.5.20，更新一下，上面代码有点问题，代码有所更改
-
下面是抓取信息的函数
def get_item_info(url, data=None):
wb_data = requests.get(url, headers=headers)
# 检查页面是否不存在，或者被封ip
if wb_data.status_code != 200:
return
soup = BeautifulSoup(wb_data.text, 'lxml')
prices = soup.select('.f22.fc-orange.f-type')
pub_dates = soup.select('.pr-5')
areas = soup.select('ul.det-infor > li:nth-of-type(3) > a')
cates = soup.select('ul.det-infor > li:nth-of-type(1) > span')
print(areas)
data = {'title': soup.title.text.strip(), 'price': prices[0].text.strip() if len(prices) > 0 else 0, 'pub_date': pub_dates[0].text.strip().split(' ')[0] if len(pub_dates) > 0 else "", 'area': [area.text.strip() for area in areas if area.text.strip() != "-"], 'cates': [cate.text.strip() for cate in cates], 'state': soup.select('ul.second-det-infor.clearfix > li')[0].text.split('：')[-1].strip()
if soup.find('ul','second-det-infor') and soup.select('ul.second-det-infor.clearfix > li')[0].text.split('：')[0].strip() == '新旧程度' else None, 'url': url }
print(data)
item_info.insert_one(data)
-
下面是调用上面函数的代码
from get_third_url import get_item_info
from get_third_url import whole_third_url
from multiprocessing import Pool
if __name__ == '__main__':
pool = Pool()
for i in whole_third_url.find():
#print(i['url'])
pool.map(get_item_info,i['url'].split())
