用所学的前两节内容卸了一个简单的爬虫功能，爬取猫眼电影中的电影名字，导演，封面，类型信息
思路分析：
先看一下http://maoyan.com/robots
http://maoyan.com/films 我们要抓取的信息在这个页面，我们可以通过该页面的下方的页码元素，使用firebug查看将所有页码中包含的电影链接取出，通过链接进入电影详情页面抓取我们需要的信息，我抓取了电影名字，导演，类型，封面路径信息并保存到csv中
需要使用的模块及文档说明：
- requests 模块
- lxml模块
- re模块
- cssselect模块
- csv 模块
下载页面:代码中已有注释
#!/usr/bin/env python
# -*-coding:utf-8 -*-
import requests
import lxml.html
import re
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
class Downloader:
def __init__(self, url, scrape_callback=None):
self.url = url
self.scrape_callback = scrape_callback
def get_download_url(self):
# '''获取每个电影详情页的url'''
url_list =[]
# for i in range(1,22803):
for i in range(1):
url = self.url+'?offset='+str(30*i)
response = requests.get(url)
if response.status_code == 200:
html = response.text
tree = lxml.html.fromstring(html)
# 使用xpath方式获取div下的所有a标签
ll = tree.xpath(u"//div/a")
# 遍历ll
for i in ll:
# 获取a标签的href信息
url_str = i.get('href')
# 判断url_str
if url_str is not None:
# 判断url_str是否有'films'中,有的话则是我们要找的内容
if re.search('films', url_str) is not None:
# 判断，不保存重复的url
if i.get('href') not in url_list:
url_list.append(i.get('href'))
return url_list
def download(self):
# ''''抓取url信息中的电影名称，等信息'''
# 获取所有详情页的url
url_list = self.get_download_url()
for url_num in url_list:
# 查看元素可知返回的是‘films/2333’只需把后缀的/和数字填入到默认url中就是具体详情页
urlpage = self.url+'/'+str(url_num).split('/')[-1]
# url请求
response = requests.get(urlpage)
# 判断，如果能正常相应则继续
if response.status_code == 200:
html = response.text
# 规范html
tree = lxml.html.fromstring(html)
# 判断scrape_callback，True则保存信息
if self.scrape_callback:
# 通过CSS选择器获取想要的内容
l_img = tree.cssselect('div.avater-shadow > img.avater')[0].get('src')
l_filmname = tree.cssselect('div.movie-brief-container > h3.name')[0].text_content()
l_type = tree.cssselect('div.movie-brief-container > ul> li.ellipsis')[0].text_content()
l_director = tree.cssselect('div.info > a.name')[0].text_content()
row = [l_filmname, l_director, l_type, l_img]
self.scrape_callback(row)
保存页面：讲抓取的信息保存到csv中
#!/usr/bin/env python
# -*-coding:utf-8 -*-
import csv
import downloader
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
class ScrapeCallback:
'''下载的信息保存到csv中'''
def __init__(self):
fileobj=open("maoyan.csv", 'w')
self.writer = csv.writer(fileobj)
self.films = ('film_name', 'director', 'film_type', 'film_cover')
self.writer.writerow(self.films)
def __call__(self, row):
self.writer.writerow(row)
if __name__ == '__main__':
url = 'http://maoyan.com/films'
d = downloader.Downloader(url, scrape_callback=ScrapeCallback())
d.download()
运行ScrapeCallback会得到maoyan.csv文件，用excel打开会出现乱码，须将utf-8转成ansi 相关链接
