7个主要方法

Response对象的属性

r.encoding：如果header中不存在charset，则认为编码为ISO‐8859‐1
r.text根据r.encoding显示网页内容
r.apparent_encoding：根据网页内容分析出的编码方式
可以看作是r.encoding的备选

Requests库和Response的异常

r.raise_for_status()在方法内部判断r.status_code是否等于200，不需要
增加额外的if语句，该语句便于利用try‐except进行异常处理

通用代码框架

import requests

def getHTMLText(url):
    try:
        r = requests.get(url, timeout=30)
        r.raise_for_status() # 如果状态码不是200，则引发HTTPError异常
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return '产生异常'
if __name__ == '__main__':
    url = ''
    ...  

HTTP协议对资源的操作

PATCH和 PUT区别

假设URL位置有一组数据UserInfo，包括UserID、UserName等20个字段
需求：用户修改了UserName，其他不变
• 采用PATCH，仅向URL提交UserName的局部更新请求
• 采用PUT，必须将所有20个字段一并提交到URL，未提交字段被删除
PATCH的最主要好处：节省网络带宽

HTTP协议中的方法与Reqests库方法一一对应

7个方法详解

requests.request(method, url, **kwargs)

参数详解

kv = {'key1': 'value1', 'key2': 'value2'}
r = requests.request('GET', 'http://python123.io/ws', params=kv)
print(r.url)  # http://python123.io/ws?key1=value1&key2=value2

kv = {'key1': 'value1', 'key2': 'value2'}
r = requests.request('POST', 'http://python123.io/ws', data=kv)
body = '主体内容'
r = requests.request('POST', 'http://python123.io/ws', data=body)

kv = {'key1': 'value1'}
r = requests.request('POST', 'http://python123.io/ws', json=kv)

hd = {'user‐agent': 'Chrome/10'}
r = requests.request('POST', 'http://python123.io/ws', headers=hd)

# 向某个连接提交文件
fs = {'file': open('data.xls', 'rb')}
r = requests.request('POST', 'http://python123.io/ws', files=fs)

pxs = { 'http': 'http://user:pass@10.10.10.1:1234', 'https': 'https://10.10.10.1:4321' }
r = requests.request('GET', 'http://www.baidu.com', proxies=pxs)

requests.get(url, params=None, **kwargs)

import requests

# 发送get请求
r = requests.get('http://www/baidu.com/s', params={'kw': 'python'})
r = requests.get('http://www.baidu.com/s', params=dict(wd='python'))


# 下载图片
url = 'https://timgsa.baidu.com/timg?image&quality=80&size=b9999_10000&sec=1531368811695&di=6c119a485d39d482d54fcf31856b6de1&imgtype=0&src=http%3A%2F%2Fimg.shouyoutan.com%2FUploads-s%2Fnews%2F2016-04-11%2F570b63dbc401b.jpg'
r = requests.get(url)
with open('demo.jpg', 'wb') as f:
    f.write(r.content)

    
# 下载大文件
url = 'http://sqdownb.onlinedown.net/down/FastStoneCapture44264.zip'
filename = url.split('/')[-1]
r = requests.get(url, stream=True)
with open(filename, 'wb') as f:
    for chunk in r.iter_content(chunk_size=512):
        if chunk:
            f.write(chunk)

                        
# requests请求响应信息
import requests

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWe'
                 'bKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'
}

resp = requests.get('http://www.baidu.com/', headers=headers)


### 示例
# 获取响应中的头信息
print(resp.headers)

# 获取请求的头信息
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWe'
                 'bKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36'
}

resp = requests.get('http://www.baidu.com/', headers=headers)
# 获取响应中的头信息
print(resp.headers)

# 获取请求的头信息
print(resp.request.headers)
print(resp.text)  # 字符串格式
print(resp.content) # 二进制格式
print(resp.raw)  # 原始响应内容
print(resp.json) # json格式


# 重定向
r = requests.get('http://www.baidu.com/', allow_redirects=False) # 获得重定向之前的内容
r = requests.get('http://10.31.161.59:8888/admin', allow_redirect=True) # 获得重定向之后的内容


# 发送cookie
url = 'http://10.31.161.59:8888'

cookies = dict(
    test = 'testing'
)
resp = requests.get(url, cookies=cookies)

requests.head(url, **kwargs)

requests.post(url, data=None, json=None, **kwargs)

# 发送post请求
url = 'http://10.31.161.59:8888/user/register/'
r = requests.post(url, data=dict(username='python', password='123456'))
print(r.content
      
# 文件上传
url = ''
files = {
    'file': open('somefile.txt', 'rb')
}
r = requests.post(url, files=files)
      

# 发送json数据
import json
url = ''
user = {
    'username': 'gavin'
}
requests.post(url, data=json.dumps(user))
requests.post(url, json=user)

# 使用代理
proxies = {
    'http':'http://101.236.35.98:8866',
    'https':'https://180.121.132.184:808'
}

r = requests.get('http://www.baidu.com', proxies=proxies)
print(r.headers)

requests.put(url, data=None, **kwargs) 同Patch

requests.delete(url, **kwargs)

requests请求响应信息

import requests
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64)'
              ' AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
}

resp = requests.get("http://10.31.161.59:8888",headers=headers)

# 获得响应中的头信息
print(resp.headers)

# 获得请求的头信息
print(resp.request.headers)

# 获得响应的内容
print(resp.text)  # 字符串格式
print(resp.content) # bytes格式
print(resp.raw)   # 原始响应内容
print(resp.json())  # json格式

requests超时时间

import requests

url = 'http://www.baidu.com'
try:
    r = requests.get(url=url,timeout=1)
except Exception as e:
    print("{url}请求超时.".format(url=url))

print(type(r.request))

页面下载及解决编码问题

              decode                    encode
str(byte类型) ---------> str(Unicode) ---------> str(byte类型)


import requests
import codecs
# pip install requests

def fetch(url):
    """
    获取url对应的页面的内容.

    参数:
        -url- : 网页的地址
    返回值:
        如果页面成功获取,以r.content二进制内容返回，有2个方式保存为文件，直接以wb方式写入，或者先将二进制数据decode成str，然后指定编码格式 encoding='GBK',然后写入文件。
        如果页面以r.text返回，需要和通用代码框架中一样设定 r.encoding = 'gbk'或 r.encoding = r.apparent_encoding，然后直接写入文件即可。
    """
    r= requests.get(url)
    if r.status_code is 200:
        return r.content
    return None



def main():
    result = fetch("https://www.poxiao.com/")
    
    if result is not None:
        with open("poxiao.html","wb") as f:
            f.write(result)
    return None
    # if result is not None:
    #     with codecs.open("poxiao.html","w",encoding='GBK') as f:
    #         f.write(result.decode('GBK'))
    # return None

if __name__ == '__main__':
    main()

保存数据到数据库

官方版本

import pymysql
import pymysql.cursors

from bs4 import BeautifulSoup
with open('poxiao.html','rb') as f:
    # pip install lxml
    poxiao  = BeautifulSoup(f,'lxml')


#  div   content clear
# 找到div 
content_div = poxiao.find(name='div',class_='content clear')

# 从div中找所有li
li_list = content_div.findAll(name='li')

# Connect to the database
connection = pymysql.connect(host='localhost',
                             user= 'spider',
                             password='123456',
                             db='spider',
                             charset='utf8mb4',
                             cursorclass=pymysql.cursors.DictCursor)
# 获得每个li中的文本和超链接
try:
    for li in li_list:
        print(li.a.get("href"))
        print(li.get_text())
        href = li.a.get("href")
        title = li.get_text()
        with connection.cursor() as cursor:
            # Create a new record
            sql = "INSERT INTO `movie` (`title`, `link`) VALUES (%s, %s)"
            cursor.execute(sql, (title, href))

        # connection is not autocommit by default. So you must commit to save
        # your changes.
        connection.commit()

        with connection.cursor() as cursor:
            # Read a single record
            sql = "SELECT `title`, `link` FROM `movie` WHERE `movie_id`=%s"
            cursor.execute(sql, (1,))
            result = cursor.fetchone()
            print(result)
finally:
    connection.close()

简写版本

import pymysql
from bs4 import BeautifulSoup

with open('poxiao.html', 'rb') as f:
    poxiao = BeautifulSoup(f, 'lxml')

content_div = poxiao.find(name='div', class_='content clear')

li_list = content_div.findAll(name='li')


db = pymysql.connect('127.0.0.1', 'root', '123456', 'spider')
db.set_charset('utf8')

cursor = db.cursor()

try:
    for li in li_list:
        href = 'https://www.poxiao.com' + li.a.get('href')
        title = li.get_text()
        # sql = 'insert into movie values(null, "%s", "%s")' % (title, href)
        sql = "INSERT INTO `movie` (`title`, `link`) VALUES (%s, %s)"
        cursor.execute(sql, (title, href))
        db.commit()

except:
    db.rollback()
db.close()

requests会话

import requests

s = requests.Session()

# 地址级别的参数可以跨请求保持
s.get('http://httpbin.org/cookies/set/sessioncookie/123456789')
r = s.get("http://httpbin.org/cookies")

print(r.text)

# 函数级别的参数不会被跨请求保持
resp = s.get('http://httpbin.org/cookies', cookies={'name':'xiaoming'})
print(resp.text)

resp = s.get('http://httpbin.org/cookies')
print(resp.text)


# 会话可以用来提供默认数据

session = requests.Session()
session.auth = ('dai','123456')

# 会话也可用来为请求方法提供缺省数据。这是通过为会话对象的属性提供数据来实现的：
s = requests.Session()
s.auth = ('user', 'pass')
s.headers.update({'x-test': 'true'})

# both 'x-test' and 'x-test2' are sent
r= s.get('http://httpbin.org/headers', headers={'x-test2': 'true'})
print(r.request.headers)

# 在 with 前后文管理器中使用session，确保session自动关闭
with requests.Session as session:
    session.get('http://www.baidu.com')

预请求

from requests import Request, Session

session = Session()
url = 'http://www.baidu.com'

# 构建一个请求对象
req = Request('GET', url, headers=None)

# 获得带有状态的预请求对象
prepped = session.prepare_request(req)
print(type(prepped))
# <class 'requests.models.PreparedRequest'>

# 发送请求
with session.send(prepped, timeout=0.5) as response:
    print(response.status_code)

requests下载图片

import requests
url = "http://pic171.nipic.com/file/20180705/5053868_230805401034_2.jpg"
r = requests.get(url)
with open('demo.jpg','wb') as f:
    f.write(r.content)

requests下载大文件

url = 'http://sqdownb.onlinedown.net/down/FastStoneCapture44264.zip'
import requests
filename = url.split('/')[-1]
r = requests.get(url,stream=True)
with open(filename,'wb') as f:
    for chunk in r.iter_content(chunk_size=512):
        if chunk:
            f.write(chunk)

requests发送cookie

import requests

url = 'http://10.31.161.59:8888/'
headers = {
    'User-Agent':'afasdfasfasdf',
}
cookies = dict(
    sessionid = 'swkbut2xrt2c3rhhwubamp80oa2p7tzw',
    csrftoken = 'e6JLs6n2u4gPDrd8l3XVkUhhvCldxUvXhJ0SHJGiV1oUbnTwv12CR9GTvzX6l9EC'
)
resp = requests.get(url,headers=headers,cookies=cookies)
print(resp.text)

requests文件上传

import requests

url = ''
files = {
    'file':open('somefile.txt','rb')
}

r = requests.post(url,files=files)