Python网络数据采集2-wikipedia
随机链接跳转
获取维基百科的词条超链接，并随机跳转。可能侧边栏和低栏会有其他链接。这不是我们想要的，所以定位到正文。正文在id为bodyContent的div标签里。
import random
import re
import requests
from bs4 import BeautifulSoup
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/52.0.2743.116 Safari/537.36 Edge/15.16193'}
start_url = '/wiki/Wiki'
def get_links(url):
r = requests.get('https://en.wikipedia.org' + url, headers=headers)
soup = BeautifulSoup(r.text, 'lxml')
# /wiki/some_words
link_list = soup.find('div', id='bodyContent').find_all('a', href=re.compile(r'^/wiki/[^/]*$'))
return link_list
links = get_links(start_url)
while len(links) > 0:
# 随机选择一个链接
link = random.choice(links).get('href')
print(link)
# 新的词条覆盖了原来的超链接，一直搜寻
links = get_links(link)
/wiki/Personal_wiki
/wiki/Database_management_system
/wiki/Netezza
/wiki/C%2B%2B
/wiki/C%2B%2B#Standardization
/wiki/ISO_9984
/wiki/Georgian_script
...
从首页开始，将首页的所有词条放入集合中（去重），再遍历集合，从集合中的链接递归搜索。
import re
import requests
from bs4 import BeautifulSoup
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/52.0.2743.116 Safari/537.36 Edge/15.16193'}
pages = set()
def get_links(url):
global pages
r = requests.get('https://en.wikipedia.org' + url, headers=headers)
soup = BeautifulSoup(r.text, 'lxml')
# /wiki/some_words
link_list = soup.find('div', id='bodyContent').find_all('a', href=re.compile(r'^/wiki/[^:/]*$'))
for link in link_list:
if link['href'] not in pages:
new_page = link['href']
pages.add(new_page)
print(new_page)
get_links(new_page)
if __name__ == '__main__':
# 空字符串表示，url为wiki主页https://en.wikipedia.org
get_links('')
获取词条的标题、正文
标题在h1标签中，正文在id为mw-content-text的div标签中。
import re
import requests
from bs4 import BeautifulSoup
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/52.0.2743.116 Safari/537.36 Edge/15.16193'}
pages = set()
def get_links(url):
global pages
r = requests.get('https://en.wikipedia.org' + url, headers=headers)
soup = BeautifulSoup(r.text, 'lxml')
# /wiki/some_words
try:
print(soup.h1.string)
# 只打印第一段
print(soup.find(id='mw-content-text').find('p').text)
except AttributeError:
print('页面缺少一些属性。')
link_list = soup.find('div', id='bodyContent').find_all('a', href=re.compile(r'^/wiki/[^:/]*$'))
for link in link_list:
if link['href'] not in pages:
new_page = link['href']
pages.add(new_page)
print('----------\n' + new_page)
get_links(new_page)
if __name__ == '__main__':
# 空字符串表示，url为wiki主页https://en.wikipedia.org
get_links('')
Main Page
Noye's Fludde is a one-act opera written largely for young amateur performers, created by the British composer Benjamin Britten. First performed in 1958 at the annual Aldeburgh Festival, it is based on the 15th-century Chester "mystery" play which recounts the biblical story of Noah, the flood and the ark. Britten had written numerous works for mixed
...
--------
/wiki/Wikipedia
Wikipedia
Wikipedia (/ˌwɪkᵻˈpiːdiə/ ( listen) or /ˌwɪkiˈpiːdiə/ ( listen) WIK-i-PEE-dee-ə) is a free online encyclopedia with the aim to allow anyone to edit articles.[3] Wikipedia is the largest and most popular general reference work on the Internet[4][5][6] and is ranked among the ten most popular websites.[7] Wikipedia is owned by the nonprofit Wikimedia Foundation.[8][9][10]
--------
/wiki/Main_Page
...
寻找外链
从https://www.oreilly.com开始不断寻找外链，如果某个页面没有外链，则进入该页面的某个内链，再重新找外链。感觉这个例子不是很好，因为从其他外链又可能回到初始页面。
import re
import random
import requests
from bs4 import BeautifulSoup
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/52.0.2743.116 Safari/537.36 Edge/15.16193'}
def get_random_external_link(start_page):
r = requests.get(start_page, headers=headers)
soup = BeautifulSoup(r.text, 'lxml')
# 返回分割地址的第一个元素，一般是主页的地址
ex_links = get_external_links(soup, split_address(start_page)[0])
# 如果该页面没有外链，则获取内链,再从内链里随机选取一个，递归，直到获取到外链为止。
if len(ex_links) == 0:
internal_links = get_internal_links(soup, split_address(start_page)[0])
return get_random_external_link(random.choice(internal_links))
else:
return random.choice(ex_links)
def get_internal_links(bs, include_url):
internal_links = []
# 找出所有以为'/'开头的链接，此为内链
in_links = bs.find_all('a', href=re.compile(r'^/|' + include_url))
for link in in_links:
if link['href'] not in internal_links:
internal_links.append(link['href'])
return internal_links
def get_external_links(bs, exclude_url):
external_links = []
# 找出所有以http、https开头的链接，且不含内链字符的，此为外链,(?!...)表示不包含
ex_links = bs.find_all('a', href=re.compile(r'^(https|http)((?!' + exclude_url + ').)*$'))
for link in ex_links:
if link['href'] not in external_links:
external_links.append(link['href'])
return external_links
def split_address(address):
address_parts = []
if address.split(':')[0] == 'http':
address_parts = address.replace('http://', '').split('/')
elif address.split(':')[0] == 'https':
address_parts = address.replace('https://', '').split('/')
return address_parts
# 只搜索外链
def follow_external_only(url):
external_link = get_random_external_link(url)
print(external_link)
follow_external_only(external_link)
all_ex_links = set()
all_in_links = set()
# 获得所有外链和内链，并打印了外链
def get_all_external_links(url):
r = requests.get(url, headers=headers)
soup = BeautifulSoup(r.text, 'lxml')
internal_links = get_internal_links(soup, split_address(url)[0])
external_links = get_external_links(soup, split_address(url)[0])
for link in external_links:
if link not in all_ex_links:
all_ex_links.add(link)
print(link)
for link in internal_links:
if link not in all_in_links:
all_in_links.add(link)
get_all_external_links(link)
if __name__ == '__main__':
# follow_external_only('https://www.oreilly.com')
get_all_external_links('https://www.oreilly.com')
https://www.safaribooksonline.com/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170601+nav
http://shop.oreilly.com/
http://members.oreilly.com
https://www.safaribooksonline.com/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170505+homepage+get+started+now
https://www.safaribooksonline.com/accounts/login/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170203+homepage+sign+in
https://www.safaribooksonline.com/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170710+homepage+get+started+now
https://www.safaribooksonline.com/public/free-trial/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170710+homepage+start+free+trial
https://www.safaribooksonline.com/accounts/login/?utm_medium=content&utm_source=oreilly.com&utm_campaign=lgen&utm_content=20170710+homepage+sign+in
...
上面的代码经常会出错，可能是正则表达式匹配的原因，也有可能是网络原因。
by @sunhaiyu
2017.7.14
