爬取 豆瓣电影我不是药师 评论分析
#--*--coding:utf-8--*--
import requests
import json
import os
import sys
import random
import time
proxies = {
"http": "proxy.xx.com:911",
"https": "proxy.xx.com:911",
}
#url=r"https://movie.douban.com/subject/26752088/comments?status=P"
"""
https://movie.douban.com/subject/26752088/comments?start=0&limit=20&sort=new_score&status=P&percent_type=
https://movie.douban.com/subject/26752088/comments?start=40&limit=20&sort=new_score&status=P&percent_type=
https://movie.douban.com/subject/26752088/comments?start=60&limit=20&sort=new_score&status=P&percent_type=
"""
u1=r"https://movie.douban.com/subject/26752088/comments?start="
u2=r"&limit=20&sort=new_score&status=P&percent_type="
for i in range(20,1320,20):
url=u1+str(i)+u2
print(url)
r=requests.get(url,proxies=proxies)
print(r.status_code)
html=str(r.content, encoding = "utf_8_sig")
print(html)
f = open('c:/users/ffan2/desktop/wbsys_pl.txt','a',encoding='utf_8_sig')
f.write(html)
# coding: utf-8# In[1]:# -*- coding: utf-8 -*-import requestsimport jsonimport osimport sysimport randomimport timeimport reimport numpy as npimport pandas as pdfrom pandas import DataFrameimport matplotlib.pyplot as plt#df=pd.DataFrame(pd.read_csv('c:\\users\\ffan2\\desktop\\jd_data.csv',encoding='UTF_8'))# In[3]:html=open('c:\\users\\ffan2\\desktop\\wbsys.txt','r',encoding = "UTF_8").read()# In[158]:votes_0=re.findall(r'.*?',html)votes_1=re.sub(r'.*?','',str(votes_0))
votes=re.sub(r'',r'',str(votes_1))votes # get all vote numberslen(votes)# In[185]:comments_0=re.findall(r'
.*',html)
comments=re.sub(r'
.*?','',str(comments_0))#comments# In[165]:comments=re.sub(r'','',str(comments))len(comments)# In[166]:comment_time_0=re.findall(r'\'',r'',str(comment_time_2))#comment_time# In[167]:user_name_0=re.findall(r'','',str(user_name_0))
user_name=re.sub('<','',str(user_name_1))
#user_name
# In[154]:
a={'user_name':user_name,'comment_time':comment_time,'votes':votes,'comments':comments}
print(len(a['comments']))
print(len(a['votes']))
print(len(a['comment_time']))
print(len(a['user_name']))
# In[173]:
table=pd.DataFrame(a[comments])
#table.to_csv('c:\\users\\ffan2\\desktop\\wsys.csv',encoding = "utf_8_sig")
#df=pd.DataFrame(pd.read_csv('c:\\users\\ffan2\\desktop\\wsys.csv',encoding='UTF_8_sig'))
#print(table.shape)
#使用jieba 分词 分析rank~
word_str=''.join(comments)
word_rank=jieba.analyse.extract_tags(word_str, topK=20, withWeight=True, allowPOS=())
print(word_rank)
word_rank = pd.DataFrame(word_rank,columns=['word','rank'])
word_rank.set_index('rank')
rank word
0.159256电影
0.063695徐峥
0.056682题材
0.052965煽情
0.046412现实
0.042310王传君
0.040851药神
0.033984一部
0.032249达拉斯
0.028569这部
0.026092片子
0.026046希望
0.025504导演
0.025464真实
0.025396故事
0.024536剧情
0.024384中国
0.024108社会
0.023750国产片
0.023353演员
