python爬取豆瓣评论制作词云代码

一、爬取豆瓣热评

该程序进行爬取豆瓣热评，将爬取的评论(json文件)保存到与该python文件同一级目录下
注意需要下载这几个库:requests、lxml、json、time

import requestsfrom lxml import etreeimport jsonimport timeclass Spider(object):def __init__(self):#seif.ure=\'https://movie.douban.com/subject/23885074/reviews?start=0\'self.headers={\'User-Agent\':\'Mozilla/5.0(Windows NT6.1;Win64;x64)AppleWebKit/537.36(KHTML,like Gecko)Chrome/75.0.3700.100Safari/537.36\'}def get_data(self,url):response = requests.get(url,headers=self.headers).content.decode(\'utf-8\')page=etree.HTML(response)#xpath 对象#获取所有数据节点node_list = page.xpath(\'//div[@class=\"review-list  \"]/div\')for node in node_list:#作者author = node.xpath(\'.//header[@class=\"main-hd\"]//a[2]/text()\')[0]#评论text = node.xpath(\'string(.//div[@class=\"main-bd\"]//div[@class=\"short-content\"])\')items={\'author\':author,\'text\':text.strip()}#持久化存储with open(\'yewen.json\',\'a\',encoding=\'utf-8\') as f:f.write(json.dumps(items,ensure_ascii=False)+\'\\n\')def run(self):for i in range(1,47):url=\'https://movie.douban.com/subject/26885074/reviews?start{}\'.format(i*20)print(\'正在爬取第{}页\'.format(i))self.get_data(url)time.sleep(3)if __name__==\'__main__\':s=Spider()s.run()

二、制作词云

该程序将json中的数据进行处理，提取重要信息，并用wordcloud库制作词云图片，同样保存到与该python文件同一级目录下
注意需要下载这几个库:jieba、wordcloud、json

import jiebafrom wordcloud import WordCloudimport jsonf= open(\"yewen.json\", \"r\", encoding=\"utf-8\")data_list= f.readlines()str =\'\'for data in data_list:text= json.loads(data)[\'text\']str +=text#替换无关紧要的词语result_str = str.replace(\'展开\', \'\').replace(\'这篇\',\'\').replace(\'影评\',\'\').replace(\'电影\',\'\').replace(\'这部\', \'\').replace(\'可能\', \'\').replace(\'剧情\',\'\')cut_text = jieba.lcut(result_str)result = \" \".join(cut_text)wc = WordCloud(font_path=\'simhei.ttf\',background_color=\"white\",max_words=600,width=1000,height=1000,min_font_size=20,max_font_size=100,)#mast=plt.imreda(\'snake.jpg\')#背景图片wc.generate(result)#转化为词云的操作wc.to_file(\"text.jpg\")#保存f.close()