自学教程：python爬取网页版QQ空间,生成各类图表

pip install 包名 -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com

import timefrom selenium import webdriverfrom lxml import etree# 这里一定要设置编码格式，防止后面写入文件时报错friend = '1569339843'  # 朋友的QQ号，朋友的空间要求允许你能访问user = '783533896'  # 你的QQ号pw = '1323mkoNJI.@'  # 你的QQ密码# 获取浏览器驱动chrome_driver = 'chromedriver.exe'driver = webdriver.Chrome(executable_path=chrome_driver)# 浏览器窗口最大化driver.maximize_window()# 浏览器地址定向为qq登陆页面driver.get("http://i.qq.com")# 所以这里需要选中一下frame，否则找不到下面需要的网页元素driver.switch_to.frame("login_frame")time.sleep(3)# 自动点击账号登陆方式driver.find_element_by_id("switcher_plogin").click()time.sleep(3)# 账号输入框输入已知qq账号driver.find_element_by_id("u").send_keys(user)time.sleep(5)# 密码框输入已知密码driver.find_element_by_id("p").send_keys(pw)time.sleep(5)# 自动点击登陆按钮driver.find_element_by_id("login_button").click()time.sleep(5)# 让webdriver操纵当前页driver.switch_to.default_content()time.sleep(5)# 跳到说说的url, friend你可以任意改成你想访问的空间driver.get("http://user.qzone.qq.com/" + friend + "/311")time.sleep(5)next_num = 0  # 初始“下一页”的idwhile True:    # 下拉滚动条，使浏览器加载出动态加载的内容，    # 我这里是从1开始到6结束 分5 次加载完每页数据    for i in range(1, 6):        height = 20000 * i  # 每次滑动20000像素        strWord = "window.scrollBy(0," + str(height) + ")"        driver.execute_script(strWord)        time.sleep(4)    # 很多时候网页由多个<frame>或<iframe>组成，webdriver默认定位的是最外层的frame，    # 所以这里需要选中一下说说所在的frame，否则找不到下面需要的网页元素    driver.switch_to.frame("app_canvas_frame")    selector = etree.HTML(driver.page_source)    divs = selector.xpath('//*[@id="msgList"]/li/div[3]')    # 这里使用 a 表示内容可以连续不清空写入    with open('qq_word.txt', 'a', encoding="utf-8") as f:        for div in divs:            qq_name = div.xpath('./div[2]/a/text()')            qq_content = div.xpath('./div[2]/pre/text()')            qq_time = div.xpath('./div[4]/div[1]/span/a/text()')            qq_praise = div.xpath('./div[4]/div[2]/span/span/a[2]/text()')            qq_comment = div.xpath('./div[4]/div[2]/a[3]/text()')            qq_name = qq_name[0] if len(qq_name) > 0 else ''            qq_content = qq_content[0] if len(qq_content) > 0 else ''            qq_content = qq_content.replace('/n', ' ')            qq_time = qq_time[0] if len(qq_time) > 0 else ''            qq_praise = qq_praise[0] if len(qq_praise) > 0 else ''            qq_comment = qq_comment[0] if len(qq_comment) > 0 else ''            print(qq_name, qq_time, qq_content, qq_praise, qq_comment)            f.write(qq_content + "/n")    # 当已经到了尾页，“下一页”这个按钮就没有id了，可以结束了    if driver.page_source.find('pager_next_' + str(next_num)) == -1:        break    # 找到“下一页”的按钮，因为下一页的按钮是动态变化的，这里需要动态记录一下    driver.find_element_by_id('pager_next_' + str(next_num)).click()    # “下一页”的id    next_num += 1    # 因为在下一个循环里首先还要把页面下拉，所以要跳到外层的frame上    driver.switch_to.parent_frame()# 关闭浏览器driver.quit()

import pandas as pdfrom pyecharts.charts import Barfrom pyecharts.charts import Pieimport pyecharts.options as optsfrom pyecharts.charts import Lineimport redf_excel = pd.read_excel('qq_excel.xlsx')  # 默认读取sheet=0 Pandas DataFrame'def getTimeStr(row):    item = row['时间']    if pd.isnull(item) | pd.isna(item):        return    data = item.split('年')[0]    return data# 按年统计说说数量def readCount(result, row):    timeData = getTimeStr(row)    if timeData == None: return    if timeData in result.keys():        result[timeData] += 1    else:        result[timeData] = 1# 按年统计说说点赞数def readThumb(result, row):    item = row['赞']    if pd.isnull(item):        return    # data = re.match(r'赞/((/d+).*', item, re.M | re.I)    if len(item.split("(")) <= 1:        return    data = item.split("(")[1].split(")")[0]    timeData = getTimeStr(row)    if timeData == None: return    if timeData in result.keys():        result[timeData] += int(data)    else:        result[timeData] = int(data)# 按年统计说说评论数def readComment(result, row):    item = row['评论']    if pd.isnull(item):        return    # data = re.match(r'赞/((/d+).*', item, re.M | re.I)    if len(item.split("(")) <= 1:        return    data = item.split("(")[1].split(")")[0]    timeData = getTimeStr(row)    if timeData == None: return    if timeData in result.keys():        result[timeData] += int(data)    else:        result[timeData] = int(data)def readExcel(df_excel):    count = {}    result = {}    thumb = {}    comment = {}    for index, row in df_excel.iterrows():        readCount(count, row)        readThumb(thumb, row)        readComment(comment, row)    result['count'] = count    result['thumb'] = thumb    result['comment'] = comment    return resultdef getKeyAndVal(keyWord):    data = readExcel(df_excel).get(keyWord)    key = []    value = []    for item in data.keys():        key.append(item)        value.append(data[item])    key.reverse()    value.reverse()    return [key, value]# 统计每年发表说说次数柱状图def paintBar():    count = readExcel(df_excel).get('count')    # V1 版本开始支持链式调用    data = getKeyAndVal('count')    print(data[0])    d = (        Bar()            .add_xaxis(data[0])            .add_yaxis("每年发表说说总数", data[1])            .render("每年发表说说总数柱状图.html")    )paintBar()# 统计点赞和评论折线图def paintLine():    commentData = getKeyAndVal('comment')    thumbData = getKeyAndVal('thumb')    xaxis_data = commentData[0]    commentValue = commentData[1]    thumbValue = thumbData[1]    d = (        Line()            .add_xaxis(xaxis_data=xaxis_data)            .add_yaxis("每年评论数", y_axis=commentValue)            .add_yaxis("每年点赞数", y_axis=thumbValue)            .render("每年点赞和评论折现图.html")  # 输出图形    )paintLine()