自学教程：python爬虫之pyppeteer库简单使用

# puppeteer支支持字典传参browser = await launch({'headless':True})# pyppeteer支持字典和关键字传参browser = await launch({'headless':True})browser = await launch(headless=True)

# puppeteer使用$符page.$()/page.%%()/page.$x()

# pyppeteer使用python风格的函数名page.querySelector()/page.querySelectorAll()/page.xpath()# 简写方式page.J()/page.JJ()/page.Jx()

content = await page.evaluate('document.body.textContent',force_expr=True)

element = await page.querySelector('h1')title = await page.evaluate('(element) => element.textContent',element)

pip install pyppeteer

pyppeteer-install

import asynciofrom pyppeteer import launchasync def main():    url = 'https://www.toutiao.com/'    # headless参数设置为Falase，则变成有头模式    browser = await launch(headless=False, ignoreDefaultArgs=['--enable-automation'])    page = await browser.newPage()        # 设置页面视图大小    await page.setViewport(viewport={'width':1600,'herght':900})        # 是否启用JS，enabled设为False，则无渲染效果    await page.setJavaScriptEnable(enabled=True)        # 等待时间1000毫秒    res = await page.goto(url，options={'timeout':1000})    resp_headers = res.headers  # 响应头    resp_status = res.status    # 响应状态        # 等待    await asyncio.sleep(2)    await page.waitFor(1000)    # 第二种方法 ，在while循环里强行查询某元素进行等待    while not await page.querySelector('.t')        # 滚动到页面底部    await page.evaluate('window.scrollBy(0,document.body.scrollHeight)')        await page.screenshot({'path':'test.png'})        # 打印网页cookies    print(await page.cookies())        # 获取所有html内容    print(await page.content())            dimensions = await page.evaluate(pageFunction='''() => {    		return {    			width:document.documentElement.clentWidth,    // 页面宽度    			height:document.documentElement.clentHeight,  // 页面高度    			deviceScaleFactor: window.devicePixelRatio,  // 像素比1.0000000149011612    			}    		}''',force_expr=False)   # force_expr=False  执行的是函数    print(dimensions)        content = await page.evaluate(pageFunction='document.body.textContent',force_expr=True)    # 只获得文本 执行js脚本，force_expr=True  执行的是表达式    print(content)        # 打印当前页面的标题    print(await page.title())            # 抓取新闻内容  可以使用xpath表达式    '''    pyppeteer 三种解析方式    page.querySelector()    page.querySelectorAll()    page.xpath()    简写方式为：    page.J()    page.JJ()    page.Jx()    '''    element = await page.querySelector(".feed-infinite-wrapper > ul>li")    print(element)            element = await page.querySelectorAll(".title-box a")    for item in element:        print(await item.getProperty('textContent'))        # 获取文本内容        title_str = await (await item.getProperty('textContent')).jsonValue()                title_link = await (await item.getProperty('textContent')).jsonValue()                # 获取属性值        # title = await (await item.getProperty('class')).jsonValue()        print(title_str,title_link)    await browser.close()asyncio.get_event_loop().run_until_complete(main())

# 模拟输入账号密码 参数{'delay':reand_int()}  延迟输入时间await page.type('#kw',"百度",delay=100)await page.type('#TPL_username_1',"asdasd")await page.waitFor(1000)await page.click('#su')

browser = await launch(headless=False, ignoreDefaultArgs=['--enable-automation'])# 添加ignoreDefaultArgs=['--enable-automation'] 参数

from bs4 import BeautifulSoupfrom pyppeteer import launchimport asynciodef screen_size():    return 1600,900async def main(url):    browser = await launch({"args":['--no-sandbox'],}) # "headless":False    page = await browser.newPage()    width, height = screen_size()    await page.setViewport(viewport={'width':width,'height':height})    await page.setJavaScriptEnabled(enabled=True)    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36')    await page.goto(url)    await page.evaluate('window.scrollBy(0, document.body.scrollHeight)')    await asyncio.sleep(1)    # content = await page.content()    li_list = await page.xpath('//*[@id="J_goodsList"]/ul/li')    item_list = []    for li in li_list:        a = await li.xpath('.//div[@class="p-img"]/a')        detail_url = await (await a[0].getProperty('href')).jsonValue()        promo_words = await (await a[0].getProperty('title')).jsonValue()        a_ = await li.xpath('.//div[@class="p-commit"]/strong/a')        p_commit = await (await a_[0].getProperty('textContent')).jsonValue()        i = await li.xpath('./div/div[3]/strong/i')        price = await (await i[0].getProperty('textContent')).jsonValue()        em = await li.xpath('./div/div[4]/a/em')        title = await (await em[0].getProperty('textContent')).jsonValue()        item = {            "title" : title,            "detail_url" : detail_url,            "promp_words" : promo_words,            "p_commit" : p_commit,            "price" : price        }        item_list.append(item)    await page_close(browser)    return item_listasync def page_close(browser):    for _page in await browser.pages():        await _page.close()    await browser.close()url = 'https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&wq='/		'%E6%89%8B%E6%9C%BA&pvid=e07184578b8442c58ddd65b221020e99&page={}&s=56&click=0 'task_list = []for i in range(1,4):    page = i * 2 - 1    task_list.append(main(url.format(page)))results = asyncio.get_event_loop().run_until_complete(asyncio.gather(*task_list))for i in results:    print(i,len(i))print('*'*100)