自学教程：Python中常见的反爬机制及其破解方法总结

import requestsfor page in range(1, 6):    response = requests.get(        'http://api.tianapi.com/topnews/index',        params={            'key': 'd5eace66dccd771e36767ce3563efa09',            'page': page,            'num': 20,            'word': '华为',            'src': '人民日报'        }    )    result = response.json()    for news in result['newslist']:        print(news['title'])        print(news['url'])

import easyocrreader = easyocr.Reader(['ch_sim', 'en'], gpu=False)print(reader.readtext('./files/captcha.jpg', detail=0))

import ioimport easyocrfrom PIL import Imagefrom selenium import webdriverfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support import expected_conditionsfrom selenium.webdriver.support.wait import WebDriverWaitbrowser = webdriver.Chrome()browser.set_window_size(1280, 960)browser.get('http://mail.1000phone.com/')# 隐式等待（下面的方法在工作时如果取不到就等10秒）browser.implicitly_wait(10)# 显式等待wait = WebDriverWait(browser, 10)wait.until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR, '.login_panel_iframe')))iframe1 = browser.find_element_by_css_selector('.login_panel_iframe')# 记录iframe1的位置（相对位置）x1, y1 = iframe1.location['x'], iframe1.location['y']# Chrome对象的switch_to属性的frame方法，可以从页面切换到iframe中browser.switch_to.frame(iframe1)iframe2 = browser.find_element_by_css_selector('#ding-login-iframe')x2, y2 = iframe2.location['x'], iframe2.location['y']browser.switch_to.frame(iframe2)username_input = browser.find_element_by_css_selector('#username')# 模拟用户输入username_input.send_keys('xx@1000phone.com')password_input = browser.find_element_by_css_selector('#password')password_input.send_keys('xxxxx!!')# 创建一个等待对象wait = WebDriverWait(browser, 10)wait.until(expected_conditions.element_to_be_clickable((By.CSS_SELECTOR, '#login_checkcode_ico')))captcha_img = browser.find_element_by_css_selector('#login_checkcode_ico')# WebElement对象的size属性代表元素宽度和高度，location属性代表元素在窗口中的位置size, location = captcha_img.size, captcha_img.locationx3, y3, width, height = location['x'], location['y'], size['width'], size['height']# 截取整个浏览器窗口的图片获得图片的二进制数据image_data = browser.get_screenshot_as_png()# bytes（只读字节串） ----> io.BytesIO（可写字节串）---> getvalue() ---> bytes# str（只读字符串） ----> io.StringIO（可写字符串）---> getvalue() ---> strbrowser_image = Image.open(io.BytesIO(image_data))# 从截图上剪裁出验证码的图片x, y = x1 + x2 + x3, y1 + y2 + y3# Windows系统的写法 ---> 如果截图有问题就把坐标写死# print(x, y, width, height)checkcode_image = browser_image.crop((x * 1.25, y * 1.25, (x + width) * 1.25, (y + height) * 1.25))# macOS系统的写法# checkcode_image = browser_image.crop((x * 2, y * 2, (x + width) * 2, (y + height) * 2))checkcode_image.save('result.png')# 通过easyocr做光学文字识别reader = easyocr.Reader(['en'], gpu=False)code = reader.readtext('result.png', detail=0)[0]# 将识别出的验证码输入文本框checkcode_input = browser.find_element_by_css_selector('#login_checkcode')checkcode_input.send_keys(code)login_button = browser.find_element_by_css_selector('#login_submit_btn')# 模拟用户点击login_button.click()

"""Pillow库 ---> PIL ---> Python Image Library"""from PIL import Image, ImageFilter# 加载图像guido_image = Image.open('guido.jpg')# 剪裁guido_image.crop((80, 40, 310, 350)).show()# 滤镜guido_image.filter(ImageFilter.CONTOUR).show()# 缩略图guido_image.thumbnail((125, 185))# 显示图像guido_image.show()

from hashlib import md5import requestsclass ChaojiyingClient:    def __init__(self, username, password, soft_id):        self.username = username        password = password.encode('utf8')        self.password = md5(password).hexdigest()        self.soft_id = soft_id        self.base_params = {            'user': self.username,            'pass2': self.password,            'softid': self.soft_id,        }        self.headers = {            'Connection': 'Keep-Alive',            'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)',        }    def post_pic(self, image_data, code_type):        """        image_data: 图片字节        code_type: 验证码类型 参考 http://www.chaojiying.com/price.html        """        params = {            'codetype': code_type,        }        params.update(self.base_params)        files = {'userfile': ('ccc.jpg', image_data)}        response = requests.post(            url='http://upload.chaojiying.net/Upload/Processing.php',            data=params,            files=files,            headers=self.headers        )        return response.json()    # 超级鹰错误反馈函数（仅用于给超级鹰平台反馈）    def report_error(self, im_id):        """        im_id:报错题目的图片ID        """        params = {            'id': im_id,        }        params.update(self.base_params)        r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers)        return r.json()if __name__ == '__main__':    chaojiying = ChaojiyingClient('账户', '密码x', 'ID')  # 用户中心>>软件ID 生成一个替换 96001    with open('img.png', 'rb') as file:        image_data = file.read()  # 本地图片文件路径 来替换 a.jpg 有时WIN系统须要//        print(chaojiying.post_pic(image_data, 1902))  # 1902 验证码类型  官方网站>>价格体系 3.4+版 print 后要加()

import ioimport easyocrfrom PIL import Imagefrom selenium import webdriverfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support import expected_conditionsfrom selenium.webdriver.support.wait import WebDriverWaitfrom chaojiying import ChaojiyingClientbrowser = webdriver.Chrome()browser.set_window_size(1280, 960)browser.get('http://mail.1000phone.com/')# 隐式等待（下面的方法在工作时如果取不到就等10秒）browser.implicitly_wait(10)# 显式等待wait = WebDriverWait(browser, 10)wait.until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR, '.login_panel_iframe')))iframe1 = browser.find_element_by_css_selector('.login_panel_iframe')# 记录iframe1的位置（相对位置）x1, y1 = iframe1.location['x'], iframe1.location['y']# Chrome对象的switch_to属性的frame方法，可以从页面切换到iframe中browser.switch_to.frame(iframe1)iframe2 = browser.find_element_by_css_selector('#ding-login-iframe')x2, y2 = iframe2.location['x'], iframe2.location['y']browser.switch_to.frame(iframe2)username_input = browser.find_element_by_css_selector('#username')# 模拟用户输入username_input.send_keys('xxxx.com')password_input = browser.find_element_by_css_selector('#password')password_input.send_keys('xxxx!!')# 创建一个等待对象wait = WebDriverWait(browser, 10)wait.until(expected_conditions.element_to_be_clickable((By.CSS_SELECTOR, '#login_checkcode_ico')))captcha_img = browser.find_element_by_css_selector('#login_checkcode_ico')# WebElement对象的size属性代表元素宽度和高度，location属性代表元素在窗口中的位置size, location = captcha_img.size, captcha_img.locationx3, y3, width, height = location['x'], location['y'], size['width'], size['height']# 截取整个浏览器窗口的图片获得图片的二进制数据image_data = browser.get_screenshot_as_png()# bytes（只读字节串） ----> io.BytesIO（可写字节串）---> getvalue() ---> bytes# str（只读字符串） ----> io.StringIO（可写字符串）---> getvalue() ---> strbrowser_image = Image.open(io.BytesIO(image_data))# 从截图上剪裁出验证码的图片x, y = x1 + x2 + x3, y1 + y2 + y3# Windows系统的写法 ---> 如果截图有问题就把坐标写死# print(x, y, width, height)checkcode_image = browser_image.crop((x * 1.25, y * 1.25, (x + width) * 1.25, (y + height) * 1.25))# macOS系统的写法# checkcode_image = browser_image.crop((x * 2, y * 2, (x + width) * 2, (y + height) * 2))checkcode_image.save('result.png')# 通过超级鹰打码平台打码chaojiying = ChaojiyingClient('账户', '密码', 'ID')with open('result.png', 'rb') as file:    image_data = file.read()    result_dict = chaojiying.post_pic(image_data, 1902)# 将识别出的验证码输入文本框checkcode_input = browser.find_element_by_css_selector('#login_checkcode')checkcode_input.send_keys(result_dict['pic_str'])login_button = browser.find_element_by_css_selector('#login_submit_btn')# 模拟用户点击login_button.click()

import reimport bs4import requestspattern = re.compile(r'/d{4,6}')resp = requests.get('https://www.yinsiduanxin.com/china-phone-number/verification-code-16521686439.html')soup = bs4.BeautifulSoup(resp.text, 'html.parser')# print(resp.text)td = soup.select_one('body > div.container > div:nth-child(4) > div:nth-child(3) > div.main > div.layui-row > table > tbody > tr:nth-child(1) > td:nth-child(2)')results = pattern.findall(td.text)print(results[0])