# -*- coding: utf-8 -*-"""Created on Wed May 26 17:53:13 2021@author: 19088"""import urllib.requestimport osimport pickleimport reimport randomimport sys#获取http代理class getHttpAgents: #初始化函数 def __init__(self): self.attArray=self.__loadAgentList() self.myagent="" #注意 返回对象未进行解码 def openUrl(self,url,istry=1): response="" ip="" if(0 != len(self.myagent.strip())): ip=self.myagent i=1 if not istry: i=99 while i<100: try: #print(self.attArray) if(0 == len(self.attArray) and 0==len(ip.strip())): req=urllib.request.Request(url) #设置访问头 req.add_header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36") response=urllib.request.urlopen(req) else: if(0 != len(self.attArray)): ip=random.choice(self.attArray) if(0 != len(self.myagent.strip())): ip=self.myagent print("以{}访问 {}".format(ip,url)) #设置代理 proxy={"http":ip} #print(proxy) #定义一个代理字段 proxy_support=urllib.request.ProxyHandler(proxy) #建立一个opener opener=urllib.request.build_opener(proxy_support) opener.addheaders=[("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36")] #urllib.request.install_opener(opener) #获得网页对象 response=opener.open(url) except: if not istry: print("{} 无法使用".format(ip)) else: print("第{}次尝试连接！".format(i)) else: break; finally: i+=1 if 11==i and istry: raise ValueError if not response: return html=response.read() #print(html) return html #检查代理池 去除掉不可用代理ip def checkMyIpPool(self): agentsResult=[] agentList=self.attArray for iter in agentList: ip=iter self.setMyIp(ip) b=self.__getMyIp() if not b: #代理不能用 #agentList.pop(-iter) pass else: agentsResult.append(ip) #print(b) #记录爬取过的可以使用的代理ip self.__writeAgentList(agentsResult) self.__setAgents(agentsResult) self.setMyIp("") #解析读取网页中所有的代理地址 def getAgents(self,html): #print(html) #匹配 ip地址 正则表达式 pattern = re.compile(r'(<td>)/s*((25[0-5]|2[0-4]/d|[0-1]/d/d|/d/d|/d)/.){3}(25[0-5]|2[0-4]/d|[0-1]/d/d|/d/d|/d)/s*</td>') ipList=[] ip=pattern.finditer(html) for ipiter in ip: ipText=ipiter.group() ipGroup=re.search(r"((25[0-5]|2[0-4]/d|[0-1]/d/d|/d/d|/d)/.){3}(25[0-5]|2[0-4]/d|[0-1]/d/d|/d/d|/d)", ipText) ipList.append(ipGroup.group()) #匹配 端口地址 正则表达式 portList=[] pattern = re.compile(r'(<td>)/s*/d+/s*</td>') port = pattern.finditer(html) for portiter in port: portText=portiter.group() portGroup=re.search(r"/d+", portText) portList.append(portGroup.group()) if(len(ipList) is not len(portList)): print("注意: ip和端口参数不匹配!") return ipDict=dict(zip(ipList,portList)) agentList=[] for key in ipDict: agentList.append(key+":"+ipDict.get(key)) agentsResult=[] for iter in agentList: ip=iter self.setMyIp(ip) b=self.__getMyIp() if not b: #代理不能用 pass #agentList.pop(-iter) else : agentsResult.append(ip) self.__setAgents(agentsResult) print("{} 可以使用".format(ip)) agentsResult.extend(self.attArray) #记录爬取过的可以使用的代理ip if(0==len(agentsResult)): return self.__writeAgentList(agentsResult) self.__setAgents(agentsResult) self.setMyIp("") return agentList def __setAgents(self,ipArray): self.attArray=ipArray def setMyIp(self,ip): self.myagent=ip #存储爬取过的ip代理 def __writeAgentList(self, agentList): if os.path.exists("agent.pkl"): os.remove("agent.pkl") #每次重新生成 要不多次 dump需要多次 load with open("agent.pkl.","wb") as f: pickle.dump(agentList, f) print("存储{}条代理".format(len(agentList))) #加载之前存储过的ip代理 def __loadAgentList(self): agentlist=[] if not os.path.exists("agent.pkl"): return agentlist with open("agent.pkl","rb") as f: agentlist=pickle.load(f) print("加载{}条代理".format(len(agentlist))) return agentlist #获取当前使用的ip地址 类的内部方法 仅供内部调用 def __getMyIp(self,ip=""): url="https://www.baidu.com/" html="" try: html=self.openUrl(url,0).decode("utf-8") except: return #匹配ip地址 #pattern = re.compile(r'((25[0-5]|2[0-4]/d|[0-1]/d/d|/d/d|/d)/.){3}(25[0-5]|2[0-4]/d|[0-1]/d/d|/d/d|/d)') #groupIp=pattern.search(html) #if groupIp: #return groupIp.group() else: return html #通过不同的网站去爬取代理 def crawlingAgents(self,index): try: url ="http://ip.yqie.com/ipproxy.htm" print(url) html=self.openUrl(url) html=html.decode("utf-8") self.setMyIp("") #不指定ip 随机挑选一个作为代理 self.getAgents(html) except Exception as e: print("{} 爬取失败".format(url)) #一共搜集多少页 page=index indexCur=1 while indexCur<=page: try: url=r"https://www.89ip.cn/index_{}.html".format(indexCur) print(url) self.setMyIp("") html=self.openUrl(url) #不指定ip 随机挑选一个作为代理 html=html.decode("utf-8") self.getAgents(html) except Exception as e: print("{} 爬取失败".format(url)) finally: indexCur+=1 indexCur=1 while indexCur<=page: try: url=r"http://www.66ip.cn/{}.html".format(indexCur) print(url) self.setMyIp("") html=a.openUrl(url) #不指定ip 随机挑选一个作为代理 html=html.decode("gb2312") self.getAgents(html) except Exception as e: print("{} 爬取失败".format(url)) finally: indexCur+=1 indexCur=1 while indexCur<=page: try: url=r"http://www.ip3366.net/?stype=1&page={}".format(indexCur) print(url) self.setMyIp("") html=a.openUrl(url) #不指定ip 随机挑选一个作为代理 html=html.decode("gb2312") self.getAgents(html) except Exception as e: print("{} 爬取失败".format(url)) finally: indexCur+=1 indexCur=1 while indexCur<=page: try: url=r"http://www.kxdaili.com/dailiip/1/{}.html".format(indexCur) print(url) self.setMyIp("") html=a.openUrl(url) #不指定ip 随机挑选一个作为代理 html=html.decode("utf-8") self.getAgents(html) except Exception as e: print("{} 爬取失败".format(url)) finally: indexCur+=1#下载图片封装类class downLoadPictures: #构造函数 def __init__(self): self.sortKey={} #定义一个搜索关键字的字典 self.urlLoad=getHttpAgents() self.bzmenuDict={} #分类信息 风景 美女 什么的分类 self.sortscreenDict={} #按照屏幕尺寸分类 self.littleSignDict={} #分类信息下面的小分类 pass def getPictures(self,url): #第一步 打开网页 读取page信息 pagerHtml=self.urlLoad.openUrl(url) #第二步 获取 pageFolder 链接和各种分类信息 返回的是一堆folder链接的url folderPictursUrl=self.readPages(pagerHtml).values() if not folderPictursUrl: print("获取图片集失败!") return for floderiterUrl in folderPictursUrl: folderUrl=str("https://www.ivsky.com/")+floderiterUrl folderHtml=self.urlLoad.openUrl(folderUrl) #第三步 读取图片集 获取单个图片的链接地址 返回的是图片集里面的一堆文件url pictursUrlDict=self.readFolders(folderHtml) for iterPictureKey in pictursUrlDict: fileName=iterPictureKey+".jpg" pictureUrl=str("https://www.ivsky.com/")+pictursUrlDict.get(iterPictureKey) #读取图片页相关信息 pictureHtml=self.urlLoad.openUrl(pictureUrl) picturDownUrl=self.readPictures(pictureHtml) pictureDownHtml=self.urlLoad.openUrl(picturDownUrl) if not pictureDownHtml: continue #保存图片 with open(fileName,"wb+") as f: f.write(pictureDownHtml) #提取匹配内容中的所有链接地址 def getHrefMap(self,html,isPicture=0,isFolder=0): hrefDict={} pattern=re.compile(r'<a/s*.*?/s*</a>',re.I) if isPicture: pattern=re.compile(r'<p>/s*?<a/s*.*?</p>',re.I) hrefIter=pattern.finditer(html) index=0 for iter in hrefIter: hrefText=iter.group() #匹配分类名字 pattern=re.compile(r'"/s*?>/s*?.*?</a>',re.I) name="" nameGroup=pattern.search(hrefText) if nameGroup: name=nameGroup.group() if(5==len(nameGroup.group().replace(" ", ""))): pattern=re.compile(r'title=".*?"',re.I) nameGroup=pattern.search(hrefText) if nameGroup: name=nameGroup.group()[7:-1] name=name[2:-4].replace(" ", '') #匹配href pattern=re.compile(r'href=".*?" rel="external nofollow" ',re.I) url="" urlGroup=pattern.search(hrefText) if urlGroup: url=urlGroup.group()[6:-1].replace(" ", '') if isFolder: index+=1 name+="_"+str(index) hrefDict[name]=url return hrefDict #读取首页信息 包含各种分类的链接地址 以及图片集的地址集合 def readPages(self,html): html=html.decode("utf-8") #检索壁纸分类 #匹配 壁纸分类信息 pattern=re.compile(r'<ul/s*class="bzmenu".*?</ul>',re.I) sortClassGroup=pattern.search(html) if sortClassGroup: sortMessage=sortClassGroup.group() self.bzmenuDict=self.getHrefMap(sortMessage) #print(self.bzmenuDict) else: print("匹配壁纸分类出错!") return #匹配 按照屏幕大小分类 pattern=re.compile(r'<ul/s*class="sall_dd".*?</ul>',re.I) sortClassGroup=pattern.search(html) if sortClassGroup: sortMessage=sortClassGroup.group() self.sortscreenDict=self.getHrefMap(sortMessage) #print(self.sortscreenDict) else: print("匹配屏幕尺寸分类失败!") return #匹配 获取小分类 pattern=re.compile(r'<div/s*class="sline".*?</div>',re.I) sortClassGroup=pattern.search(html) if sortClassGroup: sortMessage=sortClassGroup.group() #print(sortMessage) self.littleSignDict=self.getHrefMap(sortMessage) #print(self.littleSignDict) else: print("匹配小分类失败") return pictureDict={} #匹配 图片集地址 pattern=re.compile(r'<ul/s*class="ali".*?</ul>',re.I) sortClassGroup=pattern.search(html) if sortClassGroup: sortMessage=sortClassGroup.group() pictureDict=self.getHrefMap(sortMessage,1) #print(pictureDict) else: print("匹配图片集地址失败!") return #print(html) return pictureDict #解析每个图片集合对应的图片集内容 解析出单个图片的链接地址 def readFolders(self,html): if not html: return html=html.decode("utf-8") #获取图片集里面每个图片的具体地址和名称 #匹配 获取小分类 pattern=re.compile(r'<ul/s*class="pli".*?</ul>',re.I) sortClassGroup=pattern.search(html) pictureUrlDict={} if sortClassGroup: sortMessage=sortClassGroup.group() #print(sortMessage) pictureUrlDict=self.getHrefMap(sortMessage,1,1) #print(pictureUrlDict) else: print("匹配小分类失败") return return pictureUrlDict #解析每个图片集合对应的图片集内容 解析出单个图片的链接地址 def readPictures(self,html): if not html: return html=html.decode("utf-8") #获取图片集里面每个图片的具体地址和名称 #匹配 获取小分类 pattern=re.compile(r'<div/s*class="pic".*?</div>',re.I) sortClassGroup=pattern.search(html) pictureUrl="" if sortClassGroup: sortMessage=sortClassGroup.group() #匹配href pattern=re.compile(u"src='.*?'",re.I) url="" urlGroup=pattern.search(sortMessage) if urlGroup: url=urlGroup.group()[5:-1].replace(" ", '') url=url.replace('img-pre', 'img-picdown') url=url.replace('pre', 'pic') url=str("https:")+url #print(sortMessage) pictureUrlDict=url #print(url) else: print("匹配小分类失败") return return pictureUrlDict class UrlUser: def __init__(self): self.agent=getHttpAgents() self.downPicture=downLoadPictures() #下载图片调用函数 def downPictures(self): #url="https://www.ivsky.com/bizhi" #b.getPictures(url) #确定保存路径 dirPath=input("请输入保存路径:") if not os.path.exists(dirPath): os.mkdir(dirPath) if not os.path.isdir(dirPath): print("savePath is wrong!") sys.exit() os.chdir(dirPath) #切换工作目录 #url=r"https://www.ivsky.com/bizhi/nvxing_1920x1080/index_{}.html" page=input("爬取前多少页的图片?/n") indexRe = re.search(r"/d+", page) if(not indexRe): print("输入页数有误!") indexRe=int(indexRe.group()) indexCur=1 while indexCur<=indexRe: try: #注意 爬取什么类型的图片可以根据不同的网址进行设计 下载类里面已经读取了所有分类对应的地址 有兴趣可以自己完善 url=r"https://www.ivsky.com/bizhi/nvxing_1920x1080/index_{}.html".format(indexCur) print(url) self.downPicture.getPictures(url) except: print("打开出错!") pass finally: indexCur+=1 #爬取代理 def downAgents(self): page=input("爬取前多少页的代理?/n") indexRe = re.search(r"/d+", page) if(not indexRe): print("输入页数有误!") return indexRe=int(indexRe.group()) self.agent.crawlingAgents(indexRe) # 检查当前代理池是否可以 def checkPool(self): self.agent.checkMyIpPool() if __name__ == "__main__": print("*"*20) print("1.爬取代理/n") print("2.检查代理/n") print("3.爬取图片") print("*"*20) mode=input("请输入数字选择处理模式:/n") indexRe = re.search(r"/d+", mode) if(not indexRe): print("输入页数有误!") sys.exit() indexRe=int(indexRe.group()) #实例化一个对象 uesrObj=UrlUser() if 1 == indexRe: uesrObj.downAgents() elif 2 == indexRe: uesrObj.checkPool() elif 3 == indexRe: uesrObj.downPictures() else: print("模式选择错误!") sys.exit() print("爬取完毕！")
