自学教程：python爬虫scrapy基于CrawlSpider类的全站数据爬取示例解析

link = LinkExtractor(# Items只能是一个正则表达式,会提取当前页面中满足该"正则表达式"的url	　 allow=r'Items/')

import scrapy# 导入相关的包from scrapy.linkextractors import LinkExtractorfrom scrapy.spiders import CrawlSpider, Ruleclass TextSpider(CrawlSpider): name = 'text' allowed_domains = ['www.xxx.com'] start_urls = ['http://www.xxx.com/']# 链接提取器,从接受到的response对象中,根据item正则表达式提取页面中的链接	link = LinkExtractor(allow=r'Items/')	link2 = LinkExtractor(allow=r'Items/')# 规则解析器,根据callback将链接提取器提取到的链接进行数据解析# follow为true,则表示将链接提取器继续作用到链接提取器所提取到的链接页面中# 故:在我们提取多页数据时,若第一页对应的网页中包含了第2,3,4,5页的链接,# 当跳转到第5页时,第5页又包含了第6,7,8,9页的链接,# 令follow=True,就可以持续作用,从而提取到所有页面的链接 rules = (Rule(link, callback='parse_item', follow=True), 		Rule(link2,callback='parse_content',follow=False)) # 链接提取器link使用parse_item解析数据	def parse_item(self, response): item = {}  yield item # 链接提取器link2使用parse_content解析数据	def parse_content(self, response):		item = {}				yield item

scrapy genspider crawl -t spiderName www.xxx.com 

import scrapyfrom scrapy.linkextractors import LinkExtractorfrom scrapy.spiders import CrawlSpider, Rulefrom gushiPro.items import GushiproItem,ContentItemclass GushiSpider(CrawlSpider): name = 'gushi' #allowed_domains = ['www.xxx.com'] start_urls = ['https://www.gushiwen.org/'] # 链接提取器:只能使用正则表达式，提取当前页面的满足allow条件的链接 link = LinkExtractor(allow=r'/default_/d+/.aspx') # 链接提取器，提取所有标题对应的详情页url content_link = LinkExtractor(allow=r'cn/shiwenv_/w+/.aspx') rules = ( # 规则解析器，需要解析所有的页面，所有follow=True Rule(link, callback='parse_item', follow=True), # 不需要写follow，因为我们只需要解析详情页中的数据，而不是详情页中的url Rule(content_link, callback='content_item'), ) # 解析当前页面的标题 def parse_item(self, response): p_list = response.xpath('//div[@class="sons"]/div[1]/p[1]') for p in p_list: title = p.xpath('./a//text()').extract_first() item = GushiproItem() item['title'] = title yield item  # 解析详情页面的标题和内容 def content_item(self,response): # //div[@id="sonsyuanwen"]/div[@class="cont"]/div[@class="contson"] # 解析详情页面的内容 content = response.xpath('//div[@id="sonsyuanwen"]/div[@class="cont"]/div[@class="contson"]//text()').extract() content = "".join(content) # # 解析详情页面的标题 title = response.xpath('//div[@id="sonsyuanwen"]/div[@class="cont"]/h1/text()').extract_first() # print("title:"+title+"/ncontent:"+content) item = ContentItem() item["content"] = content item["title"] = title # 将itme对象传给管道 yield item

import scrapy# 不同的item类是独立的，他们可以创建不同的item对象class GushiproItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() title = scrapy.Field()class ContentItem(scrapy.Item): title = scrapy.Field() content = scrapy.Field()

from itemadapter import ItemAdapterclass GushiproPipeline: def __init__(self): self.fp = None def open_spider(self,spider): self.fp = open("gushi.txt",'w',encoding='utf-8') print("开始爬虫") def process_item(self, item, spider): # 从详情页获取标题和内容，所以需要判断爬虫文件中传来的item是什么类的item # item.__class__.__name__判断属于什么类型的item if item.__class__.__name__ == "ContentItem": content = "《"+item['title']+"》",item['content'] content = "".join(content)  print(content) self.fp.write(content) return item def close_spider(self,spider): self.fp.close() print("结束爬虫")