Scrapy教程——搭建环境、创建项目、爬取内容、保存文件（txt） - 51Testing软件测试论坛

from scrapy.item import Item, Field
class TutorialItem(Item):
# define the fields for your item here like:
# name = scrapy.Field()
parent_title = Field()
parent_url = Field()
second_title = Field()
second_url = Field()
path = Field()
link_title = Field()
link_url = Field()
head= Field()
content = Field()
pass

复制代码

# -*-coding: utf-8 -*-
__author__= 'George'
import sys, os
reload(sys)
sys.setdefaultencoding("utf-8")
from scrapy.spider import Spider
from scrapy.http import Request
from scrapy.selector import Selector
from tutorial.items import TutorialItem
base ="d:/dataset/" #存放文件分类的目录
class SinaSpider(Spider):
name= "sina"
allowed_domains= ["sina.com.cn"]
start_urls= [
"http://news.sina.com.cn/guide/"
]#起始urls列表
def parse(self, response):
items= []
sel= Selector(response)
big_urls=sel.xpath('//div[@id=\"tab01\"]/div/h3/a/@href').extract()#大类的url
big_titles=sel.xpath("//div[@id=\"tab01\"]/div/h3/a/text()").extract()
second_urls =sel.xpath('//div[@id=\"tab01\"]/div/ul/li/a/@href').extract()#小类的url
second_titles=sel.xpath('//div[@id=\"tab01\"]/div/ul/li/a/text()').extract()
for i in range(1,len(big_titles)-1):#这里不想要第一大类,big_title减去1是因为最后一个大类，没有跳转按钮，也去除
file_name = base + big_titles[i]
#创建目录
if(not os.path.exists(file_name)):
os.makedirs(file_name)
for j in range(19,len(second_urls)):
item = TutorialItem()
item['parent_title'] =big_titles[i]
item['parent_url'] =big_urls[i]
if_belong =second_urls[j].startswith( item['parent_url'])
if(if_belong):
second_file_name =file_name + '/'+ second_titles[j]
if(not os.path.exists(second_file_name)):
os.makedirs(second_file_name)
item['second_url'] = second_urls[j]
item['second_title'] =second_titles[j]
item['path'] =second_file_name
items.append(item)
for item in items:
yield Request(url=item['second_url'],meta={'item_1': item},callback=self.second_parse)
#对于返回的小类的url，再进行递归请求
def second_parse(self, response):
sel= Selector(response)
item_1= response.meta['item_1']
items= []
bigUrls= sel.xpath('//a/@href').extract()
for i in range(0, len(bigUrls)):
if_belong =bigUrls[i].endswith('.shtml') and bigUrls[i].startswith(item_1['parent_url'])
if(if_belong):
item = TutorialItem()
item['parent_title'] =item_1['parent_title']
item['parent_url'] =item_1['parent_url']
item['second_url'] =item_1['second_url']
item['second_title'] =item_1['second_title']
item['path'] = item_1['path']
item['link_url'] = bigUrls[i]
items.append(item)
for item in items:
yield Request(url=item['link_url'], meta={'item_2':item},callback=self.detail_parse)
def detail_parse(self, response):
sel= Selector(response)
item= response.meta['item_2']
content= ""
head=sel.xpath('//h1[@id=\"artibodyTitle\"]/text()').extract()
content_list=sel.xpath('//div[@id=\"artibody\"]/p/text()').extract()
for content_one in content_list:
content += content_one
item['head']= head
item['content']= content
yield item

复制代码

# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
from scrapy import signals
import json
import codecs
import sys
reload(sys)
sys.setdefaultencoding( "utf-8" )
class SinaPipeline(object):
def process_item(self, item, spider):
link_url = item['link_url']
file_name = link_url[7:-6].replace('/','_')
file_name += ".txt"
fp = open(item['path']+'/'+file_name, 'w')
fp.write(item['content'])
fp.close()
return item

复制代码

# -*- coding: utf-8 -*-
BOT_NAME = 'tutorial'
SPIDER_MODULES = ['tutorial.spiders']
NEWSPIDER_MODULE = 'tutorial.spiders'
ITEM_PIPELINES = {
'tutorial.pipelines.SinaPipeline': 300,
}
LOG_LEVEL = 'INFO'
ROBOTSTXT_OBEY = True

复制代码