scrapy基本用法详解（入门）

歪小Y_02

我们要抓取的网站是：quotes.toscrape.com
流程框架：
1.抓取第一页
2.获取内容和下一页的链接
3保存爬取结果、
4翻页爬取，请求下一页信息，分析内容并请求再下一页的链接

在命令行下如下操作：
创建项目：scrapy startproject quote
创建spider文件：scrapy genspider quotes quotes.toscrtapy.com
然后用pycharm打开

quotes.py

1. # -*- coding: utf-8 -*-
   
2. import scrapy
   
3. from quote.items import QuoteItem
   
4. 
   
5. 
   
6. class QuotesSpider(scrapy.Spider):
   
7.     name = 'quotes'
   
8.     # name指定spidedr的名称
   
9.     allowed_domains = ['quotes.toscrape.com']
   
10.     start_urls = ['http://quotes.toscrape.com/']
    
11. 
    
12.     def parse(self, response):
    
13.         # parse方法是默认的回调，当爬虫开始时会从start_url得到链接，
    
14.         # 然后自动调用parse方法进行解析
    
15.         quotes =response.css('.quote')
    
16.         # 选择出每一个区域块
    
17.         for quote in quotes:  # 循环每一个区域块
    
18.             item = QuoteItem()
    
19.             text = quote.css('.text::text').extract_first()
    
20.             author = quote.css('.author::text').extract_first()
    
21.             tags = quote.css('.tags .tag::text').extract()
    
22.             item['text'] = text
    
23.             item['author'] = author
    
24.             item['tags'] = tags
    
25.             yield item
    
26. 
    
27. 
    
28.         # .text::text意思是获取class=text的文本内容
    
29.         next = response.css('.pager .next a::attr(href)').extract_first()
    
30.         # 得到的只是url的一部分
    
31.         url = response.urljoin(next)
    
32.         # 使用join构建完整的url
    
33.         yield scrapy.Request(url=url,callback=self.parse)
    
34.         # 生成request
    
35. 
    
36.         # 命令行下数据的保存 ：scrapy crawl quotes -o quotes.json
    
37.         #也支持数据格式：('json', 'jsonlines', 'jl', 'csv', 'xml', 'marshal', 'pickle')
    

复制代码

setting.py

1. # -*- coding: utf-8 -*-
   
2. 
   
3. # Scrapy settings for quote project
   
4. #
   
5. # For simplicity, this file contains only settings considered important or
   
6. # commonly used. You can find more settings consulting the documentation:
   
7. #
   
8. #     https://doc.scrapy.org/en/latest/topics/settings.html
   
9. #     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
   
10. #     https://doc.scrapy.org/en/latest/topics/spider-middleware.html
    
11. 
    
12. BOT_NAME = 'quote'
    
13. 
    
14. SPIDER_MODULES = ['quote.spiders']
    
15. NEWSPIDER_MODULE = 'quote.spiders'
    
16. 
    
17. 
    
18. # Crawl responsibly by identifying yourself (and your website) on the user-agent
    
19. #USER_AGENT = 'quote (+http://www.yourdomain.com)'
    
20. 
    
21. # Obey robots.txt rules
    
22. ROBOTSTXT_OBEY = True
    
23. 
    
24. # Configure maximum concurrent requests performed by Scrapy (default: 16)
    
25. #CONCURRENT_REQUESTS = 32
    
26. 
    
27. # Configure a delay for requests for the same website (default: 0)
    
28. # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
    
29. # See also autothrottle settings and docs
    
30. #DOWNLOAD_DELAY = 3
    
31. # The download delay setting will honor only one of:
    
32. #CONCURRENT_REQUESTS_PER_DOMAIN = 16
    
33. #CONCURRENT_REQUESTS_PER_IP = 16
    
34. 
    
35. # Disable cookies (enabled by default)
    
36. #COOKIES_ENABLED = False
    
37. 
    
38. # Disable Telnet Console (enabled by default)
    
39. #TELNETCONSOLE_ENABLED = False
    
40. 
    
41. # Override the default request headers:
    
42. #DEFAULT_REQUEST_HEADERS = {
    
43. #   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    
44. #   'Accept-Language': 'en',
    
45. #}
    
46. 
    
47. # Enable or disable spider middlewares
    
48. # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
    
49. #SPIDER_MIDDLEWARES = {
    
50. #    'quote.middlewares.QuoteSpiderMiddleware': 543,
    
51. #}
    
52. 
    
53. # Enable or disable downloader middlewares
    
54. # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
    
55. #DOWNLOADER_MIDDLEWARES = {
    
56. #    'quote.middlewares.QuoteDownloaderMiddleware': 543,
    
57. #}
    
58. 
    
59. # Enable or disable extensions
    
60. # See https://doc.scrapy.org/en/latest/topics/extensions.html
    
61. #EXTENSIONS = {
    
62. #    'scrapy.extensions.telnet.TelnetConsole': None,
    
63. #}
    
64. 
    
65. # Configure item pipelines
    
66. # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
    
67. #ITEM_PIPELINES = {
    
68. #    'quote.pipelines.QuotePipeline': 300,
    
69. #}
    
70. 
    
71. # Enable and configure the AutoThrottle extension (disabled by default)
    
72. # See https://doc.scrapy.org/en/latest/topics/autothrottle.html
    
73. #AUTOTHROTTLE_ENABLED = True
    
74. # The initial download delay
    
75. #AUTOTHROTTLE_START_DELAY = 5
    
76. # The maximum download delay to be set in case of high latencies
    
77. #AUTOTHROTTLE_MAX_DELAY = 60
    
78. # The average number of requests Scrapy should be sending in parallel to
    
79. # each remote server
    
80. #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
    
81. # Enable showing throttling stats for every response received:
    
82. #AUTOTHROTTLE_DEBUG = False
    
83. 
    
84. # Enable and configure HTTP caching (disabled by default)
    
85. # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
    
86. #HTTPCACHE_ENABLED = True
    
87. #HTTPCACHE_EXPIRATION_SECS = 0
    
88. #HTTPCACHE_DIR = 'httpcache'
    
89. #HTTPCACHE_IGNORE_HTTP_CODES = []
    
90. #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
    

复制代码

items.py

1. # -*- coding: utf-8 -*-
   
2. 
   
3. # Define here the models for your scraped items
   
4. #
   
5. # See documentation in:
   
6. # https://doc.scrapy.org/en/latest/topics/items.html
   
7. 
   
8. import scrapy
   
9. 
   
10. 
    
11. class QuoteItem(scrapy.Item):
    
12.     text = scrapy.Field()
    
13.     author = scrapy.Field()
    
14.     tags = scrapy.Field()
    
15. 
    
16. 
    
17. 
    
18. 
    
19. 
    
20.      # define the fields for your item here like:
    
21.     # name = scrapy.Field()
    

复制代码

piplines.py

1. # -*- coding: utf-8 -*-
   
2. 
   
3. # Define your item pipelines here
   
4. #
   
5. # Don't forget to add your pipeline to the ITEM_PIPELINES setting
   
6. # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
   
7. ## 对item进行处理，保存到数据库当中
   
8. from scrapy.exceptions import DropItem
   
9. class TextPipeline(object):
   
10.     def __init__(self):
    
11.         self.limit = 50
    
12. 
    
13.     def process_item(self, item, spider):
    
14.         if item['text']:
    
15.            if len(item['text']) > self.limit:
    
16.                item['text'] = item['text'][0:self.limit].rstrip() + '...'
    
17. 
    
18.            return item
    
19.         else:
    
20.             return DropItem('missing Text')
    

复制代码

Miss_love

学习了

Miss_love

支持分享