51Testing软件测试论坛

标题: scrapy框架爬取知乎信息实例详解（详细） [打印本页]

---

作者: 歪小Y_02    时间: 2019-2-25 17:23
标题: scrapy框架爬取知乎信息实例详解（详细）
爬取知乎的关注信息作为我们scrapy框架的详解例子，爬取的知乎大v是轮子哥，然后将爬取的信息存储进mongo数据库。我将所有解释都放进例子里。虽然每一句代码都有解释，但是最好还是有爬虫的基础。
整体思路：
1.选定一位有较多关注数的知乎达人作为我们的爬取对象
2.通过知乎接口获得获得该粉丝的关注列表和粉丝列表
3.通过递归的方法实现对列表中每一个用户的爬取，爬取他们的粉丝列表和关注列表。
4.通过知乎接口获得列表中的每位用户的详细信息。

zhihu.py

1. # -*- coding: utf-8 -*-
   
2. # 出现500服务器响应的错误，原因是检测到我们不是通过浏览器访问的，我们在setting文件中修改headers
   
3. #   的值，将user-agent加入。
   
4. import scrapy
   
5. import json
   
6. from scrapy import Request
   
7. from zhihuuser.items import UserItem
   
8. class ZhihuSpider(scrapy.Spider):
   
9.     name = 'zhihu'
   
10.     allowed_domains = ['www.zhihu.com']
    
11.     start_url = ['http://www.zhihu.com/']
    
12.     start_user='excited-vczh'
    
13.     user_url='https://www.zhihu.com/api/v4/members/{user}?include={include}'
    
14.     #user意思是用户的url,即轮子哥的url
    
15. 
    
16.     user_query='allow_message,is_followed,is_following,is_org,is_blocking,employments,answer_count,follower_count,articles_count,gender,badge[?(type=best_answerer)].topics'
    
17.     follows_url='https://www.zhihu.com/api/v4/members/{user}/followees?include={include}&offset={offset}&limit={limit}'
    
18.     follows_query='data[*].answer_count,articles_count,gender,follower_count,is_followed,is_following,badge[?(type=best_answerer)].topics'
    
19.     # follows意思是他关注的人的列表
    
20. 
    
21.     followers_url = 'https://www.zhihu.com/api/v4/members/{user}/followers?include={include}&offset={offset}&limit={limit}'
    
22.     followers_query = 'data[*].answer_count,articles_count,gender,follower_count,is_followed,is_following,badge[?(type=best_answerer)].topics'
    
23.     # followers_query和user_query都是后面的include
    
24.     #followers意思是关注他的列表
    
25.     #limit和offset也是变量，limit是固定值，而offset则根据页面的变化而变化
    
26. 
    
27. 
    
28.     def start_requests(self):
    
29.         # url='https://www.zhihu.com/api/v4/members/{user}/followees?include=data%5B*%5D.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F(type%3Dbest_answerer)%5D.topics&offset=20&limit=20'
    
30.         # url='https://www.zhihu.com/api/v4/members/excited-vczh/publications?include=data%5B*%5D.cover%2Cebook_type%2Ccomment_count%2Cvoteup_count&offset=0&limit=5'
    
31.         yield Request(self.user_url.format(user=self.start_url,include=self.user_query), self.parse_user)
    
32.         yield Request(self.follows_url.format(user=self.start_user,include=self.follows_query,offset=0,limit=20), callback=self.parse_follows)
    
33.         yield Request(self.followers_url.format(user=self.start_user, include=self.followers_query, offset=0, limit=20), callback=self.parse_followers)
    
34. # 使用format方法是为了动态的构造url
    
35. # 同时也需要指定回调函数来进行url的解析
    
36. 
    
37. 
    
38.     def parse_user(self, response):
    
39.         result = json.loads(response.text)  # 利用json.load声明一个json对象
    
40.         item = UserItem()
    
41.         for field in item.fields:
    
42.             # 利用item的field属性进行赋值，实际上field输出的是所有集合的名称
    
43.             if field in result.keys():
    
44.                 # 如果field是result的键名之一则field进行赋值
    
45.                 item[field] = result.get(field)
    
46.         yield item
    
47.         yield Request(self.follows_url.format(user=result.get('url_token'),include=self.follows_query,offset=0,limit=20,callback=self.parse_follows))
    
48.         # 获取关注列表的request
    
49.         yield Request(self.followers_url.format(user=result.get('url_token'), include=self.followers_query, offset=0, limit=20,callback=self.parse_followers))
    
50. 
    
51. 
    
52.     def parse_follows(self, response):
    
53.         result = json.loads(response.text)
    
54.         if 'data' in result.keys():
    
55.             for result in result.get('data'):
    
56.                 yield Request(self.user_url.format(user=result.get('url_token'),include=self.user_query),callback=self.parse_user)
    
57.         if 'paging' in result.keys() and result.get('pagging').get('is_end') == False:
    
58.             next_page = result.get('pagging').get('next')
    
59.             # 从pagging的next键得到分页下一页的链接
    
60.             yield Request(next_page,self.parse_follows)
    
61.             #传入url链接，新建一个request请求，然后回调parse_follows
    
62. 
    
63. 
    
64. 
    
65.     def parse_followers(self, response):
    
66.         result = json.loads(response.text)
    
67.         if 'data' in result.keys():
    
68.             for result in result.get('data'):
    
69.                 yield Request(self.user_url.format(user=result.get('url_token'),include=self.user_query),callback=self.parse_user)
    
70.         if 'paging' in result.keys() and result.get('pagging').get('is_end') == False:
    
71.             next_page = result.get('pagging').get('next')
    
72.             # 从pagging的next键得到分页下一页的链接
    
73.             yield Request(next_page,self.parse_followers)
    
74.             #传入url链接，新建一个request请求，然后回调parse_follows
    

复制代码

item.py

1. # -*- coding: utf-8 -*-
   
2. 
   
3. # Define here the models for your scraped items
   
4. #
   
5. # See documentation in:
   
6. # https://doc.scrapy.org/en/latest/topics/items.html
   
7. 
   
8. import scrapy
   
9. from scrapy import Item, Field
   
10. 
    
11. class UserItem(scrapy.Item):
    
12.     id = Field()
    
13.     name = Field()
    
14.     avatar_url = Field()
    
15.     headline = Field()
    
16.     url_token =Field()
    
17.     url = Field()
    
18.     avatar_url_template = Field()
    
19.     type = Field()
    
20. 
    
21.     # define the fields for your item here like:
    
22.     # name = scrapy.Field()
    

复制代码

setting.py

1. # -*- coding: utf-8 -*-
   
2. 
   
3. # Scrapy settings for zhihuuser project
   
4. #
   
5. # For simplicity, this file contains only settings considered important or
   
6. # commonly used. You can find more settings consulting the documentation:
   
7. #
   
8. #     https://doc.scrapy.org/en/latest/topics/settings.html
   
9. #     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
   
10. #     https://doc.scrapy.org/en/latest/topics/spider-middleware.html
    
11. 
    
12. BOT_NAME = 'zhihuuser'
    
13. 
    
14. SPIDER_MODULES = ['zhihuuser.spiders']
    
15. NEWSPIDER_MODULE = 'zhihuuser.spiders'
    
16. 
    
17. 
    
18. # Crawl responsibly by identifying yourself (and your website) on the user-agent
    
19. #USER_AGENT = 'zhihuuser (+http://www.yourdomain.com)'
    
20. 
    
21. # Obey robots.txt rules
    
22. ROBOTSTXT_OBEY = False
    
23.    # 默认为true,意思是遵守rouot协议,我们改成False
    
24. 
    
25. # Configure maximum concurrent requests performed by Scrapy (default: 16)
    
26. #CONCURRENT_REQUESTS = 32
    
27. 
    
28. # Configure a delay for requests for the same website (default: 0)
    
29. # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
    
30. # See also autothrottle settings and docs
    
31. #DOWNLOAD_DELAY = 3
    
32. # The download delay setting will honor only one of:
    
33. #CONCURRENT_REQUESTS_PER_DOMAIN = 16
    
34. #CONCURRENT_REQUESTS_PER_IP = 16
    
35. 
    
36. # Disable cookies (enabled by default)
    
37. #COOKIES_ENABLED = False
    
38. 
    
39. # Disable Telnet Console (enabled by default)
    
40. #TELNETCONSOLE_ENABLED = False
    
41. 
    
42. # Override the default request headers:
    
43. DEFAULT_REQUEST_HEADERS = {
    
44.   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    
45.   'Accept-Language': 'en',
    
46. 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36'
    
47. }
    
48. 
    
49. # Enable or disable spider middlewares
    
50. # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
    
51. #SPIDER_MIDDLEWARES = {
    
52. #    'zhihuuser.middlewares.ZhihuuserSpiderMiddleware': 543,
    
53. #}
    
54. 
    
55. # Enable or disable downloader middlewares
    
56. # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
    
57. #DOWNLOADER_MIDDLEWARES = {
    
58. #    'zhihuuser.middlewares.ZhihuuserDownloaderMiddleware': 543,
    
59. #}
    
60. 
    
61. # Enable or disable extensions
    
62. # See https://doc.scrapy.org/en/latest/topics/extensions.html
    
63. #EXTENSIONS = {
    
64. #    'scrapy.extensions.telnet.TelnetConsole': None,
    
65. #}
    
66. 
    
67. # Configure item pipelines
    
68. # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
    
69. #ITEM_PIPELINES = {
    
70. #    'zhihuuser.pipelines.ZhihuuserPipeline': 300,
    
71. #}
    
72. 
    
73. # Enable and configure the AutoThrottle extension (disabled by default)
    
74. # See https://doc.scrapy.org/en/latest/topics/autothrottle.html
    
75. #AUTOTHROTTLE_ENABLED = True
    
76. # The initial download delay
    
77. #AUTOTHROTTLE_START_DELAY = 5
    
78. # The maximum download delay to be set in case of high latencies
    
79. #AUTOTHROTTLE_MAX_DELAY = 60
    
80. # The average number of requests Scrapy should be sending in parallel to
    
81. # each remote server
    
82. #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
    
83. # Enable showing throttling stats for every response received:
    
84. #AUTOTHROTTLE_DEBUG = False
    
85. 
    
86. # Enable and configure HTTP caching (disabled by default)
    
87. # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
    
88. #HTTPCACHE_ENABLED = True
    
89. #HTTPCACHE_EXPIRATION_SECS = 0
    
90. #HTTPCACHE_DIR = 'httpcache'
    
91. #HTTPCACHE_IGNORE_HTTP_CODES = []
    
92. #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
    

复制代码

---

作者: Miss_love    时间: 2020-12-31 08:53
支持分享

---

欢迎光临 51Testing软件测试论坛 (http://bbs.51testing.com/)

Powered by Discuz! X3.2