爬虫性能测试代码

胖虎

1. 同步执行，效率慢
   
2. # import requests
   
3. # def parse_page(res):
   
4. #     print("PAESE %s" %(len(res)))
   
5. #
   
6. # def get_page(url):
   
7. #     print("GET %s" %url)
   
8. #     response = requests.get(url)
   
9. #     if response.status_code == 200:
   
10. #         return response.text
    
11. #
    
12. # if __name__ == '__main__':
    
13. #     urls = [
    
14. #         "https://www.baidu.com",
    
15. #         "https://www.taobao.com",
    
16. #         "https://www.python.org",
    
17. #     ]
    
18. #     for url in urls:
    
19. #         res = get_page(url)
    
20. #         parse_page(res)
    
21. 
    
22. # 开进程线程
    
23. # import requests
    
24. # from threading import Thread,current_thread
    
25. # from multiprocessing import Process
    
26. #
    
27. # def parse_page(res):
    
28. #     print("%s PARSE %s"%(current_thread().getName(),len(res)))
    
29. #
    
30. # def get_page(url,callback=parse_page):
    
31. #     print("%s GET %s"%(current_thread().getName(),url))
    
32. #     response = requests.get(url)
    
33. #     if response.status_code == 200:
    
34. #         callback(response.text)
    
35. #
    
36. # if __name__ == '__main__':
    
37. #     import time
    
38. #     urls = [
    
39. #         "https://www.baidu.com",
    
40. #         "https://www.taobao.com",
    
41. #         "https://www.python.ort",
    
42. #     ]
    
43. #     start_time = time.time()
    
44. #     for url in urls:
    
45. #         # t = Thread(target=get_page,args=(url,))
    
46. #         p = Process(target=get_page,args=(url,))
    
47. #         p.start()
    
48. #         # t.start()
    
49. #     print("===========耗时",time.time()-start_time)
    
50. 
    
51. 
    
52. import requests
    
53. from threading import current_thread
    
54. from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor
    
55. 
    
56. # 进程池线程池
    
57. # def parse_page(res):
    
58. #     res = res.result()
    
59. #     print("%s PAESE %s"%(current_thread().getName(),len(res)))
    
60. 
    
61. # def get_page(url):
    
62. #     print("%s GET %s"%(current_thread().getName(),url))
    
63. #     response = requests.get(url)
    
64. #     if response.status_code == 200:
    
65. #         return response.text
    
66. #
    
67. # if __name__ == '__main__':
    
68. #     import time
    
69. #     start_time = time.time()
    
70. #     urls = [
    
71. #         "https://www.baidu.com",
    
72. #         "https://www.taobao.com",
    
73. #         "https://www.python.org",
    
74. #     ]
    
75. #     # pool = ThreadPoolExecutor(50)      #3.5992057323455?81
    
76. #     pool = ProcessPoolExecutor(50)       #12.549717903137207
    
77. #     for url in urls:
    
78. #         pool.submit(get_page,url).add_done_callback(parse_page)
    
79. #     pool.shutdown()
    
80. #     print("========>",time.time()-start_time)
    
81. 
    
82. 
    
83. # # 协程 实现的是仿线程
    
84. # from gevent import joinall,spawn,monkey;monkey.patch_all()
    
85. # import requests,time
    
86. # from threading import current_thread
    
87. #
    
88. # def parse_page(res):
    
89. #     print("%s PAESAE %s"%(current_thread().getName(),len(res)))
    
90. #
    
91. # def get_page(url,callback=parse_page):
    
92. #     print("%s GET %s"%(current_thread().getName(),url))
    
93. #     response = requests.get(url)
    
94. #     if response.status_code == 200:
    
95. #         callback(response.text)
    
96. #
    
97. #
    
98. # if __name__ == '__main__':
    
99. #     start_time = time.time()
    
100. #     urls = [
     
101. #         "https://www.baidu.com",
     
102. #         "https://www.taobao.com",
     
103. #         "https://www.python.org",
     
104. #     ]
     
105. #     tasks = []
     
106. #     for url in urls:
     
107. #         tasks.append(spawn(get_page,url))
     
108. #     joinall(tasks)
     
109. #     print("++++++=====>",time.time()-start_time)       #2.9541687965393066
     
110. #
     
111. 
     
112. 
     
113. # import asyncio
     
114. # @asyncio.coroutine
     
115. # def task(task_id,second):
     
116. #     print("%s run "%task_id)
     
117. #     yield from asyncio.sleep(second)
     
118. #     print("%s run done"%task_id)
     
119. #
     
120. # if __name__ == '__main__':
     
121. #     tasks = [
     
122. #         task("任务一",3),
     
123. #         task("任务二",2),
     
124. #         task("任务三",1),
     
125. #     ]
     
126. #     loop = asyncio.get_event_loop()
     
127. #     loop.run_until_complete(asyncio.wait(tasks))
     
128. #     loop.close()

复制代码

海海豚

谢谢分享~

libingyu135

666

清晨一缕阳光

six，six，six！