Python数据采集Selenium、PantomJS浅谈

海鸥一飞

        一直以来我觉得用在运维的Selenium、PantomJS是一个重器，
        不到万不得已的时候不要祭出这个大杀器，
        但是涉及到JavaScript及Ajax渲染的时候，Requests就完全懵逼了！

        最近回过头来重新审视这货，
        这个重器用反倒轻便了很多。

        1.安装Selenium、PantomJS

        Selenium可以直接通过pip安装，PantomJS则时一个exe可执行文件，需要下载解压。在使用的时候指定exe的绝对路径即可。

        2.Selenium、PantomJS基本设置

1. from selenium import webdriver
   
2. from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
   
3. dcap = DesiredCapabilities.PHANTOMJS
   
4. dcap[ "phantomjs.page.settings.userAgent"] = "Mozilla / 4.0(Windows NT 10.0; Win64;x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome/51.0.2704.79 Safari/ 537.36Edge/14.14393"
   
5. # 请求头不一样，自适应的窗口不一样，卧槽，坑爹！
   
6. driver = webdriver.PhantomJS(desired_capabilities=dcap)
   
7. driver.set_page_load_timeout(10)
   
8. driver.set_script_timeout(10) # 设置页面退出时间，没有必要等一个网页加载完了采集
   
9. # 采集网页源码
   
10.     try:
    
11.         driver.get(inurl)
    
12.         content = driver.page_source
    
13.         # print(content)
    
14.         time.sleep(1)
    
15.     except:
    
16.         driver.execute_script('window.stop()')
    
17. driver.close()
    

复制代码

        3.Selenium、PantomJS基本操作
         如果你的网络和机子足够好，基本上就不用等待网页渲染，
         否则，还需要等待，如果用time.sleep()，则有点笨拙，

1. #等待页面渲染完成
   
2. from selenium.webdriver.common.by import By
   
3. from selenium.webdriver.support.ui import WebDriverWait
   
4. from selenium.webdriver.support import expected_conditions as EC
   
5. ...
   
6. try:
   
7.     element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "loadedButton")))
   
8. # 等某个标签元素出来，不见鸭子不撒鹰。
   
9. 
   
10. finally:  # 撒鹰
    
11.     print(driver.find_element_by_id("content").text)
    
12.     driver.close()

复制代码

         或者用

1. try:
   
2.     elem == driver.find_element_by_tag_name("html")
   
3.     # 抛出StaleElementReferenceException异常说明elem元素已经消失了，也就说明页面已经跳转了。
   
4. except StaleElementReferenceException:  
   
5.     return

复制代码

      其他driver内置函数，可以通过查看源代码或者在pycharm提示获取。
        4.Xpath定位Html标签

1. 1.id定位：find_element_by_id(self, id_)
   
2. 2.name定位：find_element_by_name(self, name)
   
3. 3.class定位：find_element_by_class_name(self, name)
   
4. 4.tag定位：find_element_by_tag_name(self, name)
   
5. 5.link定位：find_element_by_link_text(self, link_text)
   
6. 6.partial_link定位find_element_by_partial_link_text(self, link_text)
   
7. 7.xpath定位：find_element_by_xpath(self, xpath)
   
8. 8.css定位：find_element_by_css_selector(self, css_selector）
   
9. 9.id复数定位find_elements_by_id(self, id_)
   
10. 10.name复数定位find_elements_by_name(self, name)
    
11. 11.class复数定位find_elements_by_class_name(self, name)
    
12. 12.tag复数定位find_elements_by_tag_name(self, name)
    
13. 13.link复数定位find_elements_by_link_text(self, text)
    
14. 14.partial_link复数定位find_elements_by_partial_link_text(self, link_text)
    
15. 15.xpath复数定位find_elements_by_xpath(self, xpath)
    
16. 16.css复数定位find_elements_by_css_selector(self, css_selector
    
17. 17.find_element(self, by='id', value=None)
    
18. 18.find_elements(self, by='id', value=None)

复制代码

         其中element方法定位到是是单数，是直接定位到元素；elements方法是复数，这个学过英文的
都知道，定位到的是一组元素，返回的是list队列。可参照Re函数中的findall理解。

          5.完整例子

          这个例子属于标准化操作，在实际中可以适当简化，并结合上面的Xpath定位完成。

1. from selenium import webdriver
   
2. import time
   
3. from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
   
4. 
   
5. dcap = dict(DesiredCapabilities.PHANTOMJS)
   
6. dcap["phantomjs.page.settings.userAgent"] = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36"
   
7. 
   
8. driver = webdriver.PhantomJS(executable_path=r'C:\Users\taojw\Desktop\pywork\phantomjs-2.1.1-windows\bin\phantomjs.exe', desired_capabilities=dcap)
   
9. driver.get("http://pythonscraping.com/pages/javascript/ajaxDemo.html")
   
10. time.sleep(3)
    
11. print(driver.find_element_by_id("content").text)
    
12. driver.close()
    
13. 
    
14. #设置PHANTOMJS的USER-AGENT
    
15. from selenium import webdriver
    
16. from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
    
17. 
    
18. dcap = dict(DesiredCapabilities.PHANTOMJS)
    
19. dcap["phantomjs.page.settings.userAgent"] = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36"
    
20. 
    
21. 
    
22. driver = webdriver.PhantomJS(executable_path='./phantomjs.exe', desired_capabilities=dcap)
    
23. driver.get("http://dianping.com/")
    
24. 
    
25. cap_dict = driver.desired_capabilities  #查看所有可用的desired_capabilities属性。
    
26. for key in cap_dict:
    
27.     print('%s: %s' % (key, cap_dict[key]))
    
28. print(driver.current_url)
    
29. driver.quit()
    
30. 
    
31. #等待页面渲染完成
    
32. from selenium.webdriver.common.by import By
    
33. from selenium.webdriver.support.ui import WebDriverWait
    
34. from selenium.webdriver.support import expected_conditions as EC
    
35. 
    
36. driver = webdriver.PhantomJS(executable_path=r'C:\Users\taojw\Desktop\pywork\phantomjs-2.1.1-windows\bin\phantomjs.exe')
    
37. driver.get("http://pythonscraping.com/pages/javascript/ajaxDemo.html")
    
38. try:
    
39.     element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "loadedButton")))
    
40. finally:
    
41.     print(driver.find_element_by_id("content").text)
    
42.     driver.close()
    
43. 
    
44. #处理Javascript重定向
    
45. from selenium import webdriver
    
46. import time
    
47. from selenium.webdriver.remote.webelement import WebElement
    
48. from selenium.common.exceptions import StaleElementReferenceException
    
49. 
    
50. def waitForLoad(driver):
    
51.     elem = driver.find_element_by_tag_name("html")
    
52.     count = 0
    
53.     while True:
    
54.         count += 1
    
55.         if count > 20:
    
56.             print("Timing out after 10 seconds and returning")
    
57.             return
    
58.         time.sleep(.5)
    
59.         try:
    
60.             elem == driver.find_element_by_tag_name("html")
    
61.         except StaleElementReferenceException:
    
62.             return
    
63. 
    
64. driver = webdriver.PhantomJS(executable_path=r'C:\Users\taojw\Desktop\pywork\phantomjs-2.1.1-windows\bin\phantomjs.exe')
    
65. driver.get("http://pythonscraping.com/pages/javascript/redirectDemo1.html")
    
66. waitForLoad(driver)
    
67. print(driver.page_source)
    
68. ######
    
69. from selenium import webdriver
    
70. from selenium.webdriver.remote.webelement import WebElement
    
71. from selenium.webdriver import ActionChains
    
72. 
    
73. driver = webdriver.PhantomJS(executable_path='phantomjs/bin/phantomjs')
    
74. driver.get('http://pythonscraping.com/pages/javascript/draggableDemo.html')
    
75. 
    
76. print(driver.find_element_by_id("message").text)
    
77. 
    
78. element = driver.find_element_by_id("draggable")
    
79. target = driver.find_element_by_id("div2")
    
80. actions = ActionChains(driver)
    
81. actions.drag_and_drop(element, target).perform()
    
82. 
    
83. print(driver.find_element_by_id("message").text)
    
84. #######
    
85. #截屏
    
86. driver.get_screenshot_as_file('tmp/pythonscraping.png')
    
87. 
    
88. ####
    
89. #登陆知乎，然后能自动点击页面下方的“更多”，以载入更多的内容
    
90. from selenium import webdriver
    
91. from selenium.webdriver.common.keys import Keys
    
92. from selenium.webdriver.support.ui import WebDriverWait
    
93. from selenium.webdriver import ActionChains
    
94. import time
    
95. import sys
    
96. 
    
97. driver = webdriver.PhantomJS(executable_path='C:\Users\Gentlyguitar\Desktop\phantomjs-1.9.7-windows\phantomjs.exe')
    
98. driver.get("http://www.zhihu.com/#signin")
    
99. #driver.find_element_by_name('email').send_keys('your email')
    
100. driver.find_element_by_xpath('//input[@name="password"]').send_keys('your password')
     
101. #driver.find_element_by_xpath('//input[@name="password"]').send_keys(Keys.RETURN)
     
102. time.sleep(2)
     
103. driver.get_screenshot_as_file('show.png')
     
104. #driver.find_element_by_xpath('//button[@class="sign-button"]').click()
     
105. driver.find_element_by_xpath('//form[@class="zu-side-login-box"]').submit()
     
106. 
     
107. try:
     
108.     #等待页面加载完毕
     
109.     dr=WebDriverWait(driver,5)
     
110.     dr.until(lambda the_driver:the_driver.find_element_by_xpath('//a[@class="zu-top-nav-userinfo "]').is_displayed())
     
111. except:
     
112.     print('登录失败')
     
113.     sys.exit(0)
     
114. driver.get_screenshot_as_file('show.png')
     
115. #user=driver.find_element_by_class_name('zu-top-nav-userinfo ')
     
116. #webdriver.ActionChains(driver).move_to_element(user).perform() #移动鼠标到我的用户名
     
117. loadmore=driver.find_element_by_xpath('//a[@id="zh-load-more"]')
     
118. actions = ActionChains(driver)
     
119. actions.move_to_element(loadmore)
     
120. actions.click(loadmore)
     
121. actions.perform()
     
122. time.sleep(2)
     
123. driver.get_screenshot_as_file('show.png')
     
124. print(driver.current_url)
     
125. print(driver.page_source)
     
126. driver.quit()

复制代码

梦想家