python用selenium爬取网页运行仅在url显示data:

测试积点老人

我在练习一本爬虫书上Selenium的实战，但写完代码运行空白网页出来

1. from selenium import webdriver
   
2. from selenium.common.exceptions import TimeoutException
   
3. from selenium.webdriver.common.by import By
   
4. from selenium.webdriver.support import expected_conditions as EC
   
5. from selenium.webdriver.support.wait import WebDriverWait
   
6. from urllib.parse import urljoin
   
7. from selenium.webdriver import ChromeOptions
   
8. import logging
   
9. logging.basicConfig(level=logging.INFO,
   
10. format='%(asctime)s - %(levelname)s: %(message)s')
    
11. INDEX_URL='https://spa2.scrape.center/page/%7Bpage%7D'
    
12. TIME_OUT=10
    
13. TOTAL_PAGE=10
    
14. 
    
15. option=ChromeOptions()
    
16. option.add_experimental_option('excludeSwitches',['enable-automation'])
    
17. option.add_experimental_option('useAutomationExtension',False)
    
18. 
    
19. browser=webdriver.Chrome(options=option)
    
20. browser.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument',{
    
21. 'source':'Object.defineProperty(navigator,"webdriver",{get:() => undefined})'
    
22. })
    
23. wait=WebDriverWait(browser,TIME_OUT)
    
24. 
    
25. def scrape_page(url,condition,locator):
    
26. logging.info('scraping %s',url)
    
27. try:
    
28. browser.get(url)
    
29. wait.until(condition(locator))
    
30. except TimeoutException:
    
31. logging.error('error occurred while scraping %s',url,exc_info=True)
    
32. 
    
33. def scrape_index(page):
    
34. url=INDEX_URL.format(page=page)
    
35. scrape_page(url,condition=EC.visibility_of_all_elements_located,
    
36. locator=(By.CSS_SELECTOR,'#index .el-row'))
    
37. 
    
38. def parse_index():
    
39. elements=browser.find_elements(by=By.CSS_SELECTOR,value='#index .el-row .name')
    
40. for element in elements:
    
41. href=element.get_attribute('href')
    
42. yield urljoin(INDEX_URL,href)
    
43. 
    
44. def main():
    
45. try:
    
46. for page in range(1,TOTAL_PAGE+1):
    
47. scrape_index(page)
    
48. detail_urls=parse_index()
    
49. logging.info('details urls %s',list(detail_urls))
    
50. finally:
    
51. browser.close()

复制代码

运行出来结果就是弹出个网页

书上正常运行可以打印输出每部电影详情页的url

qqq911

检查下驱动

kallinr

检查环境