python selenium 用scrapy爬取时，点击后的页面是否应该用response来获取

测试积点老人

代码如下

1. import scrapy
   
2. from ..items import ShujukuItem
   
3. import selenium
   
4. import time
   
5. from selenium import webdriver
   
6. from selenium.webdriver.common.action_chains import ActionChains
   
7. class SjkSpider(scrapy.Spider):
   
8.     name = 'sjk'
   
9.     allowed_domains = ['fintechdb.cn']
   
10.     start_urls = ['http://www.fintechdb.cn/']
    
11.     def parse(self, response):
    
12.         item = ShujukuItem()
    
13.         location_data = input("所在地区")
    
14.         field_data = input("业务领域")
    
15.         financing_data = input("融资轮次")
    
16.         option = webdriver.ChromeOptions()
    
17.         option.add_argument('--headless')
    
18.         option.add_argument('--disable-gpu')
    
19.         option.add_argument('--disable-javascript')
    
20.         option.add_argument('blink-settings=imagesEnabled=false')
    
21.         option.add_experimental_option("excludeSwitches", ['enable-automation', 'enable-logging'])
    
22.         driver = webdriver.Chrome(chrome_options=option)
    
23.         driver.get('http://www.fintechdb.cn/')
    
24.         location = response.xpath('//*[@id="filter-tag"]/dl/dd[1]/ul/li[*]/a/text()').getall()
    
25.         field = response.xpath('//*[@id="filter-tag"]/dl/dd[2]/ul/li[*]/a/text()').getall()
    
26.         financing = response.xpath('//*[@id="filter-tag"]/dl/dd[3]/ul/li[*]/a/text()').getall()
    
27.         a = location.index(location_data) + 1
    
28.         b = field.index(field_data) + 1
    
29.         c = financing.index(financing_data) + 1
    
30.         # print(a,b,c)
    
31.         loc = response.xpath(f'//*[@id="filter-tag"]/dl/dd[1]/ul/li[{a}]/a/text()').get()
    
32.         fie = response.xpath(f'//*[@id="filter-tag"]/dl/dd[2]/ul/li[{b}]/a/text()').get()
    
33.         fin = response.xpath(f'//*[@id="filter-tag"]/dl/dd[3]/ul/li[{c}]/a/text()').get()
    
34.         # print(loc)
    
35.         click_01 = driver.find_element_by_xpath(f'//*[@id="filter-tag"]/dl/dd[1]/ul/li[{a}]/a')
    
36.         ActionChains(driver).move_to_element(click_01).click(click_01).perform()
    
37.         click_02 = driver.find_element_by_xpath(f'//*[@id="filter-tag"]/dl/dd[2]/ul/li[{b}]/a')
    
38.         ActionChains(driver).move_to_element(click_02).click(click_02).perform()
    
39.         click_03 = driver.find_element_by_xpath(f'//*[@id="filter-tag"]/dl/dd[3]/ul/li[{c}]/a')
    
40.         ActionChains(driver).move_to_element(click_03).click(click_03).perform()
    
41.         # while True:
    
42.         #     if response.xpath('//*[@id="home-load-more"]/@style/text()').get() == 'display: block;':
    
43.         #         driver.find_element_by_id('home-load-more').click()
    
44.         #     else:
    
45.         #         break
    
46.         title = response.xpath('//*[@id="company-list"]/div[*]/a/div/div[2]/text()').getall()
    
47.         time = response.xpath('//*[@id="company-list"]/div[*]/a/div/p[1]/text()').getall()
    
48.         fie = response.xpath('//*[@id="company-list"]/div[*]/a/div/p[2]/text()').getall()
    
49.         fin = response.xpath('//*[@id="company-list"]/div[*]/a/div/p[3]/text()').getall()
    
50.         print(title)
    
51.         item['title'] = title
    
52.         item['time'] = time
    
53.         item['fie'] = fie
    
54.         item['fin'] = fin
    
55.         yield item
    

复制代码

问题是

1. 
   
2.         title = response.xpath('//*[@id="company-list"]/div[*]/a/div/div[2]/text()').getall()
   
3.         time = response.xpath('//*[@id="company-list"]/div[*]/a/div/p[1]/text()').getall()
   
4.         fie = response.xpath('//*[@id="company-list"]/div[*]/a/div/p[2]/text()').getall()
   
5.         fin = response.xpath('//*[@id="company-list"]/div[*]/a/div/p[3]/text()').getall()
   

复制代码

这些代码应该在selenium实现点击后再爬取，那么是否应该还为response.xpath，还是应该用什么
因为我过程中打印两组数据确定已经定位到位置了并且更改click的方式确保点击到
但是打印出的结果仍为未点击的结果
所以就在思考是不是最后这里response的问题

海海豚

https://www.cnblogs.com/fighter007/p/13720315.html  看下这个

qqq911

返回是可以直接存为变量的