'Selenium python : my current_url doesnt update after click
Im scraping a website where I need to retrieve values from the url when i click on a button providing different form values.
I have a problem: when i click the button and retrieve the current_url, the provided values in the forms doesnt reflect in the url which should be updated (it's a search button).
My code to retrieve the url value is :
driver = webdriver.Firefox()
driver.get(url)
arrlist = []
idlist = []
service=value
for i in key_list:
form = driver.find_elements(by=By.XPATH, value='//input[@id="geo_nav"]')
form[0].send_keys(i)
form2=driver.find_elements(by=By.XPATH, value='//input[@id="sev_nav"]')
form2[0].send_keys(service)
button=driver.find_elements(by=By.XPATH, value='//button[@data-role="filter-apply"]')
button[0].click()
time.sleep(5)
url=driver.current_url
print(dept)
print(i)
id=re.findall(r"(?<=\[population\]=)(\d{9})",url)[0]
arrlist.append(i)
idlist.append(id)
the button html code is :
<button class="filter-apply cta-navigate relative hide-mobile flex withNumber" data-role="filter-apply">
<p class="hide-mobile m-r-4">Appliquer</p>
<div class="svg relative">
<span class="filters-apply-length">2</span>
<svg height="18" viewBox="0 0 16 18" width="16" xmlns="http://www.w3.org/2000/svg"><path d="m10.877 17.457 2.026 1.533v-4.553c0-.166.042-.329.12-.475l4.3-7.962h-10.68l4.122 7.978c.074.142.112.3.112.459zm3.026 4.543c-.213 0-.426-.068-.603-.203l-4.026-3.045c-.25-.189-.397-.484-.397-.797v-3.274l-4.765-9.222c-.161-.31-.148-.681.034-.979.181-.298.505-.48.854-.48h14c.352 0 .678.185.859.488.18.302.188.677.021.987l-4.977 9.215v6.31c0 .379-.214.726-.554.895-.141.07-.294.105-.446.105z" fill="#0579c7" fill-rule="evenodd" transform="translate(-4 -4)"></path></svg> </div>
</button>
I've tried to use
driver.switch_to.window(driver.window_handles[-1]);
following this post : Python Selenium Chromedriver - Can't Get current_url of new opened tab after click()
But I dont have tab or new windows issues.
I tried to click autocompletion lists in the 2 forms in inputand one of the form produces a modification of the url but not the other (the one of which effects on the url i need to monitor). The form code that works is :
<form data-component="sev_nav_input" data-no-results="Sans résultats" data-default-pho="Services" data-selected-name="Achat compulsif" data-selected-id="5928" class="filter-input filter-services relative">
<input type="text" placeholder="Services" autocomplete="off" name="sev_nav" id="sev_nav" data-role="js_filter" data-id="5928" class="autocomplete-with-result">
<span id="clear-sev-input" class="clear-sev-input" style="display: none;">
<img src="data:image/svg+xml;base64,PHN2ZyBoZWlnaHQ9IjI0IiB2aWV3Qm94PSIwIDAgMjQgMjQiIHdpZHRoPSIyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48ZyBmaWxsPSJub25lIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiPjxwYXRoIGQ9Im0wIDBoMjR2MjRoLTI0eiIvPjxwYXRoIGQ9Im0xMS43NSAyMC41YzIuNDIyIDAgNC40ODYtLjg1MyA2LjE5MS0yLjU1OSAxLjcwNi0xLjcwNSAyLjU1OS0zLjc3IDIuNTU5LTYuMTkxIDAtMi40MjItLjg1My00LjQ4Ni0yLjU1OS02LjE5MS0xLjcwNS0xLjcwNi0zLjc3LTIuNTU5LTYuMTkxLTIuNTU5LTIuNDIyIDAtNC40OTIuODYtNi4yMSAyLjU3OC0xLjY5NSAxLjY5My0yLjU0IDMuNzUtMi41NCA2LjE3MnMuODUzIDQuNDg2IDIuNTU5IDYuMTkxYzEuNzA1IDEuNzA2IDMuNzcgMi41NTkgNi4xOTEgMi41NTl6bTMuMTY0LTQuNDE0Yy0uMDUyIDAtLjExNy0uMDQtLjE5NS0uMTE3bC0yLjk2OS0yLjkzLTIuOTMgMi45NjljLS4wNTIuMDUyLS4xMy4wNzgtLjIzNC4wNzhzLS4xODItLjAyNi0uMjM0LS4wNzhsLS44Mi0uODZjLS4wNTMtLjA1Mi0uMDc5LS4xMy0uMDc5LS4yMzRzLjAyNi0uMTcuMDc4LS4xOTVsMi45NjktMi45NjktMi45NjktMi45M2MtLjE1Ni0uMTU2LS4xNTYtLjMxMiAwLS40NjhsLjgyLS44MmMuMDc5LS4wNzkuMTU3LS4xMTguMjM1LS4xMTguMDUyIDAgLjExNy4wNC4xOTUuMTE3bDIuOTY5IDIuODkgMi45NjktMi44OWMuMDc4LS4wNzguMTQzLS4xMTcuMTk1LS4xMTcuMDc4IDAgLjE1Ni4wNC4yMzQuMTE3bC44Ni44MmMuMTU2LjE1Ny4xNTYuMzEzIDAgLjQ3bC0yLjk2OSAyLjkyOSAyLjkzIDIuOTNjLjA3OC4wNzguMTE3LjE1Ni4xMTcuMjM0IDAgLjEwNC0uMDQuMTgyLS4xMTcuMjM0bC0uODIuODJjLS4wNzkuMDc5LS4xNTcuMTE4LS4yMzUuMTE4eiIgZmlsbD0iIzE0OWM5NyIvPjwvZz48L3N2Zz4K"></span>
<span class="gradient"></span>
<span class="gradient" style="display: none;"></span>
<div class="spinner" style="display: none;"></div>
<div id="services-list" class="services-list" style="display: none;"><ul data-role="autocomplete-list" class="autocomplete-list"> </ul></div></form>
The form code that doesnt work is :
<form data-component="geo_nav_input" data-selected-name="" data-selected-id="" data-selected-neighborhood-id="0" data-selected-type="" data-no-results="Sans résultats" data-pho="Localité" data-default-pho="Localité" class="filter-input relative">
<div class="hide">Chercher des professionnels en/à...</div>
<span class="icon-x toggle_geo_nav hide"></span>
<label for="geo_nav" class="hidden-label">Localité</label>
<input type="text" placeholder="Localité" autocomplete="off" name="geo_nav" id="geo_nav" data-role="js_filter" data-id="" data-neighborhoodid="0" data-type="" class="autocomplete-with-result"> <span id="clear-geo-input" class="clear-geo-input" style="display: none;">
<img src="data:image/svg+xml;base64,PHN2ZyBoZWlnaHQ9IjI0IiB2aWV3Qm94PSIwIDAgMjQgMjQiIHdpZHRoPSIyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48ZyBmaWxsPSJub25lIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiPjxwYXRoIGQ9Im0wIDBoMjR2MjRoLTI0eiIvPjxwYXRoIGQ9Im0xMS43NSAyMC41YzIuNDIyIDAgNC40ODYtLjg1MyA2LjE5MS0yLjU1OSAxLjcwNi0xLjcwNSAyLjU1OS0zLjc3IDIuNTU5LTYuMTkxIDAtMi40MjItLjg1My00LjQ4Ni0yLjU1OS02LjE5MS0xLjcwNS0xLjcwNi0zLjc3LTIuNTU5LTYuMTkxLTIuNTU5LTIuNDIyIDAtNC40OTIuODYtNi4yMSAyLjU3OC0xLjY5NSAxLjY5My0yLjU0IDMuNzUtMi41NCA2LjE3MnMuODUzIDQuNDg2IDIuNTU5IDYuMTkxYzEuNzA1IDEuNzA2IDMuNzcgMi41NTkgNi4xOTEgMi41NTl6bTMuMTY0LTQuNDE0Yy0uMDUyIDAtLjExNy0uMDQtLjE5NS0uMTE3bC0yLjk2OS0yLjkzLTIuOTMgMi45NjljLS4wNTIuMDUyLS4xMy4wNzgtLjIzNC4wNzhzLS4xODItLjAyNi0uMjM0LS4wNzhsLS44Mi0uODZjLS4wNTMtLjA1Mi0uMDc5LS4xMy0uMDc5LS4yMzRzLjAyNi0uMTcuMDc4LS4xOTVsMi45NjktMi45NjktMi45NjktMi45M2MtLjE1Ni0uMTU2LS4xNTYtLjMxMiAwLS40NjhsLjgyLS44MmMuMDc5LS4wNzkuMTU3LS4xMTguMjM1LS4xMTguMDUyIDAgLjExNy4wNC4xOTUuMTE3bDIuOTY5IDIuODkgMi45NjktMi44OWMuMDc4LS4wNzguMTQzLS4xMTcuMTk1LS4xMTcuMDc4IDAgLjE1Ni4wNC4yMzQuMTE3bC44Ni44MmMuMTU2LjE1Ny4xNTYuMzEzIDAgLjQ3bC0yLjk2OSAyLjkyOSAyLjkzIDIuOTNjLjA3OC4wNzguMTE3LjE1Ni4xMTcuMjM0IDAgLjEwNC0uMDQuMTgyLS4xMTcuMjM0bC0uODIuODJjLS4wNzkuMDc5LS4xNTcuMTE4LS4yMzUuMTE4eiIgZmlsbD0iIzE0OWM5NyIvPjwvZz48L3N2Zz4K"></span>
<span class="gradient"></span>
<span class="gradient" style="display: none;"></span>
<div class="spinner" style="display: none;"></div>
<div id="location-list" class="location-list" style="display: none;"><ul data-role="autocomplete-list" class="autocomplete-list"> </ul></div>
</form>
Solution 1:[1]
Can you make a function to navigate pages, and on each page do the actions you require. And with each call of the function use driver.switch_to.window to ensure you are on the latest page.
Although based on your edits, it now seems the issue is that you are having trouble locating and following one of the links on the pages.
def navigate(n):
""" Move through the pages. Select the relevant buttons on each page"""
window_after = driver.window_handles[0]
driver.switch_to.window(window_after)
if n == 0:
form = driver.find_elements(by=By.XPATH, value='//input[@id="geo_nav"]')
button = driver.find_elements(by=By.XPATH, value='//button[@data-role="filter-apply"]').click()
elif n == 1:
pass
# Do something
else:
pass
# Do something else
for i in range(3):
navigate(i)
time.sleep(3)
Solution 2:[2]
The solution was in fact linked to the autocompletion forms. They require you to click on the autocompletion suggestions so the button is actually working.
FYI, here is the full code to autocomplete with clicking the form, deleting the content, adding the content, clicking the list and clicking the button.
def get_city_locations(service):
url='url'
#options = Options()
#options.headless = True
driver = webdriver.Firefox()#options=options)
driver.get(url)
time.sleep(2)
buttoncookie = driver.find_elements(by=By.XPATH, value='//button[@class="cf2Lf6"]')
buttoncookie[0].click()
time.sleep(1)
form2 = driver.find_elements(by=By.XPATH, value='//input[@id="sev_nav"]')
form2[0].click()
time.sleep(1)
Static.clear_text(driver)
form2[0].send_keys(service)
time.sleep(1)
autocompleteservice = driver.find_elements(by=By.XPATH, value='//li[not(@class)]')
for f in autocompleteservice:
if f.text == service:
f.click()
df_pref=pd.read_csv('arrondissement_2022.csv',sep=',')
deptlist = []
arrlist = []
idlist = []
for i in df_pref['LIBELLE']:
df_dep=df_pref[df_pref['LIBELLE']==i]
dept = df_dep.loc[df_dep.index.values[0], 'DEP']
form = driver.find_elements(by=By.XPATH, value='//input[@id="geo_nav"]')
form[0].click()
time.sleep(1)
Static.clear_text(driver)
form[0].send_keys(i)
time.sleep(3)
autocompletelocation=driver.find_elements(by=By.XPATH, value='//li[not(@class)]')
cond=0
for a in autocompletelocation:
if a.text==i:
print ('condition ok')
cond=1
a.click()
break
time.sleep(3)
button=driver.find_elements(by=By.XPATH, value='//button[@data-role="filter-apply"]')
button[0].click()
time.sleep(3)
driver.switch_to.window(driver.window_handles[-1]);
url=driver.current_url
print(dept)
print(i)
print(url)
if cond==0:
id=0
else:
id=re.findall(r"(?<=\[population\]=)(\d{7,10})",url)[0]
print(f'id = {id}')
print('\n')
deptlist.append(dept)
arrlist.append(i)
idlist.append(id)
df0 = pd.DataFrame({"dept": deptlist, "arrondissement":arrlist,"id":idlist})
df0.to_csv('arr_id.csv',sep=';',index=False)
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|---|
| Solution 1 | Aless |
| Solution 2 |