Selenium python code to click button on interactive map for web scraping not working

Question

I am trying to web scrape data from an interactive map using Selenium in python. I have been having difficulties with the code to click certain buttons to get to the data. The first "click" works fine, but the second is not working. I have tried adjusting the time and nothing is working. I want to be able to do that second click. Any help would be appreciated, thank you. Here is my code:

!apt-get update # to update ubuntu to correctly run apt install
!apt install chromium-chromedriver
!cp /usr/lib/chromium-browser/chromedriver /usr/bin
import sys
sys.path.insert(0,'/usr/lib/chromium-browser/chromedriver')
from selenium import webdriver
from selenium.webdriver.common.keys import Keys  
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select
import os
from bs4 import BeautifulSoup
import re
import pandas as pd
import sys
import re
import csv
import time
import shutil
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
wd = webdriver.Chrome('chromedriver',chrome_options=chrome_options) 

from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium import webdriver
import time
wd.maximize_window()
wd.get("https://hazards.geoplatform.gov/portal/apps/MapSeries/index.html?appid=ddf915a24fb24dc8863eed96bc3345f8")
wd.execute_script("arguments[0].click();", WebDriverWait(wd, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="nav-bar"]/div/div[1]/ul/li[4]/a'))))
wd.execute_script("arguments[0].click();", WebDriverWait(wd, 20).until(EC.element_to_be_clickable((By.XPATH, '//*[@id="widgets_About_Widget_34"]/div/div/div/div[10]/font/a'))))

The error occurs on the last line, and here is the full error:

TimeoutException Traceback (most recent call last) in () 7 wd.get("https://hazards.geoplatform.gov/portal/apps/MapSeries/index.html?appid=ddf915a24fb24dc8863eed96bc3345f8") 8 wd.execute_script("arguments[0].click();", WebDriverWait(wd, 20).until(EC.element_to_be_clickable((By.XPATH, '//[@id="nav-bar"]/div/div1/ul/li[4]/a')))) ----> 9 wd.execute_script("arguments[0].click();", WebDriverWait(wd, 60).until(EC.element_to_be_clickable((By.XPATH, '//[@id="widgets_About_Widget_34"]/div/div/div/div[10]/font/a')))) 10 11

/usr/local/lib/python3.7/dist-packages/selenium/webdriver/support/wait.py in until(self, method, message) 78 if time.time() > end_time: 79 break ---> 80 raise TimeoutException(message, screen, stacktrace) 81 82 def until_not(self, method, message=''):

TimeoutException: Message:

Here is the image of what I want to click: "Census Tracts Table" Click to see image

Prophet Prophet · Accepted Answer · 2021-08-03T15:37:24

There are several issues with your code.

The main blocker avoiding you from access the second element is because it is inside an iframe, so to acceess it you first have to switch to that iframe.
Your locators are bad.
Don't click elements with JavaScript until you have no alternative.
In headless mode you have to define the screen size.
wd.maximize_window() is used for normal mode, not headless.
So you code should be something like this:

import sys
sys.path.insert(0,'/usr/lib/chromium-browser/chromedriver')
from selenium import webdriver
from selenium.webdriver.common.keys import Keys  
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select
import os
from bs4 import BeautifulSoup
import re
import pandas as pd
import sys
import re
import csv
import time
import shutil
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--window-size=1920,1080')
chrome_options.add_argument('--disable-dev-shm-usage')
wd = webdriver.Chrome('chromedriver',chrome_options=chrome_options) 

from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium import webdriver
import time
#wd.maximize_window()
wd.get("https://hazards.geoplatform.gov/portal/apps/MapSeries/index.html?appid=ddf915a24fb24dc8863eed96bc3345f8")
wait.until(EC.visibility_of_element_located((By.XPATH, "//li[contains(@class,'entry  visible')]//a[contains(text(),'Data Download Tool')]"))).click()
wait(driver, 10).until(EC.frame_to_be_available_and_switch_to_it(driver.find_element_by_css_selector("div[class='mainMediaContainer active']>iframe")))
wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div[title='Download']"))).click()

Selenium python code to click button on interactive map for web scraping not working

The error occurs on the last line, and here is the full error:

2 Answers