We Keep Coding

Selenium element not interactable error on headless mode but works without headless

I'm trying to scrape the webpage ted.europa.eu using Python with Selenium to retrieve information from the tenders. The script is supposed to be executed once a day with the new publications. The problem I have is that navigating to the new tenders I need Selenium to apply a filter to get only the ones from the same day the script it's executed. I already have the script for this and works perfectly, the problem is that when I activate the headless mode I get the following error selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable: [object HTMLInputElement] has no size and location
This is the code I have that applies the filter I need:
import sys
import time
import re
from datetime import datetime
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from dotenv import load_dotenv
load_dotenv("../../../../.env")
sys.path.append("../src")
sys.path.append("../../../../utils")
from driver import *
from lted import LTED
from runnable import *
# start
print('start...')
counter = 0
start = datetime.now()
# get driver
driver = get_driver_from_url("https://ted.europa.eu/TED/browse/browseByMap.do%22)
actions = ActionChains(driver)
# change language to spanish
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "lgId")))
driver.find_element(By.ID, "lgId").click()
driver.find_element(By.XPATH, "//select[#id='lgId']/option[text()='español (es)']").click()
# click on "Busqueda avanzada"
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "goToSearch")))
driver.find_element(By.ID, "goToSearch").click()
# accept cookies and close tab
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "cookie-consent-banner")))
driver.find_element(By.XPATH, "//div[#id='cookie-consent-banner']/div[1]/div[1]/div[2]/a[1]").click()
driver.find_element(By.XPATH, "//div[#id='cookie-consent-banner']/div[1]/div[1]/div[2]/a[1]").click()
# click on specific date and set to today
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "publicationDateSpecific")))
element = driver.find_element(By.ID, "publicationDateSpecific")
actions.move_to_element(element).perform()
driver.find_element(By.ID, "publicationDateSpecific").click()
driver.find_element(By.CLASS_NAME, "ui-state-highlight").click()
# click on search
driver.find_element(By.ID, "search").click()
From the imports the only think I need to explain is that from the line from dirver import * comes the method get_driver_from_url() that is used later in the code. This method looks like this:
def get_driver_from_url(url):
options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--start-maximized")
options.add_argument("--headless")
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
driver.get(url)
return driver
As I said this code works perfectly without the headless mode, but when activated I get the error.
At first got another error and searching on the Internet found out that it could be because the element is not on screen, so I added the argument "--start-maximized" to make sure the Chrome tab is as big as possible and added the ActionChains to use actions.move_to_element(element).perform(), but I get this error on this exact code line.
Also tried changing the line WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "publicationDateSpecific"))) to WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "publicationDateSpecific"))) but it just didn't work.
Update: Also tried changing to EC.visibility_of_element_located as mentioned in this post but didn't work either
What am I doing wrong?

This is probably because of the window size.
Try adding this:
chrome_options = Options()
chrome_options.add_argument("--window-size=1920,1080")
chrome_options.add_argument("--start-maximized")
chrome_options.add_argument("--headless")

So, after a long time of try and error, I found that adding
element = driver.find_element(By.ID, "publicationDateSpecific")
driver.execute_script("window.scrollTo(0,"+str(element.location["y"])+")")
makes the script work both in headless mode and normal mode

ElementClickInterceptedException with EC.element_to_be_clickable using Chromedriver 88

I am attempting to download a file from the CDC website by clicking a button in a dropdown menu (I would just access the file URL directly, but the blob URL seems to change every time the download button is clicked when checking my download history on Chrome). This button can be found by clikcing the header "Data Table for Trends in Number of COVID-19 Vaccinations in the US" below the chart on https://covid.cdc.gov/covid-data-tracker/#vaccination-trends.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
import time
driver_path = "some_path"
vac_trend_url = "https://covid.cdc.gov/covid-data-tracker/#vaccination-trends"
driver = webdriver.Chrome(driver_path)
driver.maximize_window()
driver.get(vac_trend_url)
table_title_xpath = "/html/body/div[7]/div[2]/main/div[2]/div[1]/h4"
table_title = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, table_title_xpath)))
table_title.click()
download_button_xpath = "/html/body/div[7]/div[2]/main/div[2]/div[2]/div[1]/button/.."
download_button = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, download_button_xpath)))
download_button.click()
# ActionChains(driver).move_to_element(WebDriverWait(driver, 5).until(
# EC.element_to_be_clickable((By.XPATH, download_button_xpath)))).click().perform()
Although the first click to reveal the actual download button in the dropdown works, the second click to actually download the file does not.
Message: element click intercepted: Element is not clickable at point (1445, 310)
The title explains my error running the code above. When I try to uncomment the ActionChain line, I get the error described here despite using the very same method described in the post that should fix said error. The button does not seem to be overlayed with some other HTML element, so I am even more confused about what the issue is. Is there a workaround?
EDIT: I also tried MoveTargetOutOfBoundsException problem with chromedriver version >74 at another user's suggestion, still no dice.

This should simply download the file for you. By sending click to the element.
download_button_xpath = "/html/body/div[7]/div[2]/main/div[2]/div[2]/div[1]/button"
download_button = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, download_button_xpath)))
driver.execute_script("arguments[0].click();", download_button )

How do I click on button in a new tab? Python Selenium [duplicate]

Since Firefox does not support Control + T anymore for the tab, I started using
driver.execute_script("window.open('URL', 'new_window')")
I am trying to display the title of the different tab I open and switch between them. For the example below, I expect the output to be facebook, google and back to facebook. Right now the output is facebook, facebook and facebook.
I tried the answer from here but it also did not work: Switch back to parent tab using selenium webdriver
from selenium import webdriver
driver = webdriver.Firefox()
driver.get("http://www.facebook.com/")
print(driver.title)
driver.execute_script("window.open('http://google.com', 'new_window')")
print(driver.title)
driver.switch_to.window(driver.window_handles[0])
print(driver.title)
UPDATED:
I tried the follow code and it still did not work.
from selenium import webdriver
driver = webdriver.Firefox()
driver.get("http://www.facebook.com/")
print(driver.title)
window_before = driver.window_handles[0]
driver.execute_script("window.open('http://google.com', 'new_window')")
window_after = driver.window_handles[1]
driver.switch_to.window(window_after)
print(driver.title)

A few words about Tab/Window switching/handling:
Always keep track of the Parent Window handle so you can traverse back later if required as per your usecase.
Always use WebDriverWait with expected_conditions as number_of_windows_to_be(num_windows) before switching between Tabs/Windows.
Always keep track of the Child Window handles so you can traverse whenever required.
Always use WebDriverWait with expected_conditions as title_contains("partial_page_title") before extracting the Page Title.
Here is your own code with some minor tweaks mentioned above:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Firefox(executable_path=r'C:\WebDrivers\geckodriver.exe')
driver.get("http://www.facebook.com/")
print("Initial Page Title is: %s" %driver.title)
windows_before = driver.current_window_handle
driver.execute_script("window.open('http://google.com')")
WebDriverWait(driver, 10).until(EC.number_of_windows_to_be(2))
windows_after = driver.window_handles
new_window = [x for x in windows_after if x != windows_before][0]
driver.switch_to.window(new_window)
WebDriverWait(driver, 20).until(EC.title_contains("G"))
print("Page Title after first window switching is: %s" %driver.title)
driver.close()
driver.switch_to.window(windows_before)
WebDriverWait(driver, 20).until(EC.title_contains("F"))
print("Page Title after second window switching is: %s" %driver.title)
driver.quit()
Console Output:
Initial Page Title is: Facebook – log in or sign up
Page Title after first window switching is: Google
Page Title after second window switching is: Facebook – log in or sign up

window.open will open the link in a new tab. Selenium Firefox driver doesn't have the ability to switch between tabs as there is only one window handle for both of them (unlike Chrome which has 2). If you give the window() command 'specs' parameter with width and height it will open a new window and you will be able to switch.
After opening the new window the driver is still focused on the first one, you need to switch to the new window first.
size = driver.get_window_size();
driver.execute_script("window.open('http://google.com', 'new_window', 'height=argument[0], width=argument[1]')", size['height'], size['width'])
driver.switch_to.window(driver.window_handles[1])
print(driver.title)

I've used driver.getWindowHandles(); to get all windows and driver.switchTo().window(handle); to switch to required one.

This command always takes you to the recently opened new tab in chrome. Tested on latest versions of Chrome and Pytest on mac.
selenium.switch_to.window(selenium.window_handles[-1])

How to switch tabs using the expected conditions new_window_is_opened and verify the page title using Selenium and Python

I'm writing a test which goes through multiple pages and then verifies the title is being displayed. Currently my test is at the point where i can click a button and it opens a report. The problem is the report opens up in a new tab so i need my test to move tabs and verify the title in the new tab.
The title that needs to be verified is 'valuation'. I've done an assert to confirm the title is the same as i expect
I've written the code below in python. Its currently failing at wait on the 2nd line
current = self.driver.current_window_handle
wait(self.driver, 10).until(EC.new_window_is_opened(self.driver.window_handles))
self.driver.switch_to.window([w for w in self.driver.window_handles if w != current][0])
title = self.driver.title
self.assertTrue("Valuation" == self.driver.title)
I'm opening a new tab with the following lines of code:
element = driver.find_element_by_xpath("//input[#id='save' and #name='save'][#value='View Report']")
driver.execute_script("arguments[0].click();", element)

new_window_is_opened(current_handles)
new_window_is_opened(current_handles) is the expectation that a new window will be opened and have the number of windows handles increase.
Demonstration
The following example opens the url http://www.google.co.in first and then opens the url https://www.yahoo.com in the adjacent tab:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
driver = webdriver.Chrome(options=options, executable_path=r'C:\WebDrivers\chromedriver.exe')
driver.get("http://www.google.co.in")
windows_before = driver.window_handles
driver.execute_script("window.open('https://www.yahoo.com')")
WebDriverWait(driver, 20).until(EC.new_window_is_opened(windows_before))
driver.switch_to.window([x for x in driver.window_handles if x not in windows_before][0])
Note: The argument passed to new_window_is_opened(current_handles) is a list. In order to create a list we need to use windows_before = driver.window_handles
This usecase
In your code block, it is expected that when you:
current = self.driver.current_window_handle
There exists only one window handle. So moving forward, the line of code:
wait(self.driver, 10).until(EC.new_window_is_opened(self.driver.window_handles))
would wait for a new window to be opened and have the number of windows handles increases and that would happen only after an action is performed which opens a new window, which seems to be missing from your code block.
Solution
Insert the line of code which initiates opening of a new window-handles:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
windows_before = driver.window_handles
element = driver.find_element_by_xpath("//input[#id='save' and #name='save'][#value='View Report']")
driver.execute_script("arguments[0].click();", element)
WebDriverWait(self.driver, 10).until(EC.new_window_is_opened(windows_before))
driver.switch_to.window([x for x in driver.window_handles if x not in windows_before][0])
assertTrue("Valuation" == driver.title)
Note: As per the lines of code you have updated within your comments you are not using Python class, so you shouldn't use the keyword self.

How can I close a tab then go to the other tab with selenium?

Here is an example script:
from selenium import webdriver
driver = webdriver.Firefox()
driver.get('https://www.startpage.com/en/')
driver.find_element_by_xpath('//*[#id="query"]').send_keys('Example')
driver.find_element_by_xpath('/html/body/div[1]/main/div[1]/section/div[1]/div[1]/form/button[2]/span[2]').click()
driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div[1]/div[2]/div[1]/div/section[5]/div[1]/a/h3').click()
At the end, it clicks something that opens a new tab. I want to close the new tab and continue working with the original tab

driver.window_handles is the object for browser tabs.
Assuming that you want to close second tab (new tab), this works.
# continue from your code
driver.switch_to.window(driver.window_handles[1])
driver.close()
Just change i of window_handles[i] for other tab.

To close the new tab and continue working with the original tab you have to:
Switch to the new tab inducing WebDriverWait for number_of_windows_to_be(2) and close().
Switch back to the parent tab.
Code Block:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=options, executable_path=r'C:\WebDrivers\chromedriver.exe')
driver.get('https://www.startpage.com/en/')
print("Initial Page Title is : %s" %driver.title)
windows_before = driver.current_window_handle
print("First Window Handle is : %s" %windows_before)
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input.search-form__input"))).send_keys("Example")
driver.find_element_by_css_selector("span.search-form__button-icon").click()
WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//h3[#class='w-gl__label']//following::h3[1]"))).click()
WebDriverWait(driver, 10).until(EC.number_of_windows_to_be(2))
windows_after = driver.window_handles
new_window = [x for x in windows_after if x != windows_before][0]
driver.switch_to.window(new_window)
print("Page Title after Tab Switching is : %s" %driver.title)
print("Second Window Handle is : %s" %new_window)
driver.close()
driver.switch_to.window(windows_before)
print("Page Title after second Tab Switching is : %s" %driver.title)
print("Current Window Handle is : %s" %windows_before)
Console Output:
Initial Page Title is : Startpage.com - The world's most private search engine
First Window Handle is : CDwindow-18CCC5501A5F68CBE1C3094D0D0B419D
Page Title after Tab Switching is : YouTube
Second Window Handle is : CDwindow-2EDCAB04A232660E8BCBD7A079DE574B
Page Title after second Tab Switching is : Startpage.com Search results
Current Window Handle is : CDwindow-18CCC5501A5F68CBE1C3094D0D0B419D
You can find a relevant detailed discussion in Open web in new tab Selenium + Python