We Keep Coding

Selenium element not interactable error on headless mode but works without headless

I'm trying to scrape the webpage ted.europa.eu using Python with Selenium to retrieve information from the tenders. The script is supposed to be executed once a day with the new publications. The problem I have is that navigating to the new tenders I need Selenium to apply a filter to get only the ones from the same day the script it's executed. I already have the script for this and works perfectly, the problem is that when I activate the headless mode I get the following error selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable: [object HTMLInputElement] has no size and location
This is the code I have that applies the filter I need:
import sys
import time
import re
from datetime import datetime
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from dotenv import load_dotenv
load_dotenv("../../../../.env")
sys.path.append("../src")
sys.path.append("../../../../utils")
from driver import *
from lted import LTED
from runnable import *
# start
print('start...')
counter = 0
start = datetime.now()
# get driver
driver = get_driver_from_url("https://ted.europa.eu/TED/browse/browseByMap.do%22)
actions = ActionChains(driver)
# change language to spanish
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "lgId")))
driver.find_element(By.ID, "lgId").click()
driver.find_element(By.XPATH, "//select[#id='lgId']/option[text()='español (es)']").click()
# click on "Busqueda avanzada"
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "goToSearch")))
driver.find_element(By.ID, "goToSearch").click()
# accept cookies and close tab
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "cookie-consent-banner")))
driver.find_element(By.XPATH, "//div[#id='cookie-consent-banner']/div[1]/div[1]/div[2]/a[1]").click()
driver.find_element(By.XPATH, "//div[#id='cookie-consent-banner']/div[1]/div[1]/div[2]/a[1]").click()
# click on specific date and set to today
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "publicationDateSpecific")))
element = driver.find_element(By.ID, "publicationDateSpecific")
actions.move_to_element(element).perform()
driver.find_element(By.ID, "publicationDateSpecific").click()
driver.find_element(By.CLASS_NAME, "ui-state-highlight").click()
# click on search
driver.find_element(By.ID, "search").click()
From the imports the only think I need to explain is that from the line from dirver import * comes the method get_driver_from_url() that is used later in the code. This method looks like this:
def get_driver_from_url(url):
options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--start-maximized")
options.add_argument("--headless")
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
driver.get(url)
return driver
As I said this code works perfectly without the headless mode, but when activated I get the error.
At first got another error and searching on the Internet found out that it could be because the element is not on screen, so I added the argument "--start-maximized" to make sure the Chrome tab is as big as possible and added the ActionChains to use actions.move_to_element(element).perform(), but I get this error on this exact code line.
Also tried changing the line WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "publicationDateSpecific"))) to WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "publicationDateSpecific"))) but it just didn't work.
Update: Also tried changing to EC.visibility_of_element_located as mentioned in this post but didn't work either
What am I doing wrong?

This is probably because of the window size.
Try adding this:
chrome_options = Options()
chrome_options.add_argument("--window-size=1920,1080")
chrome_options.add_argument("--start-maximized")
chrome_options.add_argument("--headless")

So, after a long time of try and error, I found that adding
element = driver.find_element(By.ID, "publicationDateSpecific")
driver.execute_script("window.scrollTo(0,"+str(element.location["y"])+")")
makes the script work both in headless mode and normal mode

Click share button under youtube video with selenium

import time
from selenium import webdriver
from selenium.webdriver.common.by import By
import pyautogui
titleVideo = input("Enter the title of the video: ")
chrome_options = webdriver.ChromeOptions()
prefs = {"profile.default_content_setting_values.notifications": 2}
chrome_options.add_experimental_option("prefs", prefs)
# Add experimental options to remove "Google Chrome is controlled by automated software" notification
chrome_options.add_experimental_option("useAutomationExtension", False)
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
driver = webdriver.Chrome(r'C:\Users\iwanh\Desktop\Drivers\chromedriver.exe', options=chrome_options)
driver.get("https://www.youtube.com/")
# We use driver.find_element with the help of the By import instead of find_element_by_name or id
accept_all = driver.find_element(By.XPATH, '/html/body/ytd-app/ytd-consent-bump-v2-lightbox/tp-yt-paper-dialog/div['
'4]/div/div[6]/div[1]/ytd-button-renderer['
'2]/a/tp-yt-paper-button/yt-formatted-string')
accept_all.click()
time.sleep(5)
search_box = driver.find_element(By.XPATH, value='/html/body/ytd-app/div[1]/div/ytd-masthead/div[3]/div['
'2]/ytd-searchbox/form/div[1]/div[1]/input')
search_box.send_keys(titleVideo)
searchGo = driver.find_element(By.XPATH, value='/html/body/ytd-app/div[1]/div/ytd-masthead/div[3]/div['
'2]/ytd-searchbox/button/yt-icon')
searchGo.click()
time.sleep(3)
pyautogui.press('tab')
pyautogui.press('tab') # Slopppy way to click on the first recommended
pyautogui.press('enter')# video will fix later
time.sleep(3)
shareButton = driver.find_element(By.XPATH, "/html/body/ytd-app/div[1]/ytd-page-manager/ytd-watch-flexy/div[5]/div["
"1]/div/ytd-watch-metadata/div/div[2]/div["
"2]/div/div/ytd-menu-renderer/div[1]/ytd-button-renderer["
"1]/a/yt-icon-button/yt-interaction")
time.sleep(3)
shareButton.click()
time.sleep(2.5)
copyButton = driver.find_element(By.XPATH, value="/html/body/ytd-app/ytd-popup-container/tp-yt-paper-dialog["
"1]/ytd-unified-share-panel-renderer/div["
"2]/yt-third-party-network-section-renderer/div["
"2]/yt-copy-link-renderer/div/yt-button-renderer/a/tp-yt-paper"
"-button/yt-formatted-string")
copyButton.click()
When it executes the shareButton part it says that it is "Unable to locate element". Two things that I am suspiscious might be happening.
I am not copying the right XPATH
The XPATH changes everytime I open a new chrome tab OR everytime i rerun the program.
Output:
Share button I want its XPATH:
P.S. I have tried to find element with other ways than XPATH but same result, if someones manages to do it with another way, its perfect.

you can click the button to share with:
driver.find_element(By.XPATH, value='//*[#id="top-level-buttons-computed"]/ytd-button-renderer[1]/a').click()
though a major issue is that the element is not loaded on the page when going to the page first. So you have to wait for that share element to load you could do that a couple of ways using: (easier method)
import time
time.sleep(10) # to wait 10 seconds
or (the recommended way)
Wait until page is loaded with Selenium WebDriver for Python

Python Selenium "Unable to locate element"

I am trying to use Selenium to sign up an email account automatically whenever I need to. It's just a fun learning project for me. For the life of me I don't understand why it can't find the element. This code works fine on the sign-in page but not the sign-up page. I have tried all different Selenium commands and even tried using the ID and class name. Either is says it can't locate the element or that it is not reachable by keyboard.
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
import time
options = Options()
driver = webdriver.Firefox(options=options, executable_path=r'geckodriver.exe')
driver.get("https://mail.protonmail.com/create/new?language=en")
time.sleep(10)
username_input = driver.find_element_by_id("username").send_keys("testusername")
Also here is the HTML code: https://i.imgur.com/ZaBMTzG.png

The username field is in iframe, you need to switch to iframe to make this work.
Below is the code that works fine :
driver.get("https://mail.protonmail.com/create/new?language=en")
driver.switch_to.frame(driver.find_element_by_css_selector("iframe[title='Registration form'][class='top']"))
driver.find_element_by_id("username").send_keys("some string")
read more about iframe here
learn more about how to switch to iframe/frame/framset using Python
selenium Bindings here
Update :
wait = WebDriverWait(driver, 30)
driver.get("https://mail.protonmail.com/create/new?language=en")
driver.switch_to.frame(driver.find_element_by_css_selector("iframe[title='Registration form'][class='top']"))
driver.find_element_by_id("username").send_keys("some string")
driver.switch_to.default_content()
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
sleep(5)
driver.switch_to.frame(driver.find_element_by_css_selector("iframe[title='Registration form'][class='bottom']"))
wait.until(EC.element_to_be_clickable((By.NAME, "submitBtn"))).click()

I'm not sure if I've seen enough code to diagnose, but I think the way you are defining username_input seems problematic. driver.find_element_by_id("username").send_keys("testusername") doesn't actually return anything so it seems like you are setting username_input = null.

How can I paste a string to textarea in browser with python?

I want to send a string to the web page whose text field name is "inputfield". Actually, I can send the word to the page, but when I run the program, a new "chrome" page opens, which is used for testing purposes. However, I want to send a string to the field on a chrome page that is already open.
Here my code:
from selenium import webdriver
import time
url = "https://10fastfingers.com/typing-test/turkish"
options = webdriver.ChromeOptions()
options.binary_location = r"C://Program Files//Google//Chrome//Application//chrome.exe"
chrome_driver_binary = 'chromedriver.exe'
options.add_argument('headless')
driver = webdriver.Chrome(chrome_driver_binary, options=options)
driver.get(url)
driver.implicitly_wait(10)
text_area = driver.find_element_by_id('inputfield')
text_area.send_keys("Hello")
Nothing happens when I run this code. Can you please help? Can you run it by putting a sample web page in the url part?
Thank you.
EDIT: It is working when I deleted options. But still opening a new page when I run it. Is there a way use a page which already open on background.
chrome_driver_binary = 'chromedriver.exe'
driver = webdriver.Chrome(chrome_driver_binary)
driver.get('https://10fastfingers.com/typing-test/turkish')
text_area = driver.find_element_by_id('inputfield')
text_area.send_keys("Hello")

Click the popup prior to sending keys.
driver.get('https://10fastfingers.com/typing-test/turkish')
wait=WebDriverWait(driver, 10)
wait.until(EC.element_to_be_clickable((By.ID, "CybotCookiebotDialogBodyLevelButtonLevelOptinAllowallSelectionWrapper"))).click()
text_area = wait.until(EC.element_to_be_clickable((By.ID, "inputfield")))
text_area.send_keys("Hello")
Imports
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

I am not sure what is your question but if the issue is multiple tabs or windows being opened then:
you can switch between the windows as:
// you can move to specific handle
chwd = driver.window_handles
print(chwd)
driver.switch_to.window(chwd[-1])
you should shoul switch to the correct window before you can interact with elements on that window
just switch to the window that was already opened bypassing the index
If the problem is that you want to interact with an already opened chrome then you should follow below steps:
Start chrome with debug port:
<path>\chrome.exe" --remote-debugging-port=1559
Python :
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:1559")
driver = webdriver.Chrome(options=chrome_options)

How to click on confirmation button using Selenium with Python?

I have the following Code that goes to a URL(www.example.com), and clicks on a link(Example 1). (This part works fine)
from selenium import webdriver
driver = webdriver.Firefox()
driver.get("https://www.example.com")
link = driver.find_element_by_link_text('Example 1')
link.click()
Now, when we click on 'Example 1' link, it opens a confirmation window, with 2 buttons: 'Yes I am authorized user to this site' and 'No I am a new visitor to this site'
So, I wish to click on 'Yes I am authorized user to this site' and then finally enter my log-in credentials.
I have written these 2 lines, just below the above code, for clicking on that button. But these don't work.
button = driver.find_element_by_name("'Yes I am authorized user to this site'")
button.click()

If it is an alert window, you need to use the Alert command.
#import Alert
from selenium.webdriver.common.alert import Alert
from selenium import webdriver
driver = webdriver.Firefox()
driver.get("https://www.example.com")
link = driver.find_element_by_link_text('Example 1')
link.click()
Alert(driver).accept()
#to dismiss alert
#Alert(driver).dismiss()
I think this would have solved your query.

Based on the comment conversation, I would recommend both using an XPATH search (instead of Name or Id) and waiting for elements to be clickable or loaded. When web-driving or web-scraping, pages may intentionally or accidentally load slowly and this can cause issues if you have pauses or waits either hard coded or non-existent. This snippet of code should allow you to search Google using Selenium and Chromedriver (you can modify the driver function to use Firefox or something else if you'd like):
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import ElementNotVisibleException
from selenium.webdriver.chrome.options import Options
from time import sleep
def init_driver(drvr_path):
chrome_options = Options()
chrome_options.add_argument("--disable-extensions")
driver = webdriver.Chrome(drvr_path+'chromedriver.exe',chrome_options=chrome_options)
driver.wait = WebDriverWait(driver, 5)
return driver
def lookup(query, driver=None, drvr_path=''):
driver = None
if driver is None:
driver = init_driver(drvr_path)
driver.implicitly_wait(45) # Allow up to 45 Seconds for page to load
driver.get("http://www.google.com")
try:
box = driver.wait.until(EC.presence_of_element_located((By.XPATH, """//*[#id="lst-ib"]""")))
box.send_keys(query)
sleep(3) # Let you see the window open
button = driver.wait.until(EC.element_to_be_clickable((By.XPATH,"""//*[#id="sblsbb"]/button""")))
try:
button.click()
except ElementNotVisibleException, s:
print "Error Handled: "+str(s)
button = driver.wait.until(EC.element_to_be_clickable((By.XPATH,"""//*[#id="sblsbb"]/button""")))
try:
button.click()
except:
print "Could not search Google..."
return
resp=driver.page_source.encode('utf-8')
with open(query+'.html','wb') as f:
f.write(resp)
print 'Wrote the File...'
except:
print("Box or Button not found in google.com")
driver.quit()
For example, if your Chromedriver.exe file was located in your default Python path, you could do something like: lookup('Selenium Python XPATH Examples') and it should download an HTML file of the Google Search results. If you already have a Driver initialized, you could of course pass that to it.
Hope this helps

Try this code, hope it will help you
from selenium import webdriver
import time
driver = webdriver.Chrome('path to chromedriver\chromedriver.exe')
driver.get('https://www.example.com')
driver.maximize_window()
link = driver.find_element_by_link_text('Example 1')
link.click()
handles =driver.window_handles # this will give window handles
driver.switch_to.window(handles[1])
button = driver.find_element_by_name("'Yes I am authorized user to this site'")
button.click()