import time
from selenium import webdriver
from selenium.webdriver.common.by import By
chrome_options = webdriver.ChromeOptions()
prefs = {"profile.default_content_setting_values.notifications": 2}
chrome_options.add_experimental_option("prefs", prefs)
# Add experimental options to remove "Google Chrome is controlled by automated software" notification
chrome_options.add_experimental_option("useAutomationExtension", False)
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
driver = webdriver.Chrome(r'C:\Users\iwanh\Desktop\Drivers\chromedriver.exe', options=chrome_options)
driver.get("https://www.youtube.com/")
# We use driver.find_element with the help of the By import instead of find_element_by_name or id
accept_all = driver.find_element(By.ID, value="text")
time.sleep(7)
accept_all.submit()
So i have this code, and what i am trying to do is press the accept all button on the screenshot but I think the code doesnt recognize it with the id because it is some what of a popup window?
And here is the inspect element
Inspect Element
Screenshot
Right-click on the element and go to Copy > Copy (full) xpath.
Then use:
from selenium.webdriver.common.by import By
driver.find_elements(By.XPATH, 'xpath_value_here')
Edited to reflect the difference between 'copy xpath' and 'copy full xpath'; indeed, copy full xpath is the correct answer, as per the below comment.
Related
I'm trying to scrape the webpage ted.europa.eu using Python with Selenium to retrieve information from the tenders. The script is supposed to be executed once a day with the new publications. The problem I have is that navigating to the new tenders I need Selenium to apply a filter to get only the ones from the same day the script it's executed. I already have the script for this and works perfectly, the problem is that when I activate the headless mode I get the following error selenium.common.exceptions.ElementNotInteractableException: Message: element not interactable: [object HTMLInputElement] has no size and location
This is the code I have that applies the filter I need:
import sys
import time
import re
from datetime import datetime
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from dotenv import load_dotenv
load_dotenv("../../../../.env")
sys.path.append("../src")
sys.path.append("../../../../utils")
from driver import *
from lted import LTED
from runnable import *
# start
print('start...')
counter = 0
start = datetime.now()
# get driver
driver = get_driver_from_url("https://ted.europa.eu/TED/browse/browseByMap.do%22)
actions = ActionChains(driver)
# change language to spanish
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "lgId")))
driver.find_element(By.ID, "lgId").click()
driver.find_element(By.XPATH, "//select[#id='lgId']/option[text()='español (es)']").click()
# click on "Busqueda avanzada"
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "goToSearch")))
driver.find_element(By.ID, "goToSearch").click()
# accept cookies and close tab
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "cookie-consent-banner")))
driver.find_element(By.XPATH, "//div[#id='cookie-consent-banner']/div[1]/div[1]/div[2]/a[1]").click()
driver.find_element(By.XPATH, "//div[#id='cookie-consent-banner']/div[1]/div[1]/div[2]/a[1]").click()
# click on specific date and set to today
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "publicationDateSpecific")))
element = driver.find_element(By.ID, "publicationDateSpecific")
actions.move_to_element(element).perform()
driver.find_element(By.ID, "publicationDateSpecific").click()
driver.find_element(By.CLASS_NAME, "ui-state-highlight").click()
# click on search
driver.find_element(By.ID, "search").click()
From the imports the only think I need to explain is that from the line from dirver import * comes the method get_driver_from_url() that is used later in the code. This method looks like this:
def get_driver_from_url(url):
options = webdriver.ChromeOptions()
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
options.add_argument("--start-maximized")
options.add_argument("--headless")
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
driver.get(url)
return driver
As I said this code works perfectly without the headless mode, but when activated I get the error.
At first got another error and searching on the Internet found out that it could be because the element is not on screen, so I added the argument "--start-maximized" to make sure the Chrome tab is as big as possible and added the ActionChains to use actions.move_to_element(element).perform(), but I get this error on this exact code line.
Also tried changing the line WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "publicationDateSpecific"))) to WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "publicationDateSpecific"))) but it just didn't work.
Update: Also tried changing to EC.visibility_of_element_located as mentioned in this post but didn't work either
What am I doing wrong?
This is probably because of the window size.
Try adding this:
chrome_options = Options()
chrome_options.add_argument("--window-size=1920,1080")
chrome_options.add_argument("--start-maximized")
chrome_options.add_argument("--headless")
So, after a long time of try and error, I found that adding
element = driver.find_element(By.ID, "publicationDateSpecific")
driver.execute_script("window.scrollTo(0,"+str(element.location["y"])+")")
makes the script work both in headless mode and normal mode
I'm trying to download or rather read a csv from a website. The csv is hidden under a button. Here is the website link.
The csv I'm trying to save is on the furthest right and is denoted as a blue button labelled 'Download all data in .csv'
Here is a sample code I tried to use to download it, but the download fails:
# import the required libraries
from selenium import webdriver
from selenium.webdriver.common.by import By
options = webdriver.ChromeOptions()
prefs = {"download.default_directory" : "D:/Profession/Data Extraction and Web Scraping/Stocks Data Extraction - Core Scientific/Output"}
#example: prefs = {"download.default_directory" : "C:\Tutorial\down"};
options.add_experimental_option("prefs",prefs)
driver = webdriver.Chrome(executable_path="D:/Software/Selenium WebDrivers/chromedriver_win32/chromedriver", options=options)
try:
driver.implicitly_wait(5)
driver.get("https://defillama.com/chains")
downloadcsv = driver.find_element(By.CLASS_NAME, 'sc-8f0f10aa-1')
download_button = downloadcsv.find_element(By.TAG_NAME, 'button')
download_button.click() # this should save it to the folder I specified in prefs
time.sleep(3)
driver.close()
except:
print('There is an Error!')
This code tries to download the csv, however the download fails. Is there a better way I could download the CSV, especially one that is under a button? Thanks!
Looks like you need to use a proper wait method.
implicitly_wait waits for element presence while you need to wait for element to be clickable, this is more mature element state. WebDriverWait is the best practice to be used in such (and almost all the others) cases.
The following code works:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 10)
url = "https://defillama.com/chains"
driver.get(url)
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".sc-8f0f10aa-1 button"))).click()
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
import pyautogui
titleVideo = input("Enter the title of the video: ")
chrome_options = webdriver.ChromeOptions()
prefs = {"profile.default_content_setting_values.notifications": 2}
chrome_options.add_experimental_option("prefs", prefs)
# Add experimental options to remove "Google Chrome is controlled by automated software" notification
chrome_options.add_experimental_option("useAutomationExtension", False)
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
driver = webdriver.Chrome(r'C:\Users\iwanh\Desktop\Drivers\chromedriver.exe', options=chrome_options)
driver.get("https://www.youtube.com/")
# We use driver.find_element with the help of the By import instead of find_element_by_name or id
accept_all = driver.find_element(By.XPATH, '/html/body/ytd-app/ytd-consent-bump-v2-lightbox/tp-yt-paper-dialog/div['
'4]/div/div[6]/div[1]/ytd-button-renderer['
'2]/a/tp-yt-paper-button/yt-formatted-string')
accept_all.click()
time.sleep(5)
search_box = driver.find_element(By.XPATH, value='/html/body/ytd-app/div[1]/div/ytd-masthead/div[3]/div['
'2]/ytd-searchbox/form/div[1]/div[1]/input')
search_box.send_keys(titleVideo)
searchGo = driver.find_element(By.XPATH, value='/html/body/ytd-app/div[1]/div/ytd-masthead/div[3]/div['
'2]/ytd-searchbox/button/yt-icon')
searchGo.click()
time.sleep(3)
pyautogui.press('tab')
pyautogui.press('tab') # Slopppy way to click on the first recommended
pyautogui.press('enter')# video will fix later
time.sleep(3)
shareButton = driver.find_element(By.XPATH, "/html/body/ytd-app/div[1]/ytd-page-manager/ytd-watch-flexy/div[5]/div["
"1]/div/ytd-watch-metadata/div/div[2]/div["
"2]/div/div/ytd-menu-renderer/div[1]/ytd-button-renderer["
"1]/a/yt-icon-button/yt-interaction")
time.sleep(3)
shareButton.click()
time.sleep(2.5)
copyButton = driver.find_element(By.XPATH, value="/html/body/ytd-app/ytd-popup-container/tp-yt-paper-dialog["
"1]/ytd-unified-share-panel-renderer/div["
"2]/yt-third-party-network-section-renderer/div["
"2]/yt-copy-link-renderer/div/yt-button-renderer/a/tp-yt-paper"
"-button/yt-formatted-string")
copyButton.click()
When it executes the shareButton part it says that it is "Unable to locate element". Two things that I am suspiscious might be happening.
I am not copying the right XPATH
The XPATH changes everytime I open a new chrome tab OR everytime i rerun the program.
Output:
Share button I want its XPATH:
P.S. I have tried to find element with other ways than XPATH but same result, if someones manages to do it with another way, its perfect.
you can click the button to share with:
driver.find_element(By.XPATH, value='//*[#id="top-level-buttons-computed"]/ytd-button-renderer[1]/a').click()
though a major issue is that the element is not loaded on the page when going to the page first. So you have to wait for that share element to load you could do that a couple of ways using: (easier method)
import time
time.sleep(10) # to wait 10 seconds
or (the recommended way)
Wait until page is loaded with Selenium WebDriver for Python
I want to send a string to the web page whose text field name is "inputfield". Actually, I can send the word to the page, but when I run the program, a new "chrome" page opens, which is used for testing purposes. However, I want to send a string to the field on a chrome page that is already open.
Here my code:
from selenium import webdriver
import time
url = "https://10fastfingers.com/typing-test/turkish"
options = webdriver.ChromeOptions()
options.binary_location = r"C://Program Files//Google//Chrome//Application//chrome.exe"
chrome_driver_binary = 'chromedriver.exe'
options.add_argument('headless')
driver = webdriver.Chrome(chrome_driver_binary, options=options)
driver.get(url)
driver.implicitly_wait(10)
text_area = driver.find_element_by_id('inputfield')
text_area.send_keys("Hello")
Nothing happens when I run this code. Can you please help? Can you run it by putting a sample web page in the url part?
Thank you.
EDIT: It is working when I deleted options. But still opening a new page when I run it. Is there a way use a page which already open on background.
chrome_driver_binary = 'chromedriver.exe'
driver = webdriver.Chrome(chrome_driver_binary)
driver.get('https://10fastfingers.com/typing-test/turkish')
text_area = driver.find_element_by_id('inputfield')
text_area.send_keys("Hello")
Click the popup prior to sending keys.
driver.get('https://10fastfingers.com/typing-test/turkish')
wait=WebDriverWait(driver, 10)
wait.until(EC.element_to_be_clickable((By.ID, "CybotCookiebotDialogBodyLevelButtonLevelOptinAllowallSelectionWrapper"))).click()
text_area = wait.until(EC.element_to_be_clickable((By.ID, "inputfield")))
text_area.send_keys("Hello")
Imports
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
I am not sure what is your question but if the issue is multiple tabs or windows being opened then:
you can switch between the windows as:
// you can move to specific handle
chwd = driver.window_handles
print(chwd)
driver.switch_to.window(chwd[-1])
you should shoul switch to the correct window before you can interact with elements on that window
just switch to the window that was already opened bypassing the index
If the problem is that you want to interact with an already opened chrome then you should follow below steps:
Start chrome with debug port:
<path>\chrome.exe" --remote-debugging-port=1559
Python :
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:1559")
driver = webdriver.Chrome(options=chrome_options)
there´s any python component
to do someting like this(this is java)
https://jxbrowser.support.teamdev.com/support/solutions/articles/9000013135-jxbrowser-selenium
i use python tkinter to open chrome instances on a click button,
i'd like to run selenium chrome instance in a python widget on a click button,
at the top of python gui app a button and when you click on it open the selenium chrome instance in a tkinter frame
is it possible to do something like that with a python GUI
thanks a lot
Yes you can, I would try something like this:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import WebDriverException
#Your paths and driver might be different.
CHROME_PATH = 'C:\Program Files (x86)\Google\Chrome\Application\chrome.exe'
CHROMEDRIVER_PATH = 'chromedriver.exe'
WINDOW_SIZE = "1920,1080"
chrome_options = Options()
chrome_options.add_argument("--log-level=3")
chrome_options.add_argument("--headless") # This is optional, but faster than gui
chrome_options.add_argument("--window-size=%s" % WINDOW_SIZE)
chrome_options.binary_location = CHROME_PATH
browser = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, chrome_options=chrome_options)
url = "https://www.example.com" # Your url goes here.
browser.get(url)
# - You'll need to copy the button's xpath and place it in here.
element = WebDriverWait(browser, 10).until(
EC.presence_of_element_located((By.XPATH, 'copy_xpath_into_here')))
# click on button - You'll need to copy the button's xpath and place it in here, too.
selectElem=browser.find_element_by_xpath('copy_xpath_into_here').click()