We Keep Coding

Trying to get Selenium to scrape table and click button

I hacked together the code below to try to scrape data from an HTML table, to a data frame, and then click a button to move to the next page, but it's giving me an error tat says 'invalid selector'.
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from bs4 import BeautifulSoup
import time
from time import sleep
import pandas as pd
browser = webdriver.Chrome("C:/Utility/chromedriver.exe")
wait = WebDriverWait(browser, 10)
url = 'https://healthdata.gov/dataset/Hospital-Detail-Map/tagw-nk32'
browser.get(url)
for x in range(1, 5950, 13):
time.sleep(3) # wait page open complete
df = pd.read_html(browser.find_element_by_xpath("socrata-table frozen-columns").get_attribute('outerHTML'))[0]
submit_button = browser.find_elements_by_xpath('pager-button-next')[0]
submit_button.click()
I see the table, but I can't reference it.
Any idea what's wrong here?

I've managed to find button with find_elements_by_css_selector
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from bs4 import BeautifulSoup
import time
from time import sleep
import pandas as pd
browser = webdriver.Chrome("C:/Utility/chromedriver.exe")
wait = WebDriverWait(browser, 10)
url = 'https://healthdata.gov/dataset/Hospital-Detail-Map/tagw-nk32'
browser.get(url)
for x in range(1, 5950, 13):
time.sleep(3) # wait page open complete
df = pd.read_html(
browser.find_element_by_xpath("socrata-table frozen-columns").get_attribute(
'outerHTML'))[0]
submit_button = browser.find_elements_by_css_selector('button.pager-button-next')[1]
submit_button.click()
Sometimes pagination hangs, and submit_button.click() ends with an error
selenium.common.exceptions.ElementClickInterceptedException:
Message: element click intercepted:
Element <button class="pager-button-next">...</button>
is not clickable at point (182, 637).
Other element would receive the click: <span class="site-name">...</span>
So consider to increase timeout. For example, you can use this approach
def click_timeout(element, timeout: int = 60):
for i in range(timeout):
time.sleep(1)
try:
element.click()
except WebDriverException:
pass
element.click()
So, you click an element as fast as it will be ready

python using selenium webdriver

I'm trying to open the "digikey" website and use the search bar to send some data. Here's an example of the code but it looks like the send key is not working. I need help. Thank you.
import time
from openpyxl import load_workbook
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
driver = webdriver.Chrome(executable_path='C:/Users/amuri/AppData/Local/Microsoft/WindowsApps/PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0/site-packages/chromedriver.exe')
driver.implicitly_wait(3)
url ='https://www.digikey.com/'
driver.get(url)
print(driver.title)
elem = driver.find_element_by_xpath("/html/body/header/div[2]/div[1]/div/div[2]/div[2]/input")
elem.click()
elem.send_keys("myString")
print(elem.text)

For elem use css selector, not long xpath:
elem = driver.find_element_by_css_selector("#main-layout-content .header__searchinput")
header__searchinput class is not unique, that's why I used main-layout-content id to as a parent.
Also add explicit wait:
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
wait = WebDriverWait(driver, timeout=30)
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#main-layout-content .header__searchinput")))
elem = driver.find_element_by_css_selector("#main-layout-content .header__searchinput")
elem.click()
elem.send_keys("myString")

Why does my Selenium URL_to_be statement not work?

Selenium does not seem to register that I manually go to the publish0x.com page.
Does anyone know a solution?
My goal is to manually do the captcha at the login page and afterwards, when I log in and land on the main page I want the script to resume.
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import time
import datetime
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import sys
from selenium.webdriver.support.ui import WebDriverWait
def waitForLoad(inputXPath):
Wait = WebDriverWait(driver, 10)
Wait.until(EC.presence_of_element_located((By.XPATH, inputXPath)))
email = '
password = '
options = Options()
options.add_experimental_option("detach", True)
options.add_argument("--window-size=1920,1080")
## options.add_argument("user-data-dir=/Users/vadim/Library/Application Support/BraveSoftware/Brave-Browser")
options.binary_location = '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser'
driver_path = '/usr/local/bin/chromedriver'
driver = webdriver.Chrome(options=options, executable_path=driver_path)
driver.get('https://www.publish0x.com/login')
waitForLoad('//*[#id="email"]')
E_Mail_vak = driver.find_element_by_xpath('//*[#id="email"]')
E_Mail_vak.send_keys(email)
Pass_vak = driver.find_element_by_xpath('//*[#id="password"]')
Pass_vak.send_keys(password)
frame = driver.find_element_by_xpath('//iframe[contains(#src, "recaptcha")]')
driver.switch_to.frame(frame)
Captcha = driver.find_element_by_xpath("//*[#id='recaptcha-anchor']")
Captcha.click()
wait = WebDriverWait(driver, 500)
wait.until(EC.url_to_be("publish0x.com"))
driver.get('https://www.publish0x.com/newposts')
post = driver.find_element_by_css_selector('#main > div.infinite-scroll > div:nth-child(1) > div.content')
title = post.find_element_by_css_selector('h2 > a').text
author = post.find_element_by_css_selector('p.text-secondary > small:nth-child(4) > a').text
title.click()
slider = driver.find_element_by_xpath('//*[#id="tipslider"]')

There are two ways I can think of, one by adding an Input statement like this:
options.add_argument('--disable-gpu')#For properly seeing the outputs
input("Please do the captcha and press any key...)
In this way, the user would complete the data and then press any key for the script to continue.
The other way is by adding a try and except statement.
try:
driver.find_element_by_id("some-element")
except NoSuchElementException:
#do something like
print("Captcha Failed or Incomplete...")
In this, replace the element id "some-element" with any element that is present and only present after the user logs in, for e.g elements like Profile_nav or Settings are only present when someone logs in. So if the element doesn't exist then it would mean that the user didn't complete the captcha.

youtube page scroll down using selenium, python

url_main = "https://www.youtube.com/feed/history"
I want to scroll down infinitely in the webpage above until all contents are visible, so I tried
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
import time
driver = webdriver.Chrome(driverpath)
wait = WebDriverWait(driver, 10)
driver.get(url_main)
login = driver.find_element_by_css_selector('a[href="https://accounts.google.com/ServiceLogin?service=youtube&uilel=3&passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Faction_handle_signin%3Dtrue%26app%3Ddesktop%26hl%3Dko%26next%3Dhttps%253A%252F%252Fwww.youtube.com%252Ffeed%252Fhistory&hl=ko&ec=65620"]')
login.click()
login = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR,'input[type="email"]')))
login.send_keys(email)
login.send_keys(Keys.RETURN)
login = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR,'input[type="password"]')))
login.send_keys(password)
login.send_keys(Keys.RETURN)
content = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR,'div[id="primary"] div[id="contents"]')))
no_of_pagedowns = 1
while no_of_pagedowns:
content.send_keys(Keys.PAGE_DOWN)
time.sleep(1)
no_of_pagedowns-=1
But not working. I also tried copy-pasting the code in this link but it didn't work at all also.

You can implement something like this:
# save the max scrollHeight of the page
scrollHeight = d.execute_script("return window.scrollMaxY")
scrolled_pages = 0
# while we have not reached the max scrollHeight
while d.execute_script("return window.pageYOffset") < scrollHeight:
# scroll one page down
d.execute_script("window.scrollByPages(1)")
scrolled_pages += 1
# wait to load any lazy loaded images (may not be needed, depending on your usecase)
time.sleep(0.2)

Double clicking in python selenium

I am using selenium with python. Im able to get the code below to click where I want but I want it to dbl click. Im not very good with the action chains and I know I need that for dbl click. Can anyone help with what I need to change around?
user = self.find_element_by_id("selUsers")
for option in user.find_elements_by_tag_name("option"):
if option.text == "Admin, Ascender":
option.click()

Action chains is the only best option as far i know
from selenium.webdriver.common.action_chains import ActionChains
driver=self.webdriver
user = self.find_element_by_id("selUsers")
for option in user.find_elements_by_tag_name("option"):
if option.text == "Admin, Ascender":
actionChains = ActionChains(driver)
actionChains.double_click(option).perform()

try this:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from time import sleep
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
usernameStr = 'xxxxxx'
passwordStr = 'xxxxx'
browser = webdriver.Chrome()
browser.get(('https://accounts.google.com/ServiceLogin?'
'service=mail&continue=https://mail.google'
'.com/mail/#identifier'))
username = browser.find_element_by_id('identifierId')
username.send_keys(usernameStr)
nextButton = browser.find_element_by_id('identifierNext')
nextButton.click()
# wait for transition then continue to fill items
password = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.XPATH,'//* [#id="password"]/div[1]/div/div[1]/input')))
password.send_keys(passwordStr)
signInButton = browser.find_element_by_id('passwordNext')
signInButton.click()
apsButton = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.XPATH,'//[#id="gbwa"]/div/a')))
apsButton.click()
driveButton = WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.XPATH,'//*[#id="gb49"]/span[1]')))
driveButton.click()