I am trying to write a program that loads a webpage in Selenium and then scroll to the bottom of that page. Currently, my program can load the page but cannot scroll to the bottom of it. Below is my full code. Any help will be greatly appreciated, thanks!
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait as wait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
from selenium.webdriver.chrome.options import Options
from pynput.mouse import Button, Controller
import time
mouse = Controller()
chrome_options = Options()
chrome_options.add_argument("--user-data-dir=chrome-data")
driver = webdriver.Chrome(r"C:\Users\Brian\Downloads\chromedriver.exe", options=chrome_options)
driver.get('https://www.tradingview.com/chart/lUsimB6z/')
time.sleep(5)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
#open("source.html", "w").write(str(html))
I have tried below code and its workingf for me
driver.maximize_window()
wait = WebDriverWait(driver, 10)
driver.get("https://www.tradingview.com/chart/lUsimB6z/")
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
output:
Related
I am finding it difficult to switch to iframe and click on O/U on this page. I need help with this guys!
Here is my code below;
from random import *
import random
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support.ui import Select
from selenium.webdriver import ActionChains
from selenium.webdriver.chrome.options import Options
driver.get("https://www.sportybet.com/ke/sporty-instant-virtuals/")
driver.maximize_window()
instantplay= driver.find_element(By.XPATH, "//span[text()='Instant Virtuals']")
instantplay.click()
# find the frame
frame2 = driver.find_element(By.XPATH, "//iframe[#id='instantwin-sport']")
time.sleep(3)
driver.switch_to.frame(frame2)
time.sleep(3)
# Click O/U
driver.find_element(By.XPATH, "//li[text()[normalize-space()='O/U']]")
driver.quit()
You never click on the element. Add .click() after finding the element.
You can also remove the sleep before the iframe and put an implicit wait to wait on the O/U button
driver.get("https://www.sportybet.com/ke/sporty-instant-virtuals/")
driver.maximize_window()
instantplay= driver.find_element(By.XPATH, "//span[text()='Instant Virtuals']")
instantplay.click()
# find the frame
frame2 = driver.find_element(By.XPATH, "//iframe[#id='instantwin-sport']")
driver.switch_to.frame(frame2)
# Click O/U
driver.implicitly_wait(10)
driver.find_element(By.XPATH, "//li[text()[normalize-space()='O/U']]").click()
I can't execute selenium code on the iframe page on streamlit. When I run the .py file by executing streamlit run app5.py on cmd, it generates two browsers with one being the streamlit app (localhost:8501) and the other one being an empty page with an alert message on the top saying "Chrome is being controlled by automated test software." My guess for the reason why selenium code is not running is because it's executing selenium on the empty page not on the streamlit page. Is there a way to remove or not populate the empty page?
Here's my code:
import streamlit as st
from dateutil.parser import parse
import streamlit.components.v1 as components
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
st.title("Auto Search App")
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 20)
components.iframe("https://wego.here.com/", height = 500)
search_input = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "input.input_search")))
search_input.click()
search_input.send_keys('Seattle')
search_input.send_keys(Keys.ENTER)
Here is the screenshot of the two pages (empty page is generated most recently) when I execute streamlit run app5.py on cmd:
Run selenium headless.
Code:
import streamlit as st
from dateutil.parser import parse
import streamlit.components.v1 as components
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
driver = webdriver.Chrome(options=chrome_options)
wait = WebDriverWait(driver, 20)
components.iframe("https://wego.here.com/", height = 500)
search_input = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "input.input_search")))
search_input.click()
search_input.send_keys('Seattle')
search_input.send_keys(Keys.ENTER)
You can do it by using window_handles and switch_to_window method.
Before clicking the iframe first store the window handle
See if it works
import streamlit as st
from dateutil.parser import parse
import streamlit.components.v1 as components
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
st.title("Auto Search App")
driver = webdriver.Chrome()
wait = WebDriverWait(driver, 20)
#Switch to the other window
window_before = driver.window_handles[0]
driver.switch_to_window(window_before)
components.iframe("https://wego.here.com/", height = 500)
search_input = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "input.input_search")))
search_input.click()
search_input.send_keys('Seattle')
search_input.send_keys(Keys.ENTER)
I'm trying to click on the "next page" button using selenium, but I'm having no success. Am I using the right CSS selector or should I change it to something else?
from bs4 import BeautifulSoup
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
import pandas as pd
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument("--enable-javascript")
options.add_argument('--no-sandbox')
options.add_argument("window-size=1200x600")
driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)
driver.get(f'https://www.stakingrewards.com/cryptoassets')
driver.implicitly_wait(10)
button = driver.find_element(By.CSS_SELECTOR,"button[data-testid='next-page-button']")
button.click()
driver.quit()
change
button = driver.find_element(By.CSS_SELECTOR,"button[data-testid='next-page-button']")
to
button = driver.find_element(By.XPATH, "//button[#data-testid='next-page-button']")
You were previously trying to select with an XPATH, but had it as CSS selector. It also needed relative pathing/corrections. Let me know if there's anything else I can help with or missed.
I am using selenium and Python to scrape a website.I am not able to scrape particular table using Beautiful Soup.
Here is the code
from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from bs4 import BeautifulSoup
import pandas as pd
from bs4 import BeautifulSoup
from selenium.webdriver.support import expected_conditions
link='http://omms.nic.in/#'
browser=webdriver.Firefox()
browser.get(link)
time.sleep(15)
WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div/div/ul/li[3]/a'))).click()
time.sleep(10)
WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div/div/ul/li[3]/ul/li[1]/ul/li[5]/a'))).click()
select_state=Select(browser.find_element_by_xpath('/html/body/div[2]/div[1]/form/table/tbody/tr[1]/td[3]/select'))
select_state.select_by_index(35)
WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[2]/div[1]/form/table/tbody/tr[3]/td[7]/input[1]'))).click()
select_district=Select(browser.find_element_by_xpath('/html/body/div[2]/div[1]/form/table/tbody/tr[1]/td[5]/select'))
options = [x.text for x in select_district.options]
select_district.select_by_index(3)
select_year=Select(browser.find_element_by_xpath('/html/body/div[2]/div[1]/form/table/tbody/tr[2]/td[3]/select'))
select_year.select_by_index(8)
time.sleep(10)
WebDriverWait(browser, 10).until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[2]/div[1]/form/table/tbody/tr[4]/td[3]/input'))).click()
time.sleep(5)
soup=BeautifulSoup(browser.page_source,"html.parser")
table=soup.find_all('table',attrs={'class':"A35402edea1d24691942da96210fa88a3382"})
data_name = pd.read_html(str(table))[0]
I am getting the error as No tables found
This is probably because the table you are looking for is not under browser.page_source. It is loaded from a separate iframe. We can switch to the frame and then get the source.
browser.switch_to.frame(driver.find_element_by_xpath("//*[#id='loadReport']//iframe"))
print(browser.page_source)
soup = BeautifulSoup(browser.page_source,"html.parser")
In case if you need to interact with the page again, You can switch back using
browser.switch_to.default_content()
Also consider increasing the time to wait for the table to load.
I am scraping one of the URLs like this which loads data via Ajax. When using Firefox it's able to scrape HTML but upon using PhantomJS it return:
<html><head></head><body></body></html>
Code is below:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import selenium.webdriver.support.ui as ui
import sys
import os
from time import sleep
driver = None
url = 'https://sports.bovada.lv/live-betting/event/2391243'
driver = webdriver.PhantomJS('/Setups/phantomjs-1.9.8-macosx/bin/phantomjs')
driver.set_window_size(1128, 768) # optional
driver.get(url)
wait = ui.WebDriverWait(driver, 3000)
sleep(40)
#wait.until(EC.staleness_of(driver.find_element_by_id("coupon")), 'visible')
html = driver.page_source
#userElement = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.ID, "coupon")))
print(html)
Update
Ok this is happening with every URL regardless of ajax or non-Ajax