I want to get some information from a website and Chrome should run in the background to fulfill that task. Down there you can see my code. It works so far and I get the desired output, but when I add the chrome_options so that Chrome is hidden I don't get the output anymore.
What's the problem and how can I fix this?
from selenium import webdriver
def get_stockname(wkn):
PATH = r"***placeholder***chromedriver.exe"
url = "***placeholder***"
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("headless")
browser = webdriver.Chrome(PATH, options=chrome_options)
browser.get(url)
search_box = browser.find_element_by_class_name('input-field__text-input')
search_box.send_keys(wkn)
search_box.submit()
name = browser.find_element_by_xpath("/html/body/div[2]/div[1]/div[2]/div[13]/div[2]/div[1]/h2").text
name = name[13:]
print(name)
try like that, but with replacing your url:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
url = "https://google.com"
chrome_options = Options()
chrome_options.add_argument("--headless")
browser = webdriver.Chrome(options=chrome_options)
browser.get(url)
search_box = browser.find_element_by_class_name('input-field__text-input')
search_box.send_keys(wkn)
search_box.submit()
name = browser.find_element_by_xpath("(//h2[#class='box-headline'])[2]").get_attribute('innerText')
name = name[13:]
print(name)
By the way, can you show DOM code snipped to make element more unique with such external relation or even URL and element what info you want to get out?
Related
I am using Selenium webdriver to fill out a form, the fields are good, but when I do a .submit using the login button, or do a .submit after the password field, the chrome browser will say "Unable to connect to internet. Please check your internet connection."
the internet is definitely up since I am able to open other websites on another browser (including a new session of chrome).
This is my code:
import time
import selenium
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = webdriver.ChromeOptions();
chrome_options.add_experimental_option("excludeSwitches", ['enable-automation']);
options = Options()
# options.add_argument('start-maximumized')
# options.add_argument('disable-infobars')
chrome_options.add_argument('no-sandbox')
PATH = "c:\scripts\chromedriver.exe"
# driver = webdriver.Chrome(executable_path=PATH)
driver = webdriver.Chrome(chrome_options=chrome_options,
executable_path='c:\scripts\chromedriver.exe')
driver.get('https://stupidwebsite.org')
time.sleep(1)
input_username = driver.find_element_by_id('username')
time.sleep(1)
input_username.send_keys("myself#email.com")
input_password = driver.find_element_by_id('password')
input_password.send_keys("mypassword#123")
input_password.submit()
# esubmit=driver.find_element_by_xpath
('/html/body/div/div[1]/div[2]/div/div/form/div[2]/div/button')
# esubmit = driver.find_element_by_name('MuiButton-label')
# esubmit.submit()
time.sleep(2)
# driver.quit()
I changed it a bit, and tried out google, and it works:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
#set chromodriver.exe path
driver = webdriver.Chrome(executable_path="C:\scripts\chromedriver.exe")
driver.implicitly_wait(0.5)
#launch URL
driver.get("https://www.google.com/")
#identify search box
m = driver.find_element_by_name("q")
#enter search text
m.send_keys("Tutorialspoint")
time.sleep(0.2)
#perform Google search with Keys.ENTER
m.send_keys(Keys.ENTER)
Please help me out. Thank you.
I am trying to create bot for ServiceNow. But Somehow Selenium is not able to see Javascript elements. I use method find_element_by_xpath. But element is not visible, like it is not render. I print page by .page_source. Javascript is not there. How to run Javascript to be visible by Selenium
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import Select
from selenium.webdriver.chrome.options import Options
import time
PATH = ".\\chromedriver.exe"
driver = webdriver.Chrome(PATH)
chrome_options = Options()
chrome_options.headless = True
chrome_options.add_argument("--enable-javascript")
driver = webdriver.Chrome(options=chrome_options)
driver = webdriver.PhantomJS()
driver.get(https://cannotsharepage.com/)
# driver.maximize_window()
sso = driver.find_element_by_xpath("//*[#id='bySelection']/div[2]").click()
searchClick = driver.find_element_by_xpath("/html/body/div[5]/div/div/header/div[1]/div/div[2]/div/div[4]/form/div/label/span").click()
ritNumber="RITM0179411"
searchText = driver.find_element_by_xpath("//*[#id='sysparm_search']").send_keys(ritNumber)
time.sleep(3)
SearchText =driver.find_element_by_xpath("//*[#id='sysparm_search']").send_keys(Keys.ENTER)
time.sleep(15)
approvers= driver.execute_script("//*[#id='tabs2_list']/span[3]/span")
approvers = driver.page_source
I have implemented full page screenshot by selenium webdriver.
The code as follow
import time
import os
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
jenkinsJobName = os.getenv("JOB_NAME")
url = "https://www.ccode.com/sg"
save_fn = "testResult.PNG"
option = webdriver.ChromeOptions()
option.add_argument('--headless')
option.add_argument('--disable-gpu')
option.add_argument("--window-size=1280,1024")
option.add_argument("--hide-scrollbars")
driver = webdriver.Chrome(chrome_options=option)
driver.get(url)
print(driver.title)
scroll_width = driver.execute_script('return document.body.parentNode.scrollWidth')
scroll_height = driver.execute_script('return document.body.parentNode.scrollHeight')
driver.set_window_size(scroll_width, scroll_height)
driver.save_screenshot(save_fn)
driver.quit()
that is working fine.
But I use below code to capture full page screenshot in mobile
import time
import os
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
jenkinsJobName = os.getenv("JOB_NAME")
url = "https://www.ccode.com/sg"
save_fn = "testResyyult.PNG"
option = webdriver.ChromeOptions()
option.add_argument('--headless')
option.add_argument('--disable-gpu')
mobile_emulation = {"deviceName": "iPhone 6"}
option.add_experimental_option("mobileEmulation", mobile_emulation)
option.add_argument("--auto-open-devtools-for-tabs")
driver = webdriver.Chrome(chrome_options=option)
driver.get(url)
print(driver.title)
scroll_width = driver.execute_script('return document.body.parentNode.scrollWidth')
scroll_height = driver.execute_script('return document.body.parentNode.scrollHeight')
driver.set_window_size(scroll_width, scroll_height)
driver.save_screenshot(save_fn)
driver.quit()
The result image just take half of page, not for full page like in the first segement code.
enter image description here
How can I fix this code?
Thanks
You need to use Firefox for full-page screenshots:
https://selenium-python.readthedocs.io/api.html#module-selenium.webdriver.firefox.webdriver
Here is your code modified:
I have not run this code in context to your use case but I have used it as a part of my own project to get fullscreen screenshots, you will need to check the compatibility of other parameters that you have described with firefox.
import time
import os
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
jenkinsJobName = os.getenv("JOB_NAME")
url = "https://www.ccode.com/sg"
save_fn = "testResyyult.PNG"
option = webdriver.FirefoxOptions() # ------ Changes here
option.add_argument('--headless')
option.add_argument('--disable-gpu')
mobile_emulation = {"deviceName": "iPhone 6"}
option.add_experimental_option("mobileEmulation", mobile_emulation)
option.add_argument("--auto-open-devtools-for-tabs")
driver = webdriver.Firefox(options=option) # ------ Changes here
driver.get(url)
print(driver.title)
scroll_width = driver.execute_script('return document.body.parentNode.scrollWidth')
scroll_height = driver.execute_script('return document.body.parentNode.scrollHeight')
driver.set_window_size(scroll_width, scroll_height)
driver.save_full_page_screenshot(save_fn) # ------ Changes here This is the main change, search for this method in the documentation
driver.quit()
Also you might want to check the URL that you are taking a screenshot of. Seems like it is unresponsive.
Edit: I also noticed that you want a mobile view. Unfortunately, I think firefox does not return a valid mobile view even after setting
option.enable_mobile
While navigating to the page, It is able to extract the data.
Code:
from selenium.webdriver.chrome.options import Options
options1 = Options()
options1.headless = True
driver = webdriver.Chrome(os.getcwd() +"/chromedriver",options = options1)
this is how I am getting the chrome driver
content = BeautifulSoup(driver.page_source,"html.parser")
this is how I am getting the content from the navigated page. While we are navigating to the page able to extract the data.
Can you try it like below?
from selenium import webdriver
options = webdriver.ChromeOptions()
options.add_argument('headless')
driver = webdriver.Chrome(os.getcwd() +"/chromedriver", chrome_options=options)
I'm using selenium with Chrome driver; How can I get the page source, without showing the page opened? What I should specify in webdriver.ChromeOptions()?
Here the code:
from selenium.common.exceptions import WebDriverException
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("???")
bowser = webdriver.Chrome(chrome_options=chrome_options)
browser = webdriver.Chrome()
try:
browser.get("www.google.com")
html_content = browser.page_source
#do stuff
browser.quit()
except WebDriverException:
print "Invalid URL"
You should not use ChromeDriver but some headless Webdriver like HtmlUnitDriver, explained here
If you are adamant to use selenium, then you can use any of the headless browsers such as htmlunit driver.
Else you can can just send a get request on the URL and get the response text.
Selenium / Chrome has a headless option, which allows you to load webpages from code:
from selenium.webdriver import Chrome
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--headless")
browser = Chrome(options=chrome_options, executable_path='path_to_chromedriver')
browser.get('https://wwww.mywebsite.com')