I am trying to use selenium webdriver in centos to test my webpage.
But,I got an error message when I execute the process.
Can someone help me?
from pyvirtualdisplay import Display
from selenium import webdriver
display=Display(visible=0, size=(320, 240)).start()
path = "/usr/bin/firefox"
driver= webdriver.Firefox(path)
driver.get("www.google.com")
html_source = driver.page_source
print html_source
driver.close()
And here is the error message:
File "/var/www/test/test.py", line 19, in <module>
driver= webdriver.Firefox(path)
File "/usr/local/lib/python2.7/site-packages/selenium/webdriver/firefox/webdriver.py", line 54, in __init__
self.NATIVE_EVENTS_ALLOWED and self.profile.native_events_enabled)
AttributeError: 'str' object has no attribute 'native_events_enabled'
Pretty sure your problem has to do with the fact that your trying to pass the path to your firefox binary as a string, instead as a "FirefoxBinary" object, furthermore the first argument to Firefox() is a FirefoxProfile(). Doing the following should resolve the issue.
from pyvirtualdisplay import Display
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
display=Display(visible=0, size=(320, 240)).start()
binary = FirefoxBinary("/usr/bin/firefox")
driver= webdriver.Firefox(firefox_binary=binary)
driver.get("www.google.com")
html_source = driver.page_source
print html_source
driver.close()
see this post for an answer to a very similar problem.
Related
I am working on amazon web scraping script in python3 so I used selenium but I got this debug
webdriver.chrome()
TypeError: 'module' object is not callable
I saw solutions to change (chrome to Chrome) but I got this debug also
FileNotFoundError: [WinError 2] The system cannot find the file specified
this is my code
# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
driver = webdriver.chrome()
driver.get('https://www.amazon.com/international-sales-offers/b/?ie=UTF8&node=15529609011&ref_=nav_navm_intl_deal_btn')
res = driver.execute_script("return document.documentElement.outerHTML")
driver.quit()
soup = BeautifulSoup(res , 'lxml')
box= soup.find('div',{'class':'a-row padCenterContainer widgetBorder'})
products=box.find_all('div',{'class':'a-section a-spacing-none tallCellView gridColumn5 singleCell'})
for details in products:
name= details.find('span',{'class':'a-declarative'}).text
link= details.find('a',{'class':'a-size-base a-link-normal dealTitleTwoLine singleCellTitle autoHeight'}).get('href')
print(name,link)
I believe it should be Chrome(), not chrome(). Try:
from selenium import webdriver
driver = webdriver.Chrome()
You can pass the path to your Chromedriver as well, set executable_path to the location where your chromedriver is located (path to chromedriver.exe or, for non-Windows users it's just called chromedriver):
driver = webdriver.Chrome(executable_path='C:/path/to/chromedriver.exe')
I'm trying to start webscraping, but whenever I try to acces an URL I get an error message.
My code is the following:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
driver = webdriver.Chrome()
driver.get('www.python.org')
This opens a new Chrome window, but that's all it does.
The error message I get is the following:
InvalidArgumentException: invalid argument
(Session info: chrome=80.0.3987.149)
I work with Spyder, which I get from Anaconda, and my chromedriver.exe is in the both in the Anaconda3 folder and the Spyder folder.
Thanks in advance!
This URL is not Valid , it has to start with http://
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
try:
driver = webdriver.Chrome()
driver.get('http://www.python.org')
except Exception as e:
print(e)
finally:
if driver is not None :
driver.close()
Please include executable path and try below solution:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
driver = webdriver.Chrome(executable_path=r"C:\New folder\chromedriver.exe")
driver.get("https://www.python.org")
Your url is not correct www.python.org , correct url : https://www.python.org
I have created simple basic automation script in Python using Selenium..
Getting unwanted exception.
File:-
import pandas as pd
from pandas import ExcelWriter
from selenium import webdriver
import selenium as sel
# Data = pd.read_excel(r"C:\Users\Admin\PycharmProjects\Web_Automation_Form_Filling\challenge.xlsx",sheet_name="Sheet1")
# browser = webdriver.Chrome(executable_path=r'C:\Program Files (x86)\Google\Chrome\Application\chrome.exe')
browser = webdriver.Chrome("C:\Program Files (x86)\Google\Chrome\Application\chrome.exe");
browser.sleep(1000);
browser.get("http://www.python.org")
Error log:-
C:\Users\Admin\PycharmProjects\Web_Automation_Form_Filling\venv\Scripts\python.exe C:/Users/Admin/PycharmProjects/Web_Automation_Form_Filling/venv/Web_Auto_Filling.py
Traceback (most recent call last):
File "C:/Users/Admin/PycharmProjects/Web_Automation_Form_Filling/venv/Web_Auto_Filling.py", line 10, in <module>
browser = webdriver.Chrome("C:\Program Files (x86)\Google\Chrome\Application\chrome.exe");
File "C:\Users\Admin\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\chrome\webdriver.py", line 73, in __init__
self.service.start()
File "C:\Users\Admin\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\common\service.py", line 98, in start
self.assert_process_still_running()
File "C:\Users\Admin\AppData\Local\Programs\Python\Python38-32\lib\site-packages\selenium\webdriver\common\service.py", line 109, in assert_process_still_running
raise WebDriverException(
selenium.common.exceptions.WebDriverException: Message: Service C:\Program Files (x86)\Google\Chrome\Application\chrome.exe unexpectedly exited. Status code was: 0
Process finished with exit code 1
Any suggestion will be appreciated..
Thanks...
instead of chrome application try providing the chrome driver instead
more information on the site : https://sites.google.com/a/chromium.org/chromedriver/getting-started
Sample code :
import time
from selenium import webdriver
driver = webdriver.Chrome('/path/to/chromedriver') # Optional argument, if not specified will search path.
driver.get('http://www.google.com/');
time.sleep(5) # Let the user actually see something!
search_box = driver.find_element_by_name('q')
search_box.send_keys('ChromeDriver')
search_box.submit()
time.sleep(5) # Let the user actually see something!
driver.quit()
Download the ChromeDriver binary for your platform under the downloads section of this site
reference link to download : chrome driver
This code should work (better to use firefox for selenium):
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
# noinspection PyUnresolvedReferences
import wget
DesiredCapabilities.PHANTOMJS[
"phantomjs.page.settings.userAgent"
] = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:16.0) Gecko/20121026 Firefox/16.0"
if browser == "firefox":
driver = webdriver.Firefox()
else:
driver = webdriver.PhantomJS(
CFG_phantomjs
) # r"D:/_devs/webserver/phantomjs-1.9.8/phantomjs.exe"
driver.get("https://tourwebsite")
username = driver.find_element_by_id("login_field")
password = driver.find_element_by_id("password")
username.clear()
The problem here in your codes is that you are passing chrome executable path rather than passing the path to chromedriver which is a different executable.
An appropriate version of chromedriver can be downloaded from here according to your Chrome version.
For more info, you can refer to the chromedriver documentation here.
And your final code should be something like:
from selenium import webdriver
path = 'C:/Users/Avinash/Downloads/chromedriver.exe'
driver = webdriver.Chrome(path)
driver.get('http://www.google.com/');
#..here what ever you want to do with page here
driver.quit()
how do I enable javascript in selenium using python? I tried several methods but mine isnt working. Does anyone know how could i fix this issue? Thanks
My code
from selenium import webdriver
import urllib
import urllib.request
import string
from bs4 import BeautifulSoup
import mysql.connector
import time
chrome_path = r"C:\chromedriver.exe"
driver = webdriver.Chrome(chrome_path)
driver.add_argument("--enable-javascript")
Error i got
Exception has occurred: AttributeError
'WebDriver' object has no attribute 'add_argument'
File "C:\Users\bustillo-ronald\Desktop\python-basics\Scrape\propertyguru.py", line 11, in <module>
driver.add_argument("--enable-javascript")
You need to create an instance of a ChromeOptions object first and add_argument to that. Then pass the options object as an argument to the Chrome webdriver.
something like this
options = webdriver.ChromeOptions()
options.add_argument("--enable-javascript")
# Now back to your code
driver = webdriver.Chrome(chrome_path, options=options)
...
I'm trying to write script to automate the process of downloading the instagram stories but I'm failing already when trying to log in.
I'm writing the code inside Pycharm. I just tried my usual approach to any problem. First, solve it with typing the commands out in the console and if it works writing the commands which worked inside the console down in a script. But here is the issue. The function which worked perfectly fine inside the python console fails inside the script.
I've noticed that my selenium was outdated but upgrading it didn't help ether. I also made a new project to test weather that made difference, which it didn't.
I've also tried skipping the first step inside the script and just opening the url to which I'm redirected. But the second commands failed as well.
When I create a new variable to store the output of the driver.find_element_by_link_text() in, it returns an empty list. This leads me to belive that somehow selenium is unable to search the contetns of the page.
I've also tried the same on Chrome and Safari. This also didn't work.
Here is the code:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
driver = webdriver.Firefox()
driver.get("https://instagram.com/")
#next command fails
driver.find_element_by_link_text("Melde dich an.").click()
#if the first command is skipped by entering in the url
#in driver.get(https://www.instagram.com/accounts/login/?source=auth_switcher)
#the following command fails as well.
driver.find_element_by_name("username").send_keys("HereIsTheUsername")
driver.find_element_by_name("password").send_keys("HereIsThePassword")
driver.find_element_by_name("password").send_keys(Keys.RETURN)
driver.close()
In the console these commands worked as mentioned,
Here is what I've entered into the console:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
driver = webdriver.Firefox()
driver.get("https://instagram.com/")
driver.find_element_by_link_text("Melde dich an.").click()
#if it failed here would be an error message
element = driver.find_element_by_name("username")
With the script the error message is this:
Traceback (most recent call last): File
"/Users/alisot2000/PycharmProjects/Instagram downloader/venv/Main.py",
line 6, in
driver.find_element_by_link_text("Melde dich an.").click() File "/Users/alisot2000/PycharmProjects/Instagram
downloader/venv/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py",
line 428, in find_element_by_link_text
return self.find_element(by=By.LINK_TEXT, value=link_text) File "/Users/alisot2000/PycharmProjects/Instagram
downloader/venv/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py",
line 978, in find_element
'value': value})['value'] File "/Users/alisot2000/PycharmProjects/Instagram
downloader/venv/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py",
line 321, in execute
self.error_handler.check_response(response) File "/Users/alisot2000/PycharmProjects/Instagram
downloader/venv/lib/python3.7/site-packages/selenium/webdriver/remote/errorhandler.py",
line 242, in check_response
raise exception_class(message, screen, stacktrace) selenium.common.exceptions.NoSuchElementException: Message: Unable to
locate element: Melde dich an.
Issues you might be experiencing:
1. Synchronization Issue
For most automation tasks, there will be different loading times of web pages based on the processing power of the machine and how strong your internet connection is.
To solve this there are library import Waits from selenium that we can use.
Here is a sample below:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Firefox()
driver.get("http://somedomain/url_that_delays_loading")
try:
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "myDynamicElement"))
)
finally:
driver.quit()
2. Wrong language set in selenium profile
Selenium will use your locale in most cases when running automation scripts but in the case that you might want another language here is a sample code for FireFox.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
profile = webdriver.FirefoxProfile()
# switch out 'de' with another two character language code
profile.set_preference("intl.accept_languages",'de')
driver = webdriver.Firefox(firefox_profile=profile, executable_path='<insert_your_gecko_driver_path_here>')
driver.get("https://instagram.com/")
driver.close()
3. Working Code(Tested on Mojave 10.14.5)
Here is a diff of your code and the altered code: https://www.diffchecker.com/G0WWB4Ry
setup a virtualenv
pip install selenium
download geckodriver
set path to gecko driver in code
run script with success result
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
# these two imports are for setting up firefox driver and options
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
# import these three lines below if you are having synchronization issues
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
profile = webdriver.FirefoxProfile()
# here is where you need to set your language explicitly if its defaulting to an undesired language
# just replace the second parameter with your 2 character language code
# this line is not needed if your desired language is locale
profile.set_preference("intl.accept_languages",'de')
# throw in your path here <insert_your_gecko_driver_path_here>
driver = webdriver.Firefox(firefox_profile=profile, executable_path='<insert_your_gecko_driver_path_here>')
driver.get("https://instagram.com/")
# added these two lines below to solve synchronization issue
# element wasnt clickable until page finished loading
wait = WebDriverWait(driver, 10)
element = wait.until(EC.element_to_be_clickable((By.LINK_TEXT, "Melde dich an.")))
#next command fails
driver.find_element_by_link_text("Melde dich an.").click()
#if the first command is skipped by entering in the url
#in driver.get(https://www.instagram.com/accounts/login/?source=auth_switcher)
#the following command fails as well.
driver.find_element_by_name("username").send_keys("HereIsTheUsername")
driver.find_element_by_name("password").send_keys("HereIsThePassword")
driver.find_element_by_name("password").send_keys(Keys.RETURN)
driver.close()
def ClickElementByName(name,driver):
while True:
try:
driver.find_element_by_name(name).click()
break
except:
sleep(1)
pass
Too long to wait the website run.
Replace ClickElementByName("username", driver)
driver.find_element_by_xpath('//input[#name="username"]').send_keys("HereIsTheUsername")
driver.find_element_by_xpath('//input[#name="password"]').send_keys("HereIsTheUsername")
driver.find_element_by_xpath('//div[text()="Log In"]').click()