Hi i have this code with tries many number on yahoo after the button of submitting is clicked
the page seems to get refreshed because i am not able to press the same button.
here is my code
import os
from tkinter import *
from tkinter import filedialog
import selenium
import time
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium import webdriver
from selenium import webdriver
import contextlib as textmanager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
PATH= "C:\chromedrivers\chromedriver.exe"
driver= webdriver.Chrome(PATH)
list=[]
file= open("numbers.txt", "r")
for line in file:
line = line.strip() #preprocess line
list.append(line)
driver.get("https://www.yahoo.com/")
element = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//*[#id='consent-page']/div/div/div/form/div[2]/div[2]/button")))
element.click();
time.sleep(1)
sign_in= driver.find_element_by_xpath("//*[#id='ybar-inner-wrap']/div[3]/div/div[3]/div[1]/div/a").click()
time.sleep(1)
forgot_phone= driver.find_element_by_xpath("//*[#id='mbr-forgot-link']").click()
time.sleep(2)
inputt= driver.find_element_by_id("username")
time.sleep(2)
buttonn= driver.find_element_by_xpath("//*[#id='yid-challenge']/form/div[2]/button")
counter=0
nonumber= open("nonumber.txt","w")
while counter != len(list):
inputt.send_keys(list[counter])
time.sleep(1)
buttonn.click()
if len(driver.find_elements_by_css_selector("p[data-error]")) > 0 :
inputt.clear()
nonumber.write(list[counter])
nonumber.close()
counter=counter+1
here is the error it says button not clickable but in first run it works iam not sure is the problem in button or input area
Traceback (most recent call last):
File "c:\Users\ramhelsinki\projects\slenium.py", line 54, in <module>
inputt.send_keys(list[counter])
File "C:\Users\ramhelsinki\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\selenium\webdriver\remote\webelement.py", line 477, in send_keys
self._execute(Command.SEND_KEYS_TO_ELEMENT,
File "C:\Users\ramhelsinki\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\selenium\webdriver\remote\webelement.py", line 633, in _execute
return self._parent.execute(command, params)
File "C:\Users\ramhelsinki\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "C:\Users\ramhelsinki\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\LocalCache\local-packages\Python39\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
(Session info: chrome=91.0.4472.124)
Each time the web page is changed / refreshed all the web elements previously fetched by Selenium become irrelevant. This is what stale element reference exception comes for.
With your code, since page is refreshed in case of wrong input, I would advice fetching the elements again after each time the page is changed so the code will look like this:
time.sleep(2)
counter=0
nonumber= open("nonumber.txt","w")
while counter != len(list):
inputt= driver.find_element_by_id("username")
buttonn= driver.find_element_by_xpath("//button")
inputt.send_keys(list[counter])
time.sleep(1)
buttonn.click()
if len(driver.find_elements_by_css_selector("p[data-error]")) > 0 :
inputt= driver.find_element_by_id("username")
inputt.clear()
nonumber.write(list[counter])
nonumber.close()
counter=counter+1
Related
I am working on a project using selenium webdriver which requires me to locate an element and click on it. The program starts by entering a website , clicking the searchbar , typing in a preentered string and clicking enter , up to this point everything is successful. The next thing I want it to do is find the first result of the search and click on it. This part I am having trouble with. I have successfully located all elements up to this point but i cant locate this one as an error pops up. Here is my code:
from selenium import webdriver
import time
trackname = input("Track Name: ")
driver = webdriver.Chrome('D:\WebDrivers\chromedriver.exe')
driver.get('https://music.apple.com/us/artist/search/166949667')
time.sleep(2)
searchbox = driver.find_element_by_tag_name('input')
searchbox.send_keys(trackname)
from keyboard import press
press('enter')
time.sleep(2)
result = driver.find_element_by_id('search-list-lockup__description')
result.click()
I have tried locating the element other ways but it wont work , I am guessing that the issue is that after searching I have to tell it to search on that page but I am not sure. Here is the error:
Traceback (most recent call last):
File "D:\Python Projekti\iTunesDataFiller\iTunesDataFiller.py", line 18, in <module>
result = driver.find_element_by_id('search-list-lockup__description')
File "D:\Python App\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 360, in find_element_by_id
return self.find_element(by=By.ID, value=id_)
File "D:\Python App\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 976, in find_element
return self.execute(Command.FIND_ELEMENT, {
File "D:\Python App\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
self.error_handler.check_response(response)
File "D:\Python App\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"css selector","selector":"[id="search-list-lockup__description"]"}
(Session info: chrome=89.0.4389.114)
Process finished with exit code 1
What do I do?
This could be due to the element you want to access not being available.
For example say you load the page, the element could not be visible to selenium. So basically you're trying to click on an invisible element.
I suggest using this template to make sure elements are loaded before accessing them.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
waitshort = WebDriverWait(driver,.5)
wait = WebDriverWait(driver, 20)
waitLonger = WebDriverWait(driver, 100)
visible = EC.visibility_of_element_located
driver = webdriver.Chrome(executable_path='path')
driver.get('link')
element_you_want_to_access = wait.until(visible((By.XPATH,'xpath')))
enter image description hereMy code is basically logging into a website (https://user.sensogram.com/signin) and download the CSV file by clicking on a button (Download CSV). However I am getting an errors that says:
File "c:\Users\USERPC\Desktop\python\sesno_login.py", line 57, in <module>
csv_file_button.click()
File "C:\Users\USERPC\AppData\Local\Programs\Python\Python36-32\lib\site-
packages\selenium\webdriver\remote\webelement.py", line 80, in click
self._execute(Command.CLICK_ELEMENT)
File "C:\Users\USERPC\AppData\Local\Programs\Python\Python36-32\lib\site-
packages\selenium\webdriver\remote\webelement.py", line 628, in _execute
return self._parent.execute(command, params)
File "C:\Users\USERPC\AppData\Local\Programs\Python\Python36-32\lib\site-
packages\selenium\webdriver\remote\webdriver.py", line 312, in execute
self.error_handler.check_response(response)
File "C:\Users\USERPC\AppData\Local\Programs\Python\Python36-32\lib\site-
packages\selenium\webdriver\remote\errorhandler.py", line 242, in
check_response raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: unknown error:
Element <tspan>...</tspan> is not clickable at point (1232, 413). Other
element would receive the click: <div class="page-loading ng-scope" ng-
if="showPreloader" style="">...</div>
It's like that the code can't locate the click button on the page for some reason. This is what I wrote so far:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from time import sleep
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.common.exceptions import NoSuchElementException
import time
chromedriver = "/webdrivers/chromedriver"
driver = webdriver.Chrome(chromedriver)
driver.maximize_window()
driver.get('https://user.sensogram.com/signin') #driver.get(url)-- We get
url by using driver which we initialy load.
print ("Opened sensogram")
time.sleep(5) #Just wait for sometime.
email = driver.find_element_by_xpath("//input[#name='usernameSignIn']")
#Find email textaera.
email.send_keys('****') #Send email to this text area.
password = driver.find_element_by_xpath("//input[#name='passwordSignIn']")
#Find password textarea.
password.send_keys('*******') #send password to the password field.
button = driver.find_element_by_xpath("//button[#ng-
click='submitted=true']") #Find login button.
button.click() #Click on login button.
driver.implicitly_wait(10)
csv_file_button = driver.find_element_by_xpath("//*[name()='tspan' and
.='Download CSV']")
csv_file_button.click()
print(csv_file_button)
can someone figure out why i Keep getting this type of error? plus this is the webpage HTML of the csv file:
<g class="highcharts-button highcharts-contextbutton highcharts-button-
normal" style="cursor:pointer;vertical-align:center;font-size:12;font-
weight:500;" stroke-linecap="round" transform="translate(379,8)">
<title>Chart context menu</title><rect fill="rgba(0, 0, 0, 0.7)" class="
highcharts- button-box" x="0.5" y="0.5" width="100" height="32" rx="2"
ry="2" stroke="none" stroke-width="1"></rect><text x="7" style="font-
weight:normal;color:#fff;fill:#fff;" y="19"><tspan>Download CSV</tspan>
</text></g>
Sometimes selenium webdriver could not find elements, so better use javascript executor to click on desired button
Example:
js = (JavascriptExecutor) driver;
js.executeScript("$('#downloadcsv').click()");
I have a simple task that I want to automate. I want to open a URL, click a button which takes me to the next page, fills in a search term, clicks the "search" button and prints out the url and source code of the results page. I have written the following.
from selenium import webdriver
import time
driver = webdriver.PhantomJS()
driver.set_window_size(1120, 550)
#open URL
driver.get("https://www.searchiqs.com/nybro/")
time.sleep(5)
#click Log In as Guest button
driver.find_element_by_id('btnGuestLogin').click()
time.sleep(5)
#insert search term into Party 1 form field and then search
driver.find_element_by_id('ContentPlaceholder1_txtName').send_keys("Moses")
driver.find_element_by_id('ContentPlaceholder1_cmdSearch').click()
time.sleep(10)
#print and source code
print driver.current_url
print driver.page_source
driver.quit()
I am not sure where I am going wrong but I have followed a number of tutorials on how to click buttons and fill forms. I get this error instead.
Traceback (most recent call last):
File "phant.py", line 12, in <module>
driver.find_element_by_id('btnGuestLogin').click()
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py", line 269, in find_element_by_id
return self.find_element(by=By.ID, value=id_)
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py", line 752, in find_element
'value': value})['value']
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py", line 236, in execute
self.error_handler.check_response(response)
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/errorhandler.py", line 192, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: {"errorMessage":"Unable to find element with id 'btnGuestLogin'","request":{"headers":{"Accept":"application/json","
Accept-Encoding":"identity","Connection":"close","Content-Length":"94","Content-Type":"application/json;charset=UTF-8","Host":"127.0.0.1:35670","User-Agent":"Python-urllib/2.7"
},"httpVersion":"1.1","method":"POST","post":"{\"using\": \"id\", \"sessionId\": \"d38e5fa0-5349-11e6-b0c2-758ad3d2c65e\", \"value\": \"btnGuestLogin\"}","url":"/element","urlP
arsed":{"anchor":"","query":"","file":"element","directory":"/","path":"/element","relative":"/element","port":"","host":"","password":"","user":"","userInfo":"","authority":""
,"protocol":"","source":"/element","queryKey":{},"chunks":["element"]},"urlOriginal":"/session/d38e5fa0-5349-11e6-b0c2-758ad3d2c65e/element"}}
Screenshot: available via screen
The error seems to suggest that the element with that id does not exist yet it does.
--- EDIT: Changed code to use WebDriverWait ---
I have changed some things around to implement WebDriverWait
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
driver = webdriver.PhantomJS()
driver.set_window_size(1120, 550)
#open URL
driver.get("https://www.searchiqs.com/nybro/")
#click Log In as Guest button
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "btnGuestLogin"))
)
element.click()
#wait for new page to load, fill in form and hit search
element2 = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "ContentPlaceholder1_cmdSearch"))
)
#insert search term into Party 1 form field and then search
driver.find_element_by_id('ContentPlaceholder1_txtName').send_keys("Moses")
element2.click()
driver.implicitly_wait(10)
#print and source code
print driver.current_url
print driver.page_source
driver.quit()
It still raises this error
Traceback (most recent call last):
File "phant.py", line 14, in <module>
EC.presence_of_element_located((By.ID, "btnGuestLogin"))
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/support/wait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
Screenshot: available via screen
The WebDriverWait approach actually works for me as is:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.PhantomJS()
driver.set_window_size(1120, 550)
driver.get("https://www.searchiqs.com/nybro/")
element = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, "btnGuestLogin"))
)
element.click()
No errors. PhantomJS version 2.1.1, Selenium 2.53.6, Python 2.7.
The issue might be related to SSL and PhantomJS, either work through http:
driver.get("http://www.searchiqs.com/nybro/")
Or, try ignoring SSL errors:
driver = webdriver.PhantomJS(service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])
I have a webscraper that is running on my system and I wanted to migrate it over to PythonAnywhere, but when I moved it now it doesn't work.
Exactly the sendkeys does not seem to work - after the following code is executed I never move on to the next webpage so an attribute error gets tripped.
My code:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import csv
import time
# Lists for functions
parcel_link =[]
token = []
csv_output = [ ]
# main scraping function
def getLinks(link):
# Open web browser and get url - 3 second time delay.
#Open web browser and get url - 3 second time delay.
driver.get(link)
time.sleep(3)
inputElement = driver.find_element_by_id("mSearchControl_mParcelID")
inputElement.send_keys(parcel_code+"*/n")
print("ENTER hit")
pageSource = driver.page_source
bsObj = BeautifulSoup(pageSource)
parcel_link.clear()
print(bsObj)
#pause = WebDriverWait(driver, 60).until(EC.presence_of_element_located((By.ID, "mResultscontrol_mGrid_RealDataGrid")))
for link in bsObj.find(id="mResultscontrol_mGrid_RealDataGrid").findAll('a'):
parcel_link.append(link.text)
print(parcel_link)
for test in parcel_link:
clickable = driver.find_element_by_link_text(test)
clickable.click()
time.sleep(2)
The link I am trying to operate is:
https://ascendweb.jacksongov.org/ascend/%280yzb2gusuzb0kyvjwniv3255%29/search.aspx
and I am trying to send: 15-100*
TraceBack:
03:12 ~/Tax_Scrape $ xvfb-run python3.4 Jackson_Parcel_script.py
Traceback (most recent call last):
File "Jackson_Parcel_script.py", line 377, in <module>
getLinks("https://ascendweb.jacksongov.org/ascend/%28biohwjq5iibvvkisd1kjmm45%29/result.aspx")
File "Jackson_Parcel_script.py", line 29, in getLinks
inputElement = driver.find_element_by_id("mSearchControl_mParcelID")
File "/usr/local/lib/python3.4/dist-packages/selenium/webdriver/remote/webdriver.py", line 206, in find_element_by_id
return self.find_element(by=By.ID, value=id_)
File "/usr/local/lib/python3.4/dist-packages/selenium/webdriver/remote/webdriver.py", line 662, in find_element
{'using': by, 'value': value})['value']
File "/usr/local/lib/python3.4/dist-packages/selenium/webdriver/remote/webdriver.py", line 173, in execute
self.error_handler.check_response(response)
File "/usr/local/lib/python3.4/dist-packages/selenium/webdriver/remote/errorhandler.py", line 164, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: 'Unable to locate element: {"method":"id","selector":"mSearchControl_mParcelID"}' ; Stac
ktrace:
at FirefoxDriver.findElementInternal_ (file:///tmp/tmpiuuqg3m7/extensions/fxdriver#googlecode.com/components/driver_component.js:9470)
at FirefoxDriver.findElement (file:///tmp/tmpiuuqg3m7/extensions/fxdriver#googlecode.com/components/driver_component.js:9479)
at DelayedCommand.executeInternal_/h (file:///tmp/tmpiuuqg3m7/extensions/fxdriver#googlecode.com/components/command_processor.js:11455)
at DelayedCommand.executeInternal_ (file:///tmp/tmpiuuqg3m7/extensions/fxdriver#googlecode.com/components/command_processor.js:11460)
at DelayedCommand.execute/< (file:///tmp/tmpiuuqg3m7/extensions/fxdriver#googlecode.com/components/command_processor.js:11402)
03:13 ~/Tax_Scrape $
Selenium Innitation:
for retry in range(3):
try:
driver = webdriver.Firefox()
break
except:
time.sleep(3)
for parcel_code in token:
getLinks("https://ascendweb.jacksongov.org/ascend/%28biohwjq5iibvvkisd1kjmm4 5%29/result.aspx")
PythonAnywhere uses a virtual instance of FireFox that is suppose to be headless like JSPhantom so I do not have a version number.
Any help would be great
RS
Well, maybe the browser used on PythonAnywhere does not load the site fast enough. So instead of time.sleep(3) try implicitly waiting for the element.
An implicit wait is to tell WebDriver to poll the DOM for a certain
amount of time when trying to find an element or elements if they are
not immediately available. The default setting is 0. Once set, the
implicit wait is set for the life of the WebDriver object instance.
Using time.sleep() with Selenium is not a good idea in general.
And give it more than just 3 second, with implicitly_wait() you specify the maximum time spent waiting for an element.
So if you set implicitly_wait(10) and the page loads, for example, in 5 seconds then Selenium will wait only 5 seconds.
driver.implicitly_wait(10)
I am trying to record every item on the tf2 market place using selenium. I am trying to record the name of each item in a file on sale. This is the link to the page. I think it is this tag I just dont know how to reference and record the name in a text file with each name on a new line.
<span id="result_0_name" class="market_listing_item_name" style="color; #7D6D00;">
Edit 1:
I have used the solution by alecxe and it works for the first page I'm now trying to run it to select the next button then run again. But to no avail this is what I am trying.
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from selenium import webdriver
url="http://steamcommunity.com/market/search?appid=440#p1_popular_desc"
driver = webdriver.Firefox()
driver.get(url)
x=1
while x==1:
WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div.market_listing_row")))
time.sleep(5)
results = [item.text for item in driver.find_elements_by_css_selector("div.market_listing_row .market_listing_item_name")]
time.sleep(5)
driver.find_element_by_id('searchResults_btn_next').click()
with open("output.dat", "a") as f:
for item in results:
f.write(item + "\n")
This produces this error
Traceback (most recent call last):
File "name.py", line 14, in <module>
results = [item.text for item in driver.find_elements_by_css_selector("div.market_listing_row .market_listing_item_name")]
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webelement.py", line 61, in text
return self._execute(Command.GET_ELEMENT_TEXT)['value']
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webelement.py", line 402, in _execute
return self._parent.execute(command, params)
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py", line 175, in execute
self.error_handler.check_response(response)
File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/errorhandler.py", line 166, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: Element is no longer attached to the DOM
Stacktrace:
at fxdriver.cache.getElementAt (resource://fxdriver/modules/web-element-cache.js:8956)
at Utils.getElementAt (file:///tmp/tmpUpLsV7/extensions/fxdriver#googlecode.com/components/command-processor.js:8546)
at WebElement.getElementText (file:///tmp/tmpUpLsV7/extensions/fxdriver#googlecode.com/components/command-processor.js:11704)
at DelayedCommand.prototype.executeInternal_/h (file:///tmp/tmpUpLsV7/extensions/fxdriver#googlecode.com/components/command-processor.js:12274)
at DelayedCommand.prototype.executeInternal_ (file:///tmp/tmpUpLsV7/extensions/fxdriver#googlecode.com/components/command-processor.js:12279)
at DelayedCommand.prototype.execute/< (file:///tmp/tmpUpLsV7/extensions/fxdriver#googlecode.com/components/command-processor.js:12221)
Any help would be greatly appreciated even if it is links to guides
You can get the names from elements with market_listing_item_name class name located in div elements having market_listing_row class:
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
url = "http://steamcommunity.com/market/search?appid=440"
driver = webdriver.Chrome()
driver.get(url)
# wait for results
WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div.market_listing_row")))
results = [item.text for item in driver.find_elements_by_css_selector("div.market_listing_row .market_listing_item_name")]
driver.quit()
# dump results to a file
with open("output.dat", "wb") as f:
for item in results:
f.write(item + "\n")
Here is the contents of the output.dat file after running the script:
Mann Co. Supply Crate Key
The Powerhouse Weapons Case
The Concealed Killer Weapons Case
Earbuds
Bill's Hat
Gun Mettle Campaign Pass
Tour of Duty Ticket
Genuine AWPer Hand
Specialized Killstreak Kit
Gun Mettle Key