Try to get the text from html page based on Xpath using python 2.7 selenium scripting ... Code Not Working is at the end of the script last 3 lines ... Any pointers will be helpful. Thanks
import os
import time
import webbrowser
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains as AC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import Select, WebDriverWait
#region Browser
driver = webdriver.Firefox()
print("Browser: " + driver.name)
driver.set_window_size(1024, 768)
#endregion
driver.get("http://xyz/...")
textName = "Max Size"
try:
textElement = driver.find_elements_by_xpath('/html/body/table/tbody/tr[1]/td[2]/span[16]')
textContent = textElement.text
print ("Text ... " + textContent)
...
Related
I am struggling with Selenium
for the url: https://pubchem.ncbi.nlm.nih.gov/compound/2078
I am trying to click the button Download, but it doesn't find the element.
My code:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.chrome.options import Options
from ipykernel import kernelapp as app
import time
options = webdriver.ChromeOptions()
driver_path = 'C:\\Users\\Idener\\Downloads\\chromedriver_win32\\chromedriver.exe'
driver = webdriver.Chrome(driver_path, options=options)
url = f"https://pubchem.ncbi.nlm.nih.gov/compound/2078"
driver.get(url)
driver.find_element_by_xpath("//*[#id='"'page-download-btn'"']").click()
enter image description here
Your XPath is not valid. You don't need so much quotes
driver.find_element_by_xpath("//*[#id='page-download-btn']").click()
You are missing a delay.
Element should clicked only when it is completely rendered and ready to accept a click event. WebDriverWait expected_conditions explicit waits should be used for that.
Also, no need to add f before URL value and '"' instead of ' in XPath expression.
The following code will work for you:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.chrome.options import Options
from ipykernel import kernelapp as app
import time
options = webdriver.ChromeOptions()
driver_path = 'C:\\Users\\Idener\\Downloads\\chromedriver_win32\\chromedriver.exe'
driver = webdriver.Chrome(driver_path, options=options)
url = "https://pubchem.ncbi.nlm.nih.gov/compound/2078"
driver.get(url)
wait = WebDriverWait(driver, 10)
wait.until(EC.element_to_be_clickable((By.ID, "page-download-btn"))).click()
i don't know this for loop or is there some other way out. I am new to selenium
from selenium.webdriver.support import ui
import time
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import NoSuchElementException
from selenium import webdriver
options = webdriver.ChromeOptions()
driver = webdriver.Chrome(chrome_options=options, executable_path='C:\Program Files (x86)/chromedriver')
driver.maximize_window()
a=["http://www.googgle.com","http://www.youtube.com","http://www.facebook.com"]
for i in a:
driver.get(a[i])
Try this instead.
from selenium.webdriver.support import ui
import time
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import NoSuchElementException
from selenium import webdriver
options = webdriver.ChromeOptions()
driver = webdriver.Chrome(chrome_options=options, executable_path='C:\Program Files (x86)/chromedriver')
driver.maximize_window()
a=["http://www.googgle.com","http://www.youtube.com","http://www.facebook.com"]
for i in a:
driver.get(a)
There is no point in using indexing there.
I've written the following code to search the UNSPSC search page. This script works, but I'm having difficulty extracting the actual results. Adding to the code below, it should then extract "Toy balloons or balls"
from selenium import webdriver
# from selenium.common import exceptions
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from selenium.webdriver.common.keys import Keys
opts = Options()
#opts.add_argument("--headless")
driver = webdriver.Firefox(
firefox_options=opts,
firefox_binary='path/Firefox/firefox.exe',
executable_path='./geckodriver.exe',
)
driver.get("https://www.unspsc.org/search-code/")
search = driver.find_element_by_id("dnn_ctr1535_UNSPSCSearch_txtsearchCode")
search.send_keys("60141001")
search.send_keys(Keys.ENTER)
To Extract the text induce WebDriverWait and wait for visibility_of_element_located() and use the following xpath
driver.get("https://www.unspsc.org/search-code/")
search = driver.find_element_by_id("dnn_ctr1535_UNSPSCSearch_txtsearchCode")
search.send_keys("60141001")
search.send_keys(Keys.ENTER)
print(WebDriverWait(driver,20).until(EC.visibility_of_element_located((By.XPATH,"//table[#id='dnn_ctr1535_UNSPSCSearch_gvDetailsSearchView']/tbody/tr[2]/td[2]"))).text)
You need to import following libraries.
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
how to redirect output of a file to a dataframe.
this code opens browser and types the given data provided and gets the first data available.
code :
selenium pincodes
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import re
import pandas as pd
import os
import html5lib
import json
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
pin=['mumbai','newyork']
for i in pin :
url = "https://www.google.com/"
chromedriver = r"C:\Users\me\chromedriver"
driver = webdriver.Chrome(chromedriver)
driver.implicitly_wait(30)
driver.get(url)
search = driver.find_element_by_name('q')
search.send_keys(i,'pincode')
search.send_keys(Keys.RETURN)
WebDriverWait(driver, 10).until(expected_conditions.visibility_of_element_located((By.XPATH, '//div[#class="IAznY"]//div[#class="title"]')))
elmts = driver.find_elements_by_xpath('//div[#class="IAznY"]//div[#class="title"]')
print(i,elmts[0].text)
time.sleep(3)
driver.quit()
this code outputs the following
newyork 10001
mumbai 230532
how to redirect this output into df like this
city pincode
newyork 10001
mumbai 230532
Declare two arrays and add into dataframe.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import re
import pandas as pd
import os
import html5lib
import json
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
pin=['mumbai','newyork']
#Declare list here
City=[]
PinCode=[]
for i in pin :
url = "https://www.google.com/"
chromedriver = r"C:\Users\me\chromedriver"
driver = webdriver.Chrome(chromedriver)
driver.implicitly_wait(30)
driver.get(url)
search = driver.find_element_by_name('q')
search.send_keys(i,'pincode')
search.send_keys(Keys.RETURN)
WebDriverWait(driver, 10).until(expected_conditions.visibility_of_element_located((By.XPATH, '//div[#class="IAznY"]//div[#class="title"]')))
elmts = driver.find_elements_by_xpath('//div[#class="IAznY"]//div[#class="title"]')
#Append the data into list
City.append(i)
PinCode.append(elmts[0].text)
#added into dataframe
df=pd.DataFrame({"City":City,"PinCode":PinCode})
print(df)
time.sleep(3)
driver.quit()
Output:
City PinCode
0 mumbai 230532
1 newyork 10001
Like this?
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import re
import pandas as pd
import os
import html5lib
import json
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
pin=['mumbai','newyork']
df_output = pd.DataFrame(columns=["City", "pincode"])
for i in pin :
url = "https://www.google.com/"
chromedriver = r"C:\Users\me\chromedriver"
driver = webdriver.Chrome(chromedriver)
driver.implicitly_wait(30)
driver.get(url)
search = driver.find_element_by_name('q')
search.send_keys(i,'pincode')
search.send_keys(Keys.RETURN)
WebDriverWait(driver, 10).until(expected_conditions.visibility_of_element_located((By.XPATH, '//div[#class="IAznY"]//div[#class="title"]')))
elmts = driver.find_elements_by_xpath('//div[#class="IAznY"]//div[#class="title"]')
print(i,elmts[0].text)
df_output = df_output.append(pd.DataFrame(columns=["City", "pincode"], data=[[i,elmts[0].text]]))
time.sleep(3)
driver.quit()
print(df_output)
executing a script (as a subprocess) from another script and parsing the (redirected) output can technically be done, but that's definitly not the right approach (nor the easiest actually).
Instead, you want to refactor your scripts as functions in modules, and have one function working on the results of the other.
I want to go to this site: https://www.pluginillinois.org/offers.aspx?said=2
Each time I have some code that looks like the following:
import os
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
import time
import pyautogui
driverpath = r"C:\Users\chromedriver.exe"
browserProfile = webdriver.ChromeOptions()
browserProfile.add_experimental_option('prefs', {'intl.accept_languages': 'en,en_US'})
browser = webdriver.Chrome(driverpath, options=browserProfile)
browser.get('https://www.pluginillinois.org/offers.aspx?said=2')
My Selenium browser will open and go to a completely different URL instead. The browser will instead go to https://www.pluginillinois.org/OffersBegin.aspx.
Is there a way to obfuscate this behavior?