I have a table with multiple pages. I want to select say 5 elements from the table and click on the checkbox corresponding to those at a time. How is that possible through selenium python automation
def __init__(self, env):
self.driver = webdriver.Firefox()
self.driver.maximize_window()
if env == 'Staging':
self.driver.get("https://serviceconsole-stg.tivo.com/serviceconsole/login.action")
elif env == 'QE':
self.driver.get("http://serviceconsolese01.tivo.com:9090/serviceconsole")
else:
print "Environment is not available", env
print "\n Supported Environments are Staging and QE"
self.driver.quit()
raise SystemExit("Program Exited")
with open('config.json','r') as user_credentials:
config = json.load(user_credentials)
self.driver.find_element_by_id('username').send_keys(config['user']['name'])
self.driver.find_element_by_id('password').send_keys(config['user']['password'])
self.driver.find_element_by_id("signIn").click()
try:
self.driver.find_element_by_xpath('// *[ # id = "loginValidationError"]')
print "Login Not successful"
self.driver.quit()
raise SystemExit("Program Exited")
except NoSuchElementException:
print "Login Successful"
def addnewlinearpackage(self, title, enddate_days_from_today):
try:
# Select Manage
self.driver.find_element_by_xpath("//*[#id='configuration-tab']").click()
# Creating new Linear Package
self.driver.find_element_by_id("linearpublishing").click()
self.driver.find_element_by_id("linpub").click()
self.driver.find_element_by_id("addLinearPackage").click()
self.driver.find_element_by_id("linearpackageTitle").send_keys(title)
self.driver.find_element_by_id('tempPackageId').send_keys(
datetime.strftime(datetime.now(), '%Y%m%d%H%M'))
self.driver.find_element_by_id("inlineLinearPackageCheckbox").click()
start_time = self.driver.find_element_by_id('startDate')
execute = start_time.find_element_by_xpath("*//span[#class='fa fa-calendar']")
self.driver.execute_script("arguments[0].click();", execute)
time.sleep(7)
end_time = self.driver.find_element_by_id('endDate')
end_time.find_element_by_xpath("*//span[#class='fa fa-calendar']").click()
end_date = (datetime.now() + timedelta(days=enddate_days_from_today)).strftime('%m/%d/%Y')
self.driver.find_element_by_xpath("*//td[#data-day='" + end_date + "']").click()
time.sleep(7)
except NoSuchElementException as exp:
print exp
self.driver.quit()
raise SystemExit("Program Exited")
def addlinearservice(self, serviceId):
try:
self.driver.find_element_by_id("linearServiceSection").click()
time.sleep(10)
self.driver.find_element_by_id("publishLinearPackageBtn").click()
time.sleep(30)
self.driver.find_element_by_class_name("sorting_1")
linear_service_found = False
# Searching existing linear service
if linear_service_found == False:
try:# Search in first page
self.driver.find_element_by_xpath(
"/html/body/div[4]/div/div/div[2]/div/div/div/div[2]/div[2]/div/ul/li[9]/a").click()
if self.driver.find_element_by_link_text(serviceId).is_displayed():
self.driver.find_element_by_xpath(
"//a[contains(text(),'" + serviceId + "')]/following::td/input[#type='checkbox']").click()
linear_service_found = True
print "Linear service found"
except NoSuchElementException:
print"No such Element found in page 1"
try:
while linear_service_found == False: # loop to navigate to next page till finding the service ID
try: # Search till last page is reached and next button is disabled
self.driver.find_element_by_xpath(
"// *[#id = 'associatedLinearServicesTable1_next']/../li[#class ='paginate_button next disabled']")
print 'No further Page available to search'
break
except NoSuchElementException:
try:
self.driver.find_element_by_xpath(
'/html/body/div[4]/div/div/div[2]/div/div/div/div[2]/div[2]/div/ul/li[9]/a').click()
if self.driver.find_element_by_link_text(serviceId).is_displayed():
# click the checkbox of Service ID
self.driver.find_element_by_xpath(
"//a[contains(text(),'" + serviceId + "')]/following::td/input[#type='checkbox']").click()
linear_service_found = True
print "Linear Service found"
break
except NoSuchElementException:
print "No such Element found in current page"
except NoSuchElementException:
print"No such Element found"
if linear_service_found == True:
time.sleep(10)
#Click on Save button
self.driver.find_element_by_xpath('/ html / body / div[4] / div / div / div[3] / button[1]').click()
time.sleep(10)
except NoSuchElementException as exp:
print exp
self.driver.quit()
raise SystemExit("Program Exited")
def publish(self):
try:
self.driver.find_element_by_xpath('//button[contains(text(), "Publish")]').click()
time.sleep(5)
self.driver.find_element_by_xpath('//*[#id="confirmDialogOk"]').click()
time.sleep(10)
try:
self.driver.find_element_by_xpath('//*[#id="appSuccessMsg"]')
print("Linear Package Published Successfully")
except NoSuchElementException:
print ("Linear Package NOT PUBLISHED.. check the Error Message in Service console webpage")
time.sleep(60)
self.driver.quit()
raise SystemExit("Program Exited")
except NoSuchElementException as exp:
print exp
self.driver.quit()
raise SystemExit("Program Exited")
def exit(self):
print("Exiting.....")
time.sleep(5)
self.driver.quit()
Please find the full code. This code works only for selecting one element. I have to select multiple elements.
image
If the checkboxes have a common locator, you can use find_elements_by_xpath instead of find_element_by_xpath. This will return a list of WebElements, which you can then iterate over to click the boxes.
Related
I'm trying solve re-captcha in a site using 2captcha service, but always returns to me the error:
selenium.common.exceptions.JavascriptException: Message: javascript error: document.getElementById(...).submit is not a function
my code:
try:
time.sleep(0.3)
driver.find_element(by=By.XPATH, value='//*[#id="email"]').send_keys(mail)
except:
def Solver():
driver.get(page_url)
u1 = f"https://2captcha.com/in.php?key={API_KEY}&method=userrecaptcha&googlekey={data_sitekey}&pageurl={page_url}&json=1&invisible=1"
r1 = requests.get(u1)
print(r1.json())
rid = r1.json().get("request")
u2 = f"https://2captcha.com/res.php?key={API_KEY}&action=get&id={int(rid)}&json=1"
time.sleep(5)
while True:
r2 = requests.get(u2)
print(r2.json())
if r2.json().get("status") == 1:
form_tokon = r2.json().get("request")
break
time.sleep(5)
wirte_tokon_js = f'document.getElementById("g-recaptcha-response").innerHTML="{form_tokon}";'
submit_js = 'document.getElementById("g-recaptcha-response").submit();'
driver.execute_script(wirte_tokon_js)
time.sleep(3)
driver.execute_script(submit_js)
time.sleep(3)
pic of captcha
I'm currently using Python and Selenium to loop my server for specific tasks to complete, I have tried to do 2 things, to speed up the process they are:
To use options.add_argument(f"user-data-dir={script_directory}\\profile") in the Chrome driver initiasation to avoid having to log in all the time.
To try and reuse the same browser window instead of closing and then re-opening the browser all the time.
Code:
#!/usr/bin/env python
import pathlib
import time
import urllib.parse
import requests
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
USER = "..."
PASS = "..."
def upload_to_server(link, redirect, unique_hash):
try:
requests.get(
"https://www.example.com/crons.php?cronUploadToServer=1&link={0}&redirect={1}&hash={2}".format(link,
redirect,
unique_hash))
except Exception as e:
print(e)
def download_from_server():
try:
server = requests.get("https://www.example.com/crons.php?cronDownloadFromServer=1")
return server.text.strip()
except Exception as e:
print(e)
# tear down chrome.
def tear_down(_driver):
_driver.quit()
_driver.close()
def check_for_tasks():
if download_from_server() == "NO_TASKS":
print("--> NO TASKS")
else:
# init the chrome driver.
def init_driver(using_linux, proxy):
script_directory = pathlib.Path().absolute()
try:
options = Options()
options.headless = False
options.add_argument('start-maximized')
options.add_argument('--disable-popup-blocking')
options.add_argument('--disable-notifications')
options.add_argument('--log-level=3')
options.add_argument('--ignore-certificate-errors')
options.add_argument('--ignore-ssl-errors')
options.add_argument(f"user-data-dir={script_directory}\\profile")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option("detach", True)
prefs = {'profile.default_content_setting_values.notifications': 2}
options.add_experimental_option('prefs', prefs)
if proxy == "0.0.0.0:0":
print("--> PROXY DISABLED ...")
else:
print("--> PROXY: " + str(proxy) + " ...")
options.add_argument('--proxy-server=%s' % proxy)
if using_linux:
return webdriver.Chrome(options=options)
else:
return webdriver.Chrome(options=options)
except Exception as e:
print(e)
# create session.
driver = init_driver(False, "0.0.0.0:00")
# starting URL.
driver.get('https://www.example.com/logon')
# click recorded.
def topcashback_click(_driver):
try:
_driver.get('https://www.example.com/Earn.aspx?mpurl=shein&mpID=17233')
if "redirect.aspx?mpurl=shein" in _driver.current_url:
return _driver.current_url
else:
return False
except Exception as e:
print(e)
# already logged in check.
if ">Account</span>" in driver.page_source:
print("--> LOGGED IN (ALREADY) ...")
driver.get('https://www.SITE.CO.UK/Earn.aspx?mpurl=shein&mpID=17233')
try:
server = download_from_server()
data_from_server = server.split('|')
link = topcashback_click(driver)
print("--> LINK --> " + link)
time.sleep(4)
if link != driver.current_url:
print("--> LINK (REDIRECT) --> " + driver.current_url)
upload_to_server(urllib.parse.quote_plus(link),
urllib.parse.quote_plus(
driver.current_url.replace('https://www.example.com', data_from_server[0])),
data_from_server[1])
# print(driver.current_url.replace('https://www.example.com', data_from_server[0]))
print("--> LINK UPLOADED TO THE DB ...")
# tear_down(driver)
except Exception as e:
print(e)
else:
# TopCashBack login for the first time.
def topcashback_login(_driver):
_driver.get('https://www.example.com/logon')
# small sleep to let the page load.
time.sleep(1)
_driver.find_element(By.XPATH, '//*[#id="txtEmail"]').send_keys(USER)
time.sleep(1)
_driver.find_element(By.XPATH, '//*[#id="loginPasswordInput"]').send_keys(PASS)
time.sleep(1)
_driver.find_element(By.XPATH, '//*[#id="Loginbtn"]').click()
time.sleep(5)
if ">Account</span>" in _driver.page_source:
return True
else:
return False
def topcashback_click(_driver):
try:
_driver.get('https://www.SITE.CO.UK/Earn.aspx?mpurl=shein&mpID=17233')
if "redirect.aspx?mpurl=shein" in _driver.current_url:
return _driver.current_url
else:
return False
except Exception as e:
print(e)
if topcashback_login(driver):
try:
print("--> LOGGED IN ...")
server = download_from_server()
data_from_server = server.split('|')
link = topcashback_click(driver)
print("--> LINK --> " + link)
time.sleep(4)
if link != driver.current_url:
print("--> LINK (REDIRECT) --> " + driver.current_url)
upload_to_server(urllib.parse.quote_plus(link),
urllib.parse.quote_plus(
driver.current_url.replace('https://www.example.com',
data_from_server[0])),
data_from_server[1])
# print(driver.current_url.replace('https://www.example.com', data_from_server[0]))
print("--> LINK UPLOADED TO THE DB ...")
# tear_down(driver)
except Exception as e:
print(e)
else:
print("--> ERROR --> DEBUG TIME ...")
tear_down(driver)
if __name__ == "__main__":
while True:
check_for_tasks()
time.sleep(2)
It's the 2nd one I'm having trouble with, currently, with my code, I'm getting the error:
driver.get('https://www.example.com/logon')
AttributeError: 'NoneType' object has no attribute 'get'
I think this is because I'm not connecting the first browser window, instead it's opening a new one which fails with the error above straight away.
Is there possibly a way to keep the first browser open and reuse it? any help would be appreciated.
I am starting to learn python and faced with such a problem, I have main.py I want all the functionality in it to work with a password that will be taken from a text document on the site.
See here is the code
from selenium import webdriver
from os import system, name
from time import time, strftime, gmtime, sleep
import pyfiglet, os, threading
import webbrowser
def clear():
if name == 'nt':
_ = system('cls')
else:
_ = system('clear')
clear()
system('title TUT.GURU')
print(pyfiglet.figlet_format("TUT.GURU", font="slant"))
print("1. Viewbot.\n2. Heartbot.\n3. Followerbot.\n3. Sharebot.\n4. Credits.\n5. Инфо\n\nЗакажите сайт или программу у меня в ТГ: #whitehak\nЗаходи к нам на форум: TUT.GURU\n\n")
auto = int(input("Режим: "))
if auto == 1 or auto == 2 or auto == 3 or auto == 4:
vidUrl = input("Вставьте ссылку на видео: ")
start = time()
time_elapsed = strftime('%H:%M:%S', gmtime(time() - start))
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--mute-audio")
chrome_options.add_experimental_option('excludeSwitches', ['enable-logging'])
driver = webdriver.Chrome(r"chromedriver.exe", options=chrome_options)
driver.set_window_size(425, 948)
Views = 0
Hearts = 0
Followers = 0
def beautify(arg):
return format(arg, ',d').replace(',', '.')
def title1(): # Update the title IF option 1 was picked.
global Views
while True:
time_elapsed = strftime('%H:%M:%S', gmtime(time() - start))
system(f'title TUT.GURU ^| Views Sent: {beautify(Views)} ^| Программа работает: {time_elapsed}')
def title2(): # Update the title IF option 2 was picked.
global Hearts
while True:
time_elapsed = strftime('%H:%M:%S', gmtime(time() - start))
system(f'title TUT.GURU ^| Hearts Sent: {beautify(Hearts)} ^| Программа работает: {time_elapsed}')
def title3(): # Update the title IF option 3 was picked.
global Followers
while True:
time_elapsed = strftime('%H:%M:%S', gmtime(time() - start))
system(f'title TUT.GURU ^| Followers Sent: {beautify(Followers)} ^| Программа работает: {time_elapsed}')
def title4(): # Update the title IF option 1 was picked.
global Shares
while True:
time_elapsed = strftime('%H:%M:%S', gmtime(time() - start))
system(f'title TUT.GURU ^| Shares Sent: {beautify(Shares)} ^| Программа работает: {time_elapsed}')
def loop1():
global Views
sleep(10)
try:
driver.find_element_by_xpath("/html/body/div[4]/div[1]/div[3]/div/div[4]/div/button").click()
except:
print("[-] Капча не разгадана!")
driver.refresh()
loop1()
try:
sleep(2)
driver.find_element_by_xpath("//*[#id=\"sid4\"]/div/form/div/input").send_keys(vidUrl)
sleep(1)
driver.find_element_by_xpath("//*[#id=\"sid4\"]/div/form/div/div/button").click()
sleep(5)
driver.find_element_by_xpath("//*[#id=\"c2VuZC9mb2xsb3dlcnNfdGlrdG9V\"]/div[1]/div/form/button").click()
driver.refresh()
Views += 1000
print("[+] Views sended!")
sleep(300)
loop1()
except:
print("[-] Пробуем еще раз")
driver.refresh()
loop1()
def loop2():
global Hearts
sleep(10)
try:
driver.find_element_by_xpath("/html/body/div[4]/div[1]/div[3]/div/div[2]/div/button").click()
except:
print("[-] Капча не разгадана!")
driver.refresh()
loop2()
try:
sleep(2)
driver.find_element_by_xpath('//*[#id="sid2"]/div/form/div/input').send_keys(vidUrl)
sleep(1)
driver.find_element_by_xpath('//*[#id="sid2"]/div/form/div/div/button').click()
sleep(5)
driver.find_element_by_xpath('//*[#id="c2VuZE9nb2xsb3dlcnNfdGlrdG9r"]/div[1]/div/form/button').click()
sleep(6)
hearts = driver.find_element_by_xpath('//*[#id="c2VuZE9nb2xsb3dlcnNfdGlrdG9r"]/span').text.split()
Hearts += int(hearts[0])
print("[+] Hearts sended!")
sleep(5)
driver.refresh()
sleep(1800)
loop2()
except:
print("[-] Пробуем еще раз")
driver.refresh()
loop2()
def loop3():
global Followers
sleep(10)
try:
driver.find_element_by_xpath("/html/body/div[4]/div[1]/div[3]/div/div[1]/div/button").click()
except:
print("[-] Капча не разгадана!")
driver.refresh()
loop3()
try:
sleep(2)
driver.find_element_by_xpath("//*[#id=\"sid\"]/div/form/div/input").send_keys(vidUrl)
sleep(1)
driver.find_element_by_xpath("//*[#id=\"sid\"]/div/form/div/div/button").click()
sleep(5)
driver.find_element_by_xpath("//*[#id=\"c2VuZF9mb2xsb3dlcnNfdGlrdG9r\"]/div[1]/div/form/button").click()
sleep(6)
folls = driver.find_element_by_xpath('//*[#id="c2VuZF9mb2xsb3dlcnNfdGlrdG9r"]/span').text.split()
Followers += int(folls[0])
print("[+] Followers sended!")
driver.refresh()
sleep(1800)
loop3()
except:
print("[-] Пробуем еще раз")
driver.refresh()
loop3()
def loop4():
global Shares
sleep(10)
try:
driver.find_element_by_xpath("/html/body/div[4]/div[1]/div[3]/div/div[5]/div/button").click()
except:
print("[-] Капча не разгадана!")
driver.refresh()
loop4()
try:
sleep(2)
driver.find_element_by_xpath("//*[#id=\"sid7\"]/div/form/div/input").send_keys(vidUrl)
sleep(1)
driver.find_element_by_xpath("//*[#id=\"sid7\"]/div/form/div/div/button").click()
sleep(5)
driver.find_element_by_xpath("//*[#id=\"c2VuZC9mb2xsb3dlcnNfdGlrdG9s\"]/div[1]/div/form/button").click()
driver.refresh()
Shares += 100
print("[+] Shares sended!")
sleep(300)
loop4()
except:
print("[-] Пробуем еще раз")
driver.refresh()
loop4()
clear()
you need to ask for a password when you turn on the program and if it is correct, then run the program and if not, then write that the password is not correct.
The password will be stored on the website
example: domen.com/pass.txt
I can't figure out how to do this
I would just not do that. Why would you ever want to give away your plaintext password to every man, woman, and child in the world with Internet access to domain.com?
That said, if you want to fetch a file via URL, check out the requests library.
requests.get('https://example.com/file.png')
I built a selenium web scraper (see below for code). It works fine and normally takes 4-6 seconds per loop. However, if I use a different web browser to do something else, say check my email, the web scraper slows down (sometimes taking up to a couple minutes per loop) and it also takes a long time to load my email (or whatever else I am trying to do with the internet.
Is there something wrong with my scraper? Or is it not possible to run a web scraper while also using the internet to do other things? Or...
Thanks!
counter = 36386
options = Options()
options.set_headless(True)
driver = webdriver.Firefox(options=options, executable_path = r'C:\Users\jajacobs\Downloads\geckodriver.exe')
while counter <= 50000:
start_time = time.time()
try:
driver.get("url goes here")
timeout = 20
inputElement = driver.find_element_by_name("naics_lookup[companyName]")
inputElement.send_keys(naics.iloc[counter, 1])
inputElement = driver.find_element_by_name("naics_lookup[city]")
inputElement.send_keys(naics.iloc[counter, 3])
inputElement = driver.find_element_by_name("naics_lookup[state]")
inputElement.send_keys(naics.iloc[counter, 2])
inputElement.submit()
print('Looking for NAICS code of company number ', counter)
try:
element_present = EC.presence_of_element_located((By.CLASS_NAME, 'results'))
WebDriverWait(driver, timeout).until(element_present)
print("element is ready")
try:
data = driver.find_element_by_class_name('results').text
naics.at[counter, 'naics'] = re.findall(r"\D(\d{6})\D", data)[0]
print(re.findall(r"\D(\d{6})\D", data)[0])
except:
print("No NAICS code")
pass
except:
print("element did not load")
pass
list = [1000,2000,3000,4000,5000,6000,7000,8000,9000,10000,11000,12000,13000,
14000,15000,16000,17000,18000,19000,20000,21000,22000,23000,24000,25000,
25000,26000,27000,28000,29000,30000,31000,32000,33000,34000,35000,36000,
37000,38000,39000,40000,41000,42000,43000,44000,45000,46000,47000,48000,
49000,50000,]
if counter in list:
data_folder = Path('C:/Users/jajacobs/Documents/ipynb/')
file_to_save = data_folder / ('naics' + str(counter) + '.csv')
naics.to_csv(file_to_save)
counter += 1
except Exception as e:
print(e)
pass
print("total time taken this loop: ", time.time() - start_time)
driver.close()
I have a Python script with a loop.
If the code cannot find a specified element: (driver.find_element_by_xpath), he should print: "No Button". And then he should restart the loop.
But when I run the code, it will print "No button" BUT then it continues the script instead of retry/loop.
not_found=True
while not_found:
try:
browser=browser.open(link)
except HTTPError, e:
print 'The server couldn\'t fulfill the request.'
except URLError, e:
print 'Connection reset by peer'
sys.stdout.write('\a')
sys.stdout.flush()
new=0
time.sleep(5)
search_k="just some text to look for"
source="output.txt"
if not search_string(search_k, source):
print "this is a message reminder"
startstr='search text"'
endstr='" />'
file='output.txt'
prijs=float(get_html_string(file,startstr,endstr))
search_s="just some text in .txt"
filez="blee.txt"
search_z="texttt"
search_q="texttttttt"
if search_string(search_s, filez) and not search_string(search_z, filez) and not search_string(search_q, filez) and rule <= maxprijs:
driver.get(link)
try:
elem = driver.find_element_by_xpath("//input[#type='submit' and #value='submit']")
elem.click()
except NoSuchElementException:
print"No Button"
time.sleep(2)
new=0 #LOOP SCRIPT IGNORES?