im practically a fetus when it comes to python programming and i hope some of you gods would be able to help me. basically i have a loop set up that pulls values from a website but i need it run indefinitely until i press the specified hotkey which is q, the loop runs and pulls the data needed, but whenever i press q, the loop simply keeps running and i have to close the browser manually.
here is the piece of code in question:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.common.by import By
import time
import keyboard
PATH = r"C:\Users\username"
options = Options()
options.binary_location = r'C:\Program Files (x86)\Google\Chrome\Application\chrome.exe'
ser = Service(executable_path=ChromeDriverManager().install())
cweb = webdriver.Chrome(service=ser, options=options)
class Coin:
def __init__(self, coinname, datasource):
self.coinname = coinname
self.datasource = datasource
def pulldata(self):
self.values = []
time.sleep(5)
cweb.get(self.datasource)
time.sleep(5)
price = cweb.find_element(By.XPATH, '//*[#id="__next"]/div[1]/div[1]/div[2]/div/div[1]/div[2]/div/div[2]/div[1]/div/span')
percent = cweb.find_element(By.XPATH, '//*[#id="__next"]/div[1]/div[1]/div[2]/div/div[1]/div[2]/div/div[2]/div[1]/span')
upordown = cweb.find_element(By.XPATH, '//*[#id="__next"]/div[1]/div[1]/div[2]/div/div[1]/div[2]/div/div[2]/div[1]/span/span')
caret = upordown.get_attribute('class')
if caret == 'icon-Caret-up':
daychange = ('' + percent.text)
elif caret == 'icon-Caret-down':
daychange2 = ('-' + percent.text)
vol = cweb.find_element(By.XPATH, '//*[#id="__next"]/div[1]/div[1]/div[2]/div/div[1]/div[2]/div/div[3]/div[1]/div[3]/div[1]/div[2]/div')
mkcap = cweb.find_element(By.XPATH, '//*[#id="__next"]/div[1]/div[1]/div[2]/div/div[1]/div[2]/div/div[3]/div[1]/div[1]/div/div[2]/div')
circsupply = cweb.find_element(By.XPATH, '//*[#id="__next"]/div[1]/div[1]/div[2]/div/div[1]/div[2]/div/div[3]/div[1]/div[4]/div[2]/div[1]')
self.values.append(price.text)
if caret == 'icon-Caret-up':
self.values.append(daychange)
elif caret == 'icon-Caret-down':
self.values.append(daychange2)
self.values.append(vol.text)
self.values.append(mkcap.text)
self.values.append(circsupply.text)
print(self.values)
def kill_chromedriver():
for proc in psutil.process_iter():
if proc.name() == "chromedriver.exe":
proc.kill()
if proc.name() == "chrome.exe":
proc.kill()
while True:
btc = Coin('Bitcoin','https://coinmarketcap.com/currencies/bitcoin/')
btc.pulldata()
btc2 = Coin('Bitcoin','https://coinmarketcap.com/currencies/tether/')
btc2.pulldata()
if keyboard.is_pressed('q'):
kill_chromedriver()
break
print('done reading')
so how do i edit this code, so that the driver closes and the loop stops on command?
ive tried positioning the break statement in different places, and also placing the cweb.close() function out of the loops. ill continue running through other options. i would be extremely grateful for any tips or help.
edits: took some help from an extremely useful lad, but it seems as if pressing the hotkey is recognized by the computer and python, but not recognized by selenium or the browser session itself.
wish i knew more about code like yall do lmao
G-man
I've found .quit() to fail sometimes with selenium. You can use psutil instead. Call this function in the if keyboard.is_pressed('q'): block.
import psutil
def kill_chromedriver():
for proc in psutil.process_iter():
if proc.name() == "chromedriver.exe":
proc.kill()
Edit: I think your issue might be related to read_key() not executing until the data are pulled. I tested this on my cpu in Jupyter and it worked.
while True:
btc = Coin('Bitcoin','https://coinmarketcap.com/currencies/bitcoin/')
btc.pulldata()
btc2 = Coin('Bitcoin','https://coinmarketcap.com/currencies/tether/')
btc2.pulldata()
print("You can press q now.")
if keyboard.read_key() == "q":
kill_chromedriver()
break
Related
Cant send commands to selenium webdriver in detached session because link http://localhost:port died.
But if i put breakpoint 1 link stay alive
import multiprocessing
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
def create_driver_pool(q):
options = Options()
driver = webdriver.Chrome(options=options)
pass #breakpoint 1
return driver.command_executor._url
windows_pool = multiprocessing.Pool(processes=1)
result = windows_pool.map(create_driver_pool, [1])
print(result)
pass # breakpoint 2 for testing link
why is this happening and what can i do about it?
After some research i finally found the reason for this behavor.
Thanks https://bentyeh.github.io/blog/20190527_Python-multiprocessing.html and some googling about signals.
This is not signals at all.
I found this code in selenium.common.service
def __del__(self):
print("del detected")
# `subprocess.Popen` doesn't send signal on `__del__`;
# so we attempt to close the launched process when `__del__`
# is triggered.
try:
self.stop()
except Exception:
pass
This is handler for garbage collector function, that killing subprocess via SIGTERM
self.process.terminate()
self.process.wait()
self.process.kill()
self.process = None
But if you in the debug mode with breakpoint, garbage collector wont collect this object, and del wont start.
I launch multiple selenium sessions with multiprocessing and want to close them all or a few when user wants it
schema of my code:
from multiprocessing import Process
def drop():
...
driver = webdriver.Chrome(executable_path=os.path.join(path, '\\chromedriver.exe'),chrome_options=options)
...
for i in range(10):
s=Process(target = drop)
s.start()
input('press enter to stop all sessions')
###close selenium driver event
you can create an array process_list = [] and append the 's' variable in the list for every iteration process_list.append(s) and then to stop all processes
for i in process_list:
driver.close()
i.terminate()
Adding each driver instance to a list when creating them and then iterating through them one by one to close them afterwards might be able to solve your issue.
from multiprocessing import Process
list_drivers = []
def drop():
...
driver = webdriver.Chrome(executable_path=os.path.join(path, '\\chromedriver.exe'),chrome_options=options)
list_drivers.append(driver)
...
for i in range(10):
s=Process(target = drop)
s.start()
input('press enter to stop all sessions')
for driver in list_drivers:
driver.quit()
###close selenium driver event
I want to run multiple chrome instances with selenium. I tried to loop the webdrivers but selenium keeps shutting the instances down.
Here is the code:
from selenium import webdriver
user = str(input("Do you want to run this program? "))
amount = 0
if user == "yes":
amount = int(input("How many instances do you want to run? "))
for w in range(1, amount+1):
webdriver.Chrome("path of my driver")
elif user == "no":
print("Program is closing...")
else:
print("Invalid input")
The weird thing is that the instances wont close if i write them without a loop:
from selenium import webdriver
user = str(input("Do you want to run this program? "))
if user == "yes":
driver1 = webdriver.Chrome("path of driver")
driver2 = webdriver.Chrome("path of driver")
driver3 = webdriver.Chrome("path of driver")
driver4 = webdriver.Chrome("path of driver")
driver5 = webdriver.Chrome("path of driver")
elif user == "no":
print("Program is closing...")
else:
print("Invalid input")
Is there any solution for my problem?
To close the instances when you write it without a loop then do the following.
driver.close() - It closes the the browser window on which the focus
is set.
driver.quit() – It basically calls driver.dispose method which in turn
closes all the browser windows and ends the WebDriver session
gracefully.
You should use driver.quit whenever you want to end the program. It will close all opened browser window and terminates the WebDriver session.
Consider writing your loop example in the following way, this will handle shutting the instances much better.
Using either multithreading or multiprocessing or subprocess module to trigger the task in parallel (near parallel).
Multithreading example
from selenium import webdriver
import threading
import time
def test_logic():
driver = webdriver.Firefox()
url = 'https://www.google.co.in'
driver.get(url)
# Implement your test logic
time.sleep(2)
driver.quit()
N = 5 # Number of browsers to spawn
thread_list = list()
# Start test
for i in range(N):
t = threading.Thread(name='Test {}'.format(i), target=test_logic)
t.start()
time.sleep(1)
print t.name + ' started!'
thread_list.append(t)
# Wait for all thre<ads to complete
for thread in thread_list:
thread.join()
print 'Test completed!'
Here, spawning 5 browsers to run test cases at one time. Instead of implementing the test logic I have put sleep time of 2 seconds for the purpose of demonstration. The code will fire up 5 firefox browsers (tested with python 2.7), open google and wait for 2 seconds before quitting.
Logs:
C:\Python27\python.exe C:/Users/swadchan/Documents/TestPyCharm/stackoverflow/so49617485.py
Test 0 started!
Test 1 started!
Test 2 started!
Test 3 started!
Test 4 started!
Test completed!
Process finished with exit code 0
I am working on Crawl project using Selenium and Webdriver. Since that data I need to crawl while big, I want to split it to 2 threads and run at the same time. But when I start 2 Webdrivers at the same time, my code can not recognize which driver belong to which thread to fill the information.
This is the code that I setup 2 threads run main function:
if __name__ == '__main__':
data = load_data(INPUT_DIR)
t1_data = data[:250]
t2_data = data[250:]
try:
_thread.start_new_thread(main, (t1_data, LOG_FILE_T1))
_thread.start_new_thread(main, (t2_data, LOG_FILE_T2))
except:
print ("Error: unable to start thread")
while 1:
pass
This code I start the Webdriver:
def start_driver():
global driver
options = Options()
options.add_argument("--disable-notification")
options.add_argument("--disable-infobars")
options.add_argument("--mute-audio")
#options.add_argument("headless")
driver = webdriver.Chrome(options=options)
Two Webdrivers after started, I will fill in the username/password
information on facebook.com
def login(email, password):
""" Logging into our own profile """
try:
driver.get('https://mbasic.facebook.com')
time.sleep(DELAY_TIME)
driver.find_element_by_name('email').send_keys(email)
driver.find_element_by_name('pass').send_keys(password)
driver.find_element_by_name('login').click()
# deal with "Not Now" button if it show off at first time
not_now_button = driver.find_element_by_xpath("//a")
if not_now_button.size != 0:
not_now_button.click()
except Exception as e:
print('Error in Login')
print(e)
exit()
At send_keys step, both threads fill in the same text box in 1 Webdriver.
How can I change my code for 2 threads can see different Webdrive and fill the info in ?
Just found out the solution, I want to share here if someone needed.
Instead of global driver, I will change to a local driver and pass it to each function.
def start_driver():
options = Options()
options.add_argument("--disable-notification")
options.add_argument("--disable-infobars")
options.add_argument("--mute-audio")
#options.add_argument("headless")
driver = webdriver.Chrome(options=options)
return driver
def login(driver, email, password):
""" Logging into our own profile """
try:
driver.get('https://mbasic.facebook.com')
time.sleep(DELAY_TIME)
driver.find_element_by_name('email').send_keys(email)
driver.find_element_by_name('pass').send_keys(password)
driver.find_element_by_name('login').click()
# deal with "Not Now" button if it show off at first time
not_now_button = driver.find_element_by_xpath("//a")
if not_now_button.size != 0:
not_now_button.click()
except Exception as e:
print('Error in Login')
print(e)
exit()
By this, 2 threads can see different Webdriver and fill their own information.
I haven't started on the web typing part but I need help with the starting part.
Here is my code
import webbot
import os
import time
import random
os.system('pip3 install pyautogui')
driver = webbot.Browser()
print("")
driver.go_to('https://kahoot.it/')
code = input("The game id: ")
input_elements = driver.find_elements(xpath='//input')
driver.type(driver.Key.TAB,into=input_elements[0].text)
driver.type(code)
driver.type(driver.Key.ENTER,into=input_elements[0].text)
time.sleep(1.5)
username = "test"
input_elements = driver.find_elements(xpath='//input')
driver.type(driver.Key.TAB,into=input_elements[0].text)
driver.type(username)
driver.type(driver.Key.ENTER,into=input_elements[0].text)
time.sleep(2)
driver.execute_script("window.open(' ');")
time.sleep(2)
driver.go_to('https://kahoot.it/')
# keeps the loop running
while True:
time.sleep(1)
I use repl.it's pygame for this
Edit: nvm I fixed the new tab now I need to type and hit enter into the boxes