I'm building a bot to print prizes of bonds from hour to hour. However, I am incurring in two errors:
Since the task requires the browser to open from time to time, it ruins the experience while using the notebook. Is there a way to keep this task as a 'background' rule?
I am using schedule library to set the update, but I am not quite sure it is right (even though I read the manual). Or the time set is not respected (I set to 10 minutes and the code is read from 5 to 5) or the function time it is not updated (it repeats minutes/hours/seconds).
The code is below:
import sys
import os
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
import time
import datetime
import schedule
clock = datetime.datetime.now()
def preço():
os.chdir('C:/Users/Thiago/Desktop/Backup/Python')
options = webdriver.ChromeOptions()
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=options)
driver.get("https://www.google.com/")
elem = driver.find_element_by_name("q")
elem.clear()
elem.send_keys("cvcb3")
time.sleep(1)
elem = driver.find_element_by_name("btnK")
elem.click()
time.sleep(2)
cvcb3 = driver.find_element_by_xpath(".//span[#jsname = 'vWLAgc']")
preço_cvcb3 = open('preço_cvcb3.txt', 'a')
preço_cvcb3.write('O preço da ação da CVC é ' + cvcb3.get_attribute("innerHTML") + ' - Extração feita ás ' + clock.strftime("%I:%M:%S %p") + '.\n')
preço_cvcb3.close()
print('O preço da ação da CVC é ' + cvcb3.get_attribute("innerHTML") + ' - Extração feita ás ' + clock.strftime("%I:%M:%S %p") + '.\n')
driver.close()
schedule.every(1).minutes.do(preço)
while True:
schedule.run_pending()
time.sleep(1)
Related
Hey guys I have a problem with the if-else conditions.I was creating the following bot that searches and alerts me when theres a appointment available but I can't make the if-else conditons work at the final lines of the code,the bot doesn't respect the if-else conditions I've tried changing several times the code but no idea of how to resolve this problem.Thanks for the help.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.action_chains import ActionChains
from time import sleep
import time
import re
import os
import winsound
duration = 2000 # milliseconds
freq = 900 # Hz
lets_go = True
while lets_go == True:
browser = webdriver.Chrome()
browser.implicitly_wait(30)
browser.maximize_window()
chrome_options = Options()
chrome_options.add_experimental_option("detach", True)
browser.get("https://icp.administracionelectronica.gob.es/icpplus/index.html")
browser.verificationErrors = []
cookie_kill = browser.find_element_by_id("cookie_action_close_header")
cookie_kill.click()
#sleep(1)
madrid = browser.find_element_by_xpath('//*[#id="form"]/option[34]')
madrid.click()
#sleep(1)
accept = browser.find_element_by_id("btnAceptar")
accept.click()
#sleep(1)
browser.execute_script("window.scrollTo(0,document.body.scrollHeight)")
tramites_group = browser.find_element_by_xpath('/html/body/div[1]/div[2]/main/div/div/section/div[2]/form[1]/div[3]/div[1]/div[2]/div/fieldset/div[2]/select/option[3]')
tramites_group.click()
sleep(1)
aceptar = browser.find_element_by_id("btnAceptar")
aceptar.click()
sleep(1)
browser.execute_script("window.scrollTo(0,document.body.scrollHeight)")
enter_button = browser.find_element_by_id('btnEntrar')
enter_button.click()
sleep(1)
passport = browser.find_element_by_id("rdbTipoDocPas")
passport.click()
passport_number = browser.find_element_by_id("txtIdCitado").send_keys("123456789")
person_name = browser.find_element_by_id("txtDesCitado").send_keys("BORIS JOHNSON")
person_birth = browser.find_element_by_id("txtAnnoCitado").send_keys("1900")
nationality = browser.find_element_by_xpath('/html/body/div[1]/div[2]/main/div/div/section/div[2]/form/div/div/div[1]/div[5]/div/div/div/div/span/select/option[200]')
nationality.click()
browser.execute_script("window.scrollTo(0,document.body.scrollHeight)")
sleep(1)
enviar = browser.find_element_by_id("btnEnviar")
browser.execute_script("arguments[0].click();", enviar)
sleep(1)
enviar = browser.find_element_by_id("btnEnviar")
browser.execute_script("arguments[0].click();", enviar)
sleep(1)
no_appointments = browser.page_source.find("En este momento no hay citas disponibles.")
if no_appointments:
browser.close()
time.sleep(120)
else:
winsound.Beep(freq, duration)
print("found")
lets_go = False
break
page_source returns a normal Python string. The find method of a string does not return a boolean True/False. It returns the starting character number if found, and -1 if not found. Thus, you want:
no_appointments = browser.page_source.find("En este momento no hay citas disponibles.")
if no_appointments >= 0:
browser.close()
time.sleep(120)
You might consider whether it makes more sense to write:
if "En este momento no hay citas disponibles" in browser.page_source:
I have a webscraping, but the site I'm using in some days is slow and sometimes not. Using the fixed SLEEP, it gives an error in a few days. How to fix this?
I use SLEEP in the intervals of the tasks that I have placed, because the site is sometimes slow and does not return the result giving me an error.
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.firefox import options
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.ui import Select
import pandas as pd
import json
from time import sleep
options = Options()
options.headless = True
navegador = webdriver.Firefox(options = options)
link = '****************************'
navegador.get(url = link)
sleep(1)
usuario = navegador.find_element(by=By.ID, value='ctl00_ctl00_Content_Content_txtLogin')
usuario.send_keys('****************************')
sleep(1)
senha = navegador.find_element(by=By.ID, value='ctl00_ctl00_Content_Content_txtSenha')
senha.send_keys('****************************')
sleep(2.5)
botaologin = navegador.find_element(by=By.ID, value='ctl00_ctl00_Content_Content_btnEnviar')
botaologin.click()
sleep(40)
agendamento = navegador.find_element(by=By.ID, value='ctl00_ctl00_Content_Content_TreeView2t8')
agendamento.click()
sleep(2)
selecdia = navegador.find_element(By.CSS_SELECTOR, "a[title='06 de dezembro']")
selecdia.click()
sleep(2)
selecterminal = navegador.find_element(by=By.ID, value='ctl00_ctl00_Content_Content_ddlVagasTerminalEmpresa')
selecterminal.click()
sleep(1)
select = Select(navegador.find_element(by=By.ID, value='ctl00_ctl00_Content_Content_ddlVagasTerminalEmpresa'))
select.select_by_index(1)
sleep(10)
buscalink = navegador.find_elements(by=By.XPATH, value='//*[#id="divScroll"]')
for element in buscalink:
teste3 = element.get_attribute('innerHTML')
soup = BeautifulSoup(teste3, "html.parser")
Vagas = soup.find_all(title="Vaga disponível.")
print(Vagas)
temp=[]
for i in Vagas:
on_click = i.get('onclick')
temp.append(on_click)
df = pd.DataFrame(temp)
df.to_csv('test.csv', mode='a', header=False, index=False)
It returns an error because the page does not load in time and it cannot get the data, but this time is variable
Instead of all these hardcoded sleeps you need to use WebDriverWait expected_conditions explicit waits.
With it you can set some timeout period so Selenium will poll the page periodically until the expected condition is fulfilled.
For example if you need to click a button you will wait for that element clickability. Once this condition is found Selenium will return you that element and you will be able to click it.
This will reduce all the redundant delays on the one hand and will keep waiting until the condition is matched on the other hand (until it is inside the defined timeout).
So, your code can be modified as following:
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
#-----
wait = WebDriverWait(navegador, 30)
navegador.get(link)
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_ctl00_Content_Content_txtLogin"))).send_keys('****************************')
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_ctl00_Content_Content_txtSenha"))).send_keys('****************************')
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_ctl00_Content_Content_btnEnviar"))).click()
wait.until(EC.element_to_be_clickable((By.ID, "ctl00_ctl00_Content_Content_TreeView2t8"))).click()
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "a[title='06 de dezembro']"))).click()
etc.
I used web scrapping through Python with selenium in order to get daily price values for EEX French Power futures at the url "https://www.eex.com/en/market-data/power/futures#%7B%22snippetpicker%22%3A%2221%22%7D".
I guess they updated their website as the url changed recently, and now my script doesn't work properly anymore as I can't find a way to click on each displayed product button (Year, Quarter, Month, Weekend, Day).
Here is my code until the step that doesn't work (it simply doesn't click, it doesn't fail) :
import time
import datetime
from datetime import date
from dateutil.relativedelta import relativedelta
import pyodbc
from selenium.webdriver.common.action_chains import ActionChains
import pandas as pd
from selenium.webdriver.support import expected_conditions as EC
url = "https://www.eex.com/en/market-data/power/futures#%7B%22snippetpicker%22%3A%2221%22%7D"
dico_product = ('Day', 'Weekend', 'Week', 'Month', 'Quarter', 'Year')
now = datetime.datetime.now()
date_prx = now.date()
options=Options()
d = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
d.get(url)
time.sleep(6)
d.maximize_window()
cookies_button_str = "//input[#class='btn bordered uo_cookie_btn_type_1']"
d.find_element(By.XPATH, cookies_button_str).click()
time.sleep(4)
dateinput_button_str = "//div[#class = 'mv-date-input']//div[#class = 'mv-stack-block']//input[#class = 'mv-input-box']"
Date_input = date_prx
Date_input_str = str(Date_input.year) + '-' + str(Date_input.month) + '-' + str(Date_input.day)
element_view = d.find_element(By.CLASS_NAME, 'collapsed')
d.execute_script("arguments[0].scrollIntoView()", element_view)
WebDriverWait(d, 20).until(EC.presence_of_element_located((By.XPATH, dateinput_button_str)))
element = d.find_element(By.XPATH, dateinput_button_str)
time.sleep(2)
d.execute_script('arguments[0].value = "' + str(Date_input_str) + '";', element)
time.sleep(2)
element_button_str = './/div[contains(#class, "mv-button-base mv-hyperlink-button")]'
containers = d.find_elements(By.XPATH, element_button_str)
for item in containers:
if item.text in dico_product:
print('Traitement ' + str(item.text) + ' pour la date ' + str(Date_input_str) + '.')
element_button_str = './/div[contains(#class, "' + str(item.get_attribute("class")) + '") and contains(., "' + str(item.text) + '")]'
product_button = d.find_element(By.XPATH, element_button_str)
d.execute_script("arguments[0].click()", product_button)
It does find the element to click on, but it doesn't click.
What is suprising is that if you take the old url, that get you to the Austrian futures by default, it works fine. But if you take the proper url, it doesn't.
I don't know if it can be done or if it's no use, but honestly I tried everything I could think of. Could you gently help me ?
Thank you
Im cannot select the year.
Run the code please an you will see what happens at the end.
I've tried many ways.
Cannot find the solution.
# -*- coding: utf-8 -*-
from time import time
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.ui import Select
from datetime import datetime
from datetime import date
driver = webdriver.Chrome()
paginaHit = 'https://hit.com.do/solicitud-de-verificacion/'
driver.get(paginaHit)
driver.maximize_window()
embed = driver.find_element(By.CSS_SELECTOR, "embed")
driver.switch_to.frame(embed)
bl = 'SMLU7270944A'
clasificacion = 'Mudanzas'
wait =WebDriverWait(driver, 20)
wait.until(EC.visibility_of_element_located((By.ID, "billoflanding"))).send_keys(bl)
seleccion = Select(driver.find_element(By.ID, "cboClasificación"))
seleccion.select_by_visible_text(clasificacion)
btnBuscar = driver.find_element(By.XPATH, '/html/body/div/app-root/div/form/div/div[3]/div/button').click()
time.sleep(4)
consignatario = driver.find_element(By.ID, 'cosignatario').send_keys("LOGISTICA ADUANAL")
# seleccionMercancia = Select(driver.find_element(By.XPATH, '/html/body/div/app-root/div/div[2]/datos-generales/form/div/div[9]/div/select'))
# seleccionMercancia.select_by_visible_text("Articulos del hogar")
condicion = Select(driver.find_element(By.XPATH, '/html/body/div/app-root/div/div[2]/datos-generales/form/div/div[10]/div/select'))
condicion.select_by_visible_text("Verificación")
solicitante = driver.find_element(By.ID, "nombreVisitante").send_keys("JONATHAN MENDEZ GARCIA")
correo = driver.find_element(By.ID, "correo").send_keys("laduanal#gmail.com")
telefono = driver.find_element(By.ID, "telefono").send_keys("8098013610")
tipoDocumento = Select(driver.find_element(By.XPATH,'/html/body/div/app-root/div/div[2]/datos-generales/form/div/div[16]/div/select'))
tipoDocumento.select_by_visible_text("Cédula")
cedula = driver.find_element(By.ID, "cedulaVisitante2").send_keys("00111452470")
nombreYapellido = driver.find_element(By.ID, "text01").send_keys("JONATHAN MENDEZ GARCIA")
tipoDocumento2 = Select(driver.find_element(By.XPATH, '/html/body/div/app-root/div/visitante-form/form/div/div[3]/div/select'))
tipoDocumento2.select_by_visible_text("Cédula")
rolVisitante = Select(driver.find_element(By.XPATH, '/html/body/div/app-root/div/visitante-form/form/div/div[4]/div/select'))
rolVisitante.select_by_visible_text("Representante")
cedulaVisitante = driver.find_element(By.ID, "cedulaVisitante").send_keys("00111452470")
btnAgregarPersonal = driver.find_element(By.XPATH, '/html/body/div/app-root/div/visitante-form/form/div/div[7]/div/div[1]/button').click()
#SELECCIONAR FECHA DE VERFICACION
fechaDeseada = "0929 2022"
fechaVerificacion = driver.find_element(By.ID, "fechaVerificar")
fechaVerificacion.send_keys(fechaDeseada)
The problem is in the last 3 lines of the code, but you need to run it in order to see what happen. Will apreciate any help. Im trying to fill up a form because is a task that need to be done all day long.
For whatever reason, the date field on hit.com.do is buggy, so I had to enter the year first, then go back to entering the day & month. I'd replace your last three lines with:
fechaVerificacion = driver.find_element(By.ID, "fechaVerificar")
fechaVerificacion.send_keys(Keys.TAB, Keys.TAB, "2022", Keys.LEFT, Keys.LEFT, "2909")
I use that script for WhatsApp integration with my website in python.
Why My that Script is not working can you please check here in contact i maintain my contactfile and when use ipdb for trace code it show me error in line 30.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
import time
driver = webdriver.Chrome('./chromedriver')
driver.get('https://web.whatsapp.com/')
wait = WebDriverWait(driver, 600)
with open("contact") as f:
for line in f:
try:
name = (line.rstrip('\n'))
text = """Hello {}, My name is Rhevin
Have a good day :)
""".format(name)
print 'Harish'
inp_xpath_search = "//input[#title='Search or start new chat']"
input_box_search = wait.until(EC.presence_of_element_located((
By.XPATH, inp_xpath_search)))
print '################',input_box_search
input_box_search.send_keys(name + Keys.ENTER)
#x_arg_contact = '//div[#class="chat-title"]'
x_arg_contact = '//span[contains(#title,' + name + ')]'
input_box_contact = wait.until(EC.presence_of_element_located((
By.XPATH, x_arg_contact)))
print '-------------------',input_box_contact
inp_xpath = "//div[#contenteditable='true']"
input_box = wait.until(EC.presence_of_element_located((
By.XPATH, inp_xpath)))
print '+++++++++++++++++++++++',input_box
input_box.send_keys(text + Keys.ENTER)
print '==============================='
time.sleep(5)
print("send")
except:
print("error")