In this code I am trying to scrape a Linkedin profile using Selenium
but the driver is not able to load the page I guess IP has been
blocked and I am new to the concept of proxy rotating or any concept
that is used in such cases. It would be a great help if you could help
me understand how this is done.
from selenium import webdriver
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome(executable_path=r'C:\Users\chromedriver.exe')
def linkedin_login():
global driver
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_experimental_option('excludeswitches', ['enable-automation'])
options.add_experimental_option("detach", True)
try:
driver.get('https://www.linkedin.com/login')
username = 'username'
password = 'password'
WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.ID, 'username'))).send_keys(username)
driver.find_element_by_id('password').send_keys(password)
driver.find_element_by_class_name('btn__primary--large from__button--floating').click()
time.sleep(8)
except ImportError:
print('Closing')
def search_profiles():
search_profile = input('What profile do you want to search?')
search_profile = search_profile.split()
search = search_profile[0] + "%20" + search_profile[1]
The "Sign In" button's class name is incorrect, specifically is missing a dot(.)
Instead of:
driver.find_element_by_class_name('btn__primary--large from__button--floating').click()
Use:
driver.find_element_by_class_name('btn__primary--large.from__button--floating').click()
That will click the button.
Also, if you run the code you shared, you are calling the webdriver but not calling your function.
I tested the below code and worked fine (remember to update your path and LinkedIn credentials):
from selenium import webdriver
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Chrome(executable_path='/home/armentaahumada/Downloads/chromedriver')
def linkedin_login():
global driver
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_experimental_option('excludeswitches', ['enable-automation'])
options.add_experimental_option("detach", True)
try:
driver.get('https://www.linkedin.com/login')
username = 'username'
password = 'password'
WebDriverWait(driver, 30).until(EC.element_to_be_clickable((By.ID, 'username'))).send_keys(username)
driver.find_element_by_id('password').send_keys(password)
driver.find_element_by_class_name('btn__primary--large.from__button--floating').click()
time.sleep(8)
except ImportError:
print('Closing')
def search_profiles():
search_profile = input('What profile do you want to search?')
search_profile = search_profile.split()
search = search_profile[0] + "%20" + search_profile[1]
linkedin_login()
Related
I was following along with a tutorial to create the below program to
auto log into a github account. When I run the program it takes me to the sign in
page but does not not autofill the fields and submit. The tutorial was using "find_element_by_id" method. However, it was not recognized. After some googling I came up with the "By" class and it seems to work well earlier in the program when I used "By.LINK_TEXT" but fails when using it for .ID?
or maybe it is an issue with the "send_keys" method?
Any assistance would be greatly appreciated.
from selenium.webdriver.common.keys import Keys
import re
from selenium.webdriver.common.by import By
options = webdriver.ChromeOptions()
options.add_experimental_option("detach", True)
browser = webdriver.Chrome(
options=options, executable_path="C:\\Windows\\chromedriver.exe")
browser.get("https://github.com")
signin_link = browser.find_element(By.LINK_TEXT, "Sign in")
signin_link.click()
username_box = browser.find_element(By.ID, "login_field")
username_box.send_keys("username")
password_box = browser.find_element(By.ID, "password")
password_box.send_keys("password")
password_box.submit()
`
I tried the various constants for the By class (name, id, link_text, etc.)
None of them seems to make any difference. I imagine the problem is somewhere else in the program.
Add WebDriveWait so the page can load before trying to pass the username and password
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import re
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
options = webdriver.ChromeOptions()
options.add_experimental_option("detach", True)
browser = webdriver.Chrome(
options=options, executable_path="C:\\Windows\\chromedriver.exe")
browser.get("https://github.com")
signin_link = browser.find_element(By.LINK_TEXT, "Sign in")
signin_link.click()
# wait until the page loads
WebDriverWait(browser, 10).until(EC.presence_of_element_located((By.ID, 'login_field')))
username_box = browser.find_element(By.ID, "login_field")
username_box.send_keys("username")
password_box = browser.find_element(By.ID, "password")
password_box.send_keys("password")
password_box.submit()
I'm new to coding. I'am trying to make a twitter bot but when I find XPaths and paste it in my code it gives an error
I tried to find the element with id, name, selector and paste it in my code but none of them worked
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import time
class TwitterBot:
def __init__(self , username , password) :
self.username = username
self.password = password
chrome_options = Options()
self.bot = webdriver.Chrome(ChromeDriverManager().install() , options = chrome_options)
def login(self):
bot = self.bot
bot.get("https://twitter.com/login")
time.sleep(5)
email = bot.find_element(By.XPATH , '/html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/div[2]/div[1]/div[1]/div[2]/div[2]/div[1]/div[1]/div[1]/div[5]/label[1]/div[1]/div[2]/div[1]/input[1]')
email.send_keys(self.username)
f = TwitterBot("blabla" ,"blabla")
f.login()
You need to learn how to create correct, short and unique locators. Very long absolute XPaths and CSS Selectors are extremely breakable.
Also you need to use WebDriverWait expected_conditions explicit waits, not a hardcoded delays.
The following code works:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument("start-maximized")
options.add_argument('--disable-notifications')
webdriver_service = Service('C:\webdrivers\chromedriver.exe')
driver = webdriver.Chrome(options=options, service=webdriver_service)
wait = WebDriverWait(driver, 20)
url = "https://twitter.com/login"
driver.get(url)
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "[autocomplete='username']"))).send_keys("ku-ku")
The result is:
My code is
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
import time
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
options = webdriver.ChromeOptions()
options.add_experimental_option("excludeSwitches", ["enable-logging"])
driver = webdriver.Chrome(options=options, executable_path=r'C:\Program Files (x86)\Chrome\Application\chromedriver.exe')
USERNAME = input ('Enter IXL Username: ')
PASSWORD = input ('Enter IXL Password: ')
LESSONNN = input ('Enter Lesson Link: ')
driver.get(LESSONNN)
driver.refresh()
USERNAMEBOX = driver.find_element_by_xpath('//*[#id="qlusername"]')
driver.find_element_by_xpath('//*[#id="qlusername"]').click()
USERNAMEBOX.send_keys(USERNAME)
PASSWORDBOX = driver.find_element_by_xpath('//*[#id="qlpassword"]')
driver.find_element_by_xpath('//*[#id="qlpassword"]').click()
PASSWORDBOX.send_keys(PASSWORD)
driver.find_element_by_xpath('//*[#id="qlsubmit"]').click()
time.sleep(3)
QUESTION1 = driver.find_element_by_class_name('old-space-indent').text
print(QUESTION1)
driver.execute_script("window.open('https://www.mathpapa.com/algebra-calculator.html', 'new_window')")
time.sleep(20)
ele = driver.find_element_by_xpath('//*[#id="source3"]')
ele.click()
driver.switch_to_window(driver.window_handles[0])
answerbox = driver.find_element_by_class_name('fillIn')
driver.find_element_by_class_name('fillIn').click()
answerbox.send_keys(answer1)
submitbutton = driver.find_element_by_xpath('//*[#id="yui_3_18_1_1_1613027229296_179"]/div[2]/button')
driver.find_element_by_xpath('//*[#id="yui_3_18_1_1_1613027229296_179"]/div[2]/button').click()
I get the error
Message: no such element: Unable to locate element
I tried driver wait and other options, none seem to work for me.
I need help making it wait until the element is present other wise it wont work.
Please help me nothing seems to work i cant seen to figure out why im getting this error as i created a new project with the same code and it works but it wouldnt work in this project.
Can't find any solution for auto login google (gmail) account.
Here is my code :
import os
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
opt = Options()
executable_path = r'chromedriver'
os.environ["webdriver.chrome.driver"] = executable_path
opt.add_extension(r'C:\Users\SAMSUNG\Desktop\SB.crx')
opt.add_extension(r'C:\Users\SAMSUNG\Desktop\proxy.crx')
opt.add_argument('--disable-blink-features=AutomationControlled')
opt.add_experimental_option("excludeSwitches", ["enable-automation"])
opt.add_experimental_option('useAutomationExtension', False)
opt.add_argument("window-size=1280,800")
usernameStr = 'x'
passwordStr = 'y'
driver = webdriver.Chrome(r'C:\Users\SAMSUNG\Desktop\chromedriver', options=opt)
time.sleep(5)
driver.get(('https://accounts.google.com/ServiceLogin?'
'service=mail&continue=https://mail.google'
'.com/mail/#identifier'))
# fill in username and hit the next button
username = driver.find_element_by_id('identifierId')
username.send_keys(usernameStr)
time.sleep(2)
nextButton = driver.find_element_by_id('identifierNext')
nextButton.click()
# wait for transition then continue to fill items
password = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.NAME, "password")))
password.send_keys(passwordStr)
signInButton = driver.find_element_by_id('passwordNext')
signInButton.click()
But i keep getting this error : This browser or app may not be secure. Learn more
Try using a different browser. If you’re already using a supported browser, you can refresh your screen and try again to sign in.
ERROR
i have the following code and its all working correctly except for 1 issue
the file that is downloaded does not go into the specified directory
what is the issue ?
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import re
import pandas as pd
import os
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import time
url = "THE URL"
username = 'USERNAME'
password = 'PASSWORD'
profile = webdriver.FirefoxProfile()
profile.set_preference("browser.download.folderList", 2)
profile.set_preference("browser.download.manager.showWhenStarting", False)
profile.set_preference("browser.download.dir", 'C://Users/USER/OneDrive/Documents/dest_folder')
profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/csv")
while True:
driver = webdriver.Firefox(executable_path=r'C:/Users/USER/OneDrive/Documents/pycharm/geckodriver.exe',firefox_profile=profile)
driver.implicitly_wait(100)
driver.get(url)
user_field = driver.find_element_by_id("ap_email")
pass_field = driver.find_element_by_id("ap_password")
sign_in = driver.find_element_by_id("signInSubmit")
user_field.send_keys(username)
pass_field.send_keys(password)
sign_in.click()
driver.get(url)
driver.implicitly_wait(100)
time.sleep(5)
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button.yms-button-primary-alt.ng-isolate-scope[csv-header='getCsvHeader'][ng-csv^='fetchData']"))).click()
time.sleep(5)
driver.close()
try changing your directory to this
r'C:\Users\USER\OneDrive\Documents\dest_folder'