Python- Scrape LinkedIn Names - python

My code will get to the desired webpage by passing in log in and password info.
(You can try any username and password that has an account with LinkedIn in the code below)
I just need to know how to scrape the information, now that I got to the desired page.
If I can start with the names per listing, it would be great.
import time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
productlinks=[]
test1=[]
options = Options()
driver = webdriver.Chrome(ChromeDriverManager().install())
url = "https://www.linkedin.com/uas/login?session_redirect=https%3A%2F%2Fwww%2Elinkedin%2Ecom%2Fsearch%2Fresults%2Fpeople%2F%3FcurrentCompany%3D%255B%25221252860%2522%255D%26geoUrn%3D%255B%2522103644278%2522%255D%26keywords%3Dsales%26origin%3DFACETED_SEARCH%26page%3D2&fromSignIn=true&trk=cold_join_sign_in"
driver.get(url)
time.sleep(2)
username = driver.find_element_by_id('username')
username.send_keys('Example#gmail.com')
password = driver.find_element_by_id('password')
password.send_keys('ExamplePassword')
password.submit()

You can scrape html element using their Class Name, with method find_elements_by_class_name
See an example below:
import time
from selenium import webdriver
options = webdriver.ChromeOptions()
options.add_argument('--disable-gpu')
options.add_argument('--ignore-certificate-errors-spki-list')
options.add_argument('--ignore-ssl-errors')
driver = webdriver.Chrome(chrome_options=options)
driver.get("https://www.linkedin.com/login")
username = driver.find_element_by_id('username')
username.send_keys('User_example')
password = driver.find_element_by_id('password')
password.send_keys('Password_example')
password.submit()
time.sleep(1)
MyProfileName=driver.find_elements_by_class_name("profile-rail-card__actor-link")
print("MyProfileName is: " + MyProfileName[0].text.strip())
print("\n List of Names: \n");
#Redirect to another link
driver.get("https://www.linkedin.com/onboarding/start/people-you-may-know/new/")
time.sleep(2)
Names = driver.find_elements_by_class_name("onboarding-card__person-title")
for name in Names:
print(name.text.strip())

Related

Python using selenium webdriver to fill a form, when submitting the form, chrome is saying no internet connection is available

I am using Selenium webdriver to fill out a form, the fields are good, but when I do a .submit using the login button, or do a .submit after the password field, the chrome browser will say "Unable to connect to internet. Please check your internet connection."
the internet is definitely up since I am able to open other websites on another browser (including a new session of chrome).
This is my code:
import time
import selenium
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
chrome_options = webdriver.ChromeOptions();
chrome_options.add_experimental_option("excludeSwitches", ['enable-automation']);
options = Options()
# options.add_argument('start-maximumized')
# options.add_argument('disable-infobars')
chrome_options.add_argument('no-sandbox')
PATH = "c:\scripts\chromedriver.exe"
# driver = webdriver.Chrome(executable_path=PATH)
driver = webdriver.Chrome(chrome_options=chrome_options,
executable_path='c:\scripts\chromedriver.exe')
driver.get('https://stupidwebsite.org')
time.sleep(1)
input_username = driver.find_element_by_id('username')
time.sleep(1)
input_username.send_keys("myself#email.com")
input_password = driver.find_element_by_id('password')
input_password.send_keys("mypassword#123")
input_password.submit()
# esubmit=driver.find_element_by_xpath
('/html/body/div/div[1]/div[2]/div/div/form/div[2]/div/button')
# esubmit = driver.find_element_by_name('MuiButton-label')
# esubmit.submit()
time.sleep(2)
# driver.quit()
I changed it a bit, and tried out google, and it works:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
#set chromodriver.exe path
driver = webdriver.Chrome(executable_path="C:\scripts\chromedriver.exe")
driver.implicitly_wait(0.5)
#launch URL
driver.get("https://www.google.com/")
#identify search box
m = driver.find_element_by_name("q")
#enter search text
m.send_keys("Tutorialspoint")
time.sleep(0.2)
#perform Google search with Keys.ENTER
m.send_keys(Keys.ENTER)
Please help me out. Thank you.

selenium chrome options (running in backgroud)

I want to get some information from a website and Chrome should run in the background to fulfill that task. Down there you can see my code. It works so far and I get the desired output, but when I add the chrome_options so that Chrome is hidden I don't get the output anymore.
What's the problem and how can I fix this?
from selenium import webdriver
def get_stockname(wkn):
PATH = r"***placeholder***chromedriver.exe"
url = "***placeholder***"
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("headless")
browser = webdriver.Chrome(PATH, options=chrome_options)
browser.get(url)
search_box = browser.find_element_by_class_name('input-field__text-input')
search_box.send_keys(wkn)
search_box.submit()
name = browser.find_element_by_xpath("/html/body/div[2]/div[1]/div[2]/div[13]/div[2]/div[1]/h2").text
name = name[13:]
print(name)
try like that, but with replacing your url:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
url = "https://google.com"
chrome_options = Options()
chrome_options.add_argument("--headless")
browser = webdriver.Chrome(options=chrome_options)
browser.get(url)
search_box = browser.find_element_by_class_name('input-field__text-input')
search_box.send_keys(wkn)
search_box.submit()
name = browser.find_element_by_xpath("(//h2[#class='box-headline'])[2]").get_attribute('innerText')
name = name[13:]
print(name)
By the way, can you show DOM code snipped to make element more unique with such external relation or even URL and element what info you want to get out?

Find elements by input type

I'm trying Python and Selenium. My goal is to log myself into Discord (https://discordapp.com/login. But here is the problem. I can't manage to get the email and password box selected. But the worst part is trying to select a textbox on a server... I tried everything, even locating by XPath, but I can't seem to do it right. Also, doing it on ATOM is probably not the best idea since I don't get any error messages :P. Here is a snippet to select the email textbox.
from selenium
import webdriver
from selenium.webdriver.common.keys
import Keys
browser = webdriver.Firefox()
browser.get('https://discordapp.com/login')
assert 'discordapp' in browser.title
elem = browser.find_element_by_name('textarea')# this is the part where i need help
elem.send_keys('test' + Keys.ENTER)
For email this css selector should work :
input[type='email']
For password :
input[type='password']
I've tested this code :
browser.get("https://discordapp.com/login")
elem = browser.find_element_by_css_selector("input[type='email']")# this is the part where i need help
elem.send_keys("itsolidude#imail.com")
elem1 = browser.find_element_by_css_selector("input[type='password']")# this is the part where i need help
elem1.send_keys("password")
login_button = browser.find_element_by_xpath("//div[text()='Login']/parent::button")
login_button.click()
This worked fine on my machine.
you need to check the div container and add them into the xpath.
Try the following code and please debug the indents, in case that stackoverflow is not transferring them properly (well, I don't know how to do it nice and correctly.)
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
class loginPage():
def test(self):
baseUrl = 'https://discordapp.com/login'
driver = webdriver.Firefox(executable_path="G:\\webdriver/geckodriver.exe")
driver.maximize_window()
driver.implicitly_wait(5)
driver.get(baseUrl)
mail = driver.find_element(By.XPATH, "//div[3]/div[1]/div/input[contains(#type,'email')]")
time.sleep(5)
mail.send_keys("test#gmail.com")
time.sleep(3)
print("Enter mail adress")
password = driver.find_element(By.XPATH, "//div[3]/div[2]/div/input[contains(#type,'password')]")
time.sleep(5)
password.send_keys("123456789")
time.sleep(3)
print("Enter password")
time.sleep(10)
driver.quit()
ff = loginPage()
ff.test()
Login To Discord Website using Python and Selenium:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
driver = webdriver.Chrome()
driver.get("https://discord.com/login")
time.sleep(6)
username_input = driver.find_element_by_name('email')
username_input.send_keys("enter-your-username-here")
password_input = driver.find_element_by_name('password')
password_input.send_keys("Enter-your-password-here")
login_button = driver.find_element_by_xpath('//*[#id="app-mount"]/div[2]/div/div[2]/div/div/form/div/div/div[1]/div[3]/button[2]')
login_button.click()

Filling in login forms in Instagram using selenium and webdriver (chrome) python OSX

I want to log in to instagram using selenium, but I can't seem to enter values into the fields.
Here's my script:
#go to this address
browser.get('https://www.instagram.com')
#sleep for 1 seconds
sleep(1)
#find the 'login' button on homepage
login_elem = browser.find_element_by_xpath(
'//*[#id="react-root"]/section/main/article/div[2]/div[2]/p/a')
#navigate to login page
login_elem.click()
Having trouble from here onwards:
#locate the username field within the form
unform = browser.find_element_by_xpath(
'//*[#id="f3b8e6724a27994"]')
#clear the field
textunform.clear()
#enter 'test' into field
unform.send_keys('test')
There is a trick in this, instead of searching for the Button (Log In) there is a better way to log in without it. how? let's see:
Import the packages you need:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from time import sleep
#Select the driver, In our case we will use Chrome.
chromedriver_path = 'chromedriver.exe' # Change this to your own chromedriver path!
webdriver = webdriver.Chrome(executable_path=chromedriver_path)
sleep(2)
webdriver.get('https://www.instagram.com/accounts/login/?source=auth_switcher')
sleep(3)
username = webdriver.find_element_by_name('username')
username.send_keys('yourUsername')
password = webdriver.find_element_by_name('password')
password.send_keys('yourPassword')
#instead of searching for the Button (Log In) you can simply press enter when you already selected the password or the username input element.
submit = webdriver.find_element_by_tag_name('form')
submit.submit()
You can copy the code and run it directly (even without a real username or password)
To get the webdriver (chromedriver.exe) from ChromeDriver
The instagram is applying some method to leave the dynamic id, xpath and css, every time a reload happens on the page the attributes change their values, being more difficult to click or to set values:
I solved it:
#Locate the username field
unform = browser.find_element_by_name("username")
#Locate the password field
pwform = browser.find_element_by_name('password')
ActionChains(browser)\
.move_to_element(unform).click()\
.send_keys('test')\
.move_to_element(pwform).click()\
.send_keys('test')\
.perform()
#Locate login button
login_button = browser.find_element_by_xpath(
'//*[#id="react-root"]/section/main/article/div[2]/div[1]/div/form/span/button')
#Click login button
login_button.click()
The username field on Instagram is a ReactJS so you have to induce WebDriverWait and then invoke send_keys() method as follows :
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
options = Options()
options.add_argument("start-maximized")
options.add_argument("disable-infobars")
options.add_argument("--disable-extensions")
browser = webdriver.Chrome(chrome_options=options, executable_path=r'C:\path\to\chromedriver.exe')
browser.get('https://www.instagram.com')
login_elem = browser.find_element_by_xpath('//*[#id="react-root"]/section/main/article/div[2]/div[2]/p/a')
login_elem.click()
WebDriverWait(browser, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input[name='username']"))).send_keys("anon")
Browser Screenshot :
In this case IMHO it's better to use this: browser.find_element_by_name("Bermil18") / browser.find_element_by_name("1q56y3w5t9k0p3es8i1q")
Here's my solution for Sign In on Instagram
def login(self, username, password):
""" Methods that log in to Instagram by taking user's credentials as parameters"""
self.driver.get("https://www.instagram.com/accounts/login/")
try:
self.driver.find_element_by_xpath("//input[#name=\"username\"]").send_keys(username) # filling username
self.driver.find_element_by_xpath("//input[#name=\"password\"]").send_keys(password) # filling password
self.driver.find_element_by_xpath("//button[#type=\"submit\"]").click() # submit form
except NoSuchElementException:
print("Failed to log in: Unable to locate Username/Password/LogIn element(s)")
# If login is unsuccessful, Instagram will show a message "Sorry, your password was incorrect. Please double-check your password."
success = self.driver.find_elements_by_xpath("//p[#id = \"slfErrorAlert\"]")
if len(success) == 0:
print("Login successful!")
else:
print("Sorry, sign in unsuccessful. Please double-check your credentials.")
See my Github repo for more: https://github.com/mlej8/InstagramBot
def login(username,password):
driver.get(base_url)
time.sleep(3)
detail = driver.find_elements_by_class_name('_2hvTZ')
detail[0].clear()
detail[1].clear()
detail[0].send_keys(username)
detail[1].send_keys(password)
driver.find_element_by_class_name('L3NKy').click()
time.sleep(3)
for i in driver.find_elements_by_tag_name('button'):
if i.text=='Not Now':
i.click()
break
time.sleep(3)
driver.find_element_by_class_name('HoLwm').click()
base url is intagram url .
I have a made an instabot and you can find the code for logging in ,follow, unfollow ,like ,check posts in recent day ,etc in the following github link.
https://github.com/Devanshchowdhury2212/Instagram-Web-scraping-
This worked for me:
def login(self, username):
self.driver = webdriver.Chrome()
self.driver.get('https://www.instagram.com/')
sleep(1)
username_input = self.driver.find_element_by_xpath(
"//input[#name='username']")
username_input.send_keys(username)
password_input = self.driver.find_element_by_xpath(
"//input[#name='password']")
password_input.send_keys(pw)
submit_btn = self.driver.find_element_by_xpath(
"//button[#type='submit']")
submit_btn.click()
sleep(2)
save_your_login_info_not_now = self.driver.find_element_by_xpath("/html/body/div[1]/section/main/div/div/div/div/button")
save_your_login_info_not_now.click()
You will notice that i am sending the variable pw instead of my actual password. This is for security reasons. Make a new file called secrets.py and inside it, declare your password in the following format:
pw = '*********'
Try to select the field with
unform = browser.find_element_by_xpath("//input[#name='username']")
unform.send_keys(<username>)
and for password
browser.find_element_by_xpath("//input[#name='password']")

Fill username and password using selenium in python

How can I auto fill the username and password over the link below:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
chromedriver = 'C:\\chromedriver.exe'
browser = webdriver.Chrome(chromedriver)
browser.get('http://www.example.com')
After that I really do not know:
username = Select(browser.find_element_by_name('Username'))
password = Select(browser.find_element_by_name('Password'))
username.select_by_visible_text("text")
password.select_by_visible_text("text")
Docs: https://selenium-python.readthedocs.io/navigating.html
For versions 4.3.0 (released in June 2022) and later, calls to find_element_by_* and find_elements_by_* were removed from Selenium. You need to use the new API:
from selenium.webdriver.common.by import By
driver = webdriver.Firefox(...) # Or Chrome(), or Ie(), or Opera()
# To catch <input type="text" id="passwd" />
password = driver.find_element(By.ID, "passwd")
# To catch <input type="text" name="passwd" />
password = driver.find_element(By.NAME, "passwd")
password.send_keys("Pa55worD")
driver.find_element(By.NAME, "submit").click()
The original response, for API versions 4.2.0 or previous:
driver = webdriver.Firefox(...) # Or Chrome(), or Ie(), or Opera()
username = driver.find_element_by_id("username")
password = driver.find_element_by_id("password")
username.send_keys("YourUsername")
password.send_keys("Pa55worD")
driver.find_element_by_name("submit").click()
A note to your code: Select() is used to act on a Select Element (https://developer.mozilla.org/en-US/docs/Web/HTML/Element/select).
Use WebElement.send_keys method to simulate key typing.
name in the code (Username, Password) does not match actual name of the elements (username, password).
username = browser.find_element_by_name('username')
username.send_keys('user1')
password = browser.find_element_by_name('password')
password.send_keys('secret')
form = browser.find_element_by_id('loginForm')
form.submit()
# OR browser.find_element_by_id('submit').click()
user = driver.find_element_by_name("username")
password = driver.find_element_by_name("password")
user.clear()
user.send_keys("your_user_name")
password.clear()
password.send_keys("your_password")
driver.find_element_by_name("submit").click()
Note:
we useuser.clear() in order to clear the input field.
for locating submit button you can use any other method based on the page source code. for info see locating elements
In some cases when the element is not interactable, sendKeys() doesn't work and you're likely to encounter an ElementNotInteractableException.
In such cases, you can opt to execute javascript that sets the values and then can post back.
Example:
url = 'https://www.your_url.com/'
driver = Chrome(executable_path="./chromedriver")
driver.get(url)
username = 'your_username'
password = 'your_password'
#Setting the value of email input field
driver.execute_script(f'var element = document.getElementById("email"); element.value = "{username}";')
#Setting the value of password input field
driver.execute_script(f'var element = document.getElementById("password"); element.value = "{password}";')
#Submitting the form or click the login button also
driver.execute_script(f'document.getElementsByClassName("login_form")[0].submit();')
print(driver.page_source)
Reference:
https://www.quora.com/How-do-I-resolve-the-ElementNotInteractableException-in-Selenium-WebDriver
Here is the complete answer.
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
chrome_driver_path = 'C:\\chromedriver.exe'
browser = webdriver.Chrome(service=Service(chrome_driver_path))
browser.get('http://www.example.com')
username = browser.find_element(By.NAME, 'Username')
password = browser.find_element(By.NAME, 'Password')
username.send_keys("yourUsername") #type your own username here
password.send_keys("yourPassword") #type your own password here
browser.find_element(By.NAME, 'submit').click()
Since find_element_by_name() is deprecated, you can use find_element(By.NAME, 'name').
Also you have to import from selenium.webdriver.common.by import By
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support.ui import WebDriverWait
# If you want to open Chrome
driver = webdriver.Chrome()
# If you want to open Firefox
driver = webdriver.Firefox()
username = driver.find_element_by_id("username")
password = driver.find_element_by_id("password")
username.send_keys("YourUsername")
password.send_keys("YourPassword")
driver.find_element_by_id("submit_btn").click()
I am new to selenium and I tried all solutions above but they don't work.
Finally, I tried this manually by
driver = webdriver.Firefox()
import time
driver.get(url)
time.sleep(20)
print (driver.page_source.encode("utf-8"))
Then I could get contents from web.

Categories