We Keep Coding

Why doesn't my webscrape code accept cookies

I'm a beginner in web scrapping and I've followed a few YouTube videos about how to do this, but regardless to what I try I can't have my code accept the cookies.
This is the code I have so far:
from selenium import webdriver
import time
driver = webdriver.Safari()
URL = "https://www.zoopla.co.uk/new-homes/property/london/?q=London&results_sort=newest_listings&search_source=new-homes&page_size=25&pn=1&view_type=list"
driver.get(URL)
time.sleep(2) # Wait a couple of seconds, so the website doesn't suspect you are a bot
try:
driver.switch_to_frame('gdpr-consent-notice') # This is the id of the frame
accept_cookies_button = driver.find_element_by_xpath('//*[#id="save"]')
accept_cookies_button.click()
except AttributeError: # If you have the latest version of selenium, the code above won't run because the "switch_to_frame" is deprecated
driver.switch_to.frame('gdpr-consent-notice') # This is the id of the frame
accept_cookies_button = driver.find_element_by_xpath('//*[#id="save"]')
accept_cookies_button.click()
except:
pass # If there is no cookies button, we won't find it, so we can pass

I don't have safari webdriver but chrome webdriver, but I think they works similar. On chrome you close the cookie banner with this code
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver.get(URL)
# wait no more than 20 seconds for the `iframe` with id `gdpr-consent-notice` to appear, then switch to it
WebDriverWait(driver, 20).until(EC.frame_to_be_available_and_switch_to_it((By.ID, "gdpr-consent-notice")))
# click accept cookies button
driver.find_element(By.CSS_SELECTOR, '#save').click()

Click on Instagram post using Selenium Python

Hi I am using Selenium to visit a instagram profile and click on the first post. Language I am using is python. As an example, I am trying to open the profile instagram.com/yotta_life and I got the XPATH of first post : //*[#id=\"react-root\"]/section/main/article/div/div[1]/div[1]/div[1]/a/div/div[2]
I can visit the profile in Selenium but error message is showing, Element is not clickable at point (1015, 635)
This is my code :-
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
def login(driver):
username = "username" # <username here>
password = "password" # <password here>
# Load page
driver.get("https://www.instagram.com/accounts/login/")
# Login
driver.find_element_by_xpath("//div/input[#name='username']").send_keys(username)
driver.find_element_by_xpath("//div/input[#name='password']").send_keys(password)
driver.find_element_by_xpath("//span/button").click()
widgets_click = WebDriverWait(driver, 500000).until(
EC.element_to_be_clickable((By.XPATH, "/html/body/div[2]/div/div[2]/div/div/button"))
)
widgets_click.click()
def scrape(driver, account):
# Load account page
driver.get("https://www.instagram.com/{0}/".format(account))
# This is the first post click which is not working
postdivrow_click = WebDriverWait(driver, 500000).until(
EC.element_to_be_clickable((By.XPATH, "//*[#id=\"react-root\"]/section/main/article/div/div[1]/div[1]/div[1]/a/div/div[2]"))
)
postdivrow_click.click()
if __name__ == "__main__":
chromedriver = "Documents/chromedriver"
driver = webdriver.Chrome(chromedriver)
try:
login(driver)
followers = scrape(driver, "yotta_life")
finally:
driver.quit()
Problem here is that element is not clickable. I think, I am not using correct XPATH. What is the recommended way to do it.

Here, I have to use
driver.execute_script("window.scrollTo(0, Y)")
Y is the height, to scroll down the page.

how to access activity log of facebook using selenium

I logged in the facebook account through selenium and is on the home page. How can I go to activity log page using selenium ? As there is not direct id, it is difficult to get there.
from selenium import webdriver
from selenium.webdriver.support import ui
from selenium.webdriver.common.keys import Keys
def page_is_loaded(driver):
return driver.find_element_by_tag_name("body") != None
driver = webdriver.Firefox()
driver.get("https://www.facebook.com/")
wait = ui.WebDriverWait(driver, 10)
wait.until(page_is_loaded)
email_field = driver.find_element_by_id("email")
email_field.send_keys("email")
password_field = driver.find_element_by_id("pass")
password_field.send_keys("password")
password_field = driver.find_element_by_id("pass")
password_field.send_keys(Keys.RETURN)

You can use the find_element_by_link_text() method of the driver.
In addition to the code you already have, add this
logout_menu = driver.find_element_by_id('logoutMenu')
logout_menu.click()
log = driver.find_element_by_link_text('Activity Log')
log.click()
This might not always work because the logout menu sometimes takes a little while to load, but you can either add an implicit wait to the driver, or use ExpectedConditions to determine when the Activity Log element is visible and clickable.

How to click on confirmation button using Selenium with Python?

I have the following Code that goes to a URL(www.example.com), and clicks on a link(Example 1). (This part works fine)
from selenium import webdriver
driver = webdriver.Firefox()
driver.get("https://www.example.com")
link = driver.find_element_by_link_text('Example 1')
link.click()
Now, when we click on 'Example 1' link, it opens a confirmation window, with 2 buttons: 'Yes I am authorized user to this site' and 'No I am a new visitor to this site'
So, I wish to click on 'Yes I am authorized user to this site' and then finally enter my log-in credentials.
I have written these 2 lines, just below the above code, for clicking on that button. But these don't work.
button = driver.find_element_by_name("'Yes I am authorized user to this site'")
button.click()

If it is an alert window, you need to use the Alert command.
#import Alert
from selenium.webdriver.common.alert import Alert
from selenium import webdriver
driver = webdriver.Firefox()
driver.get("https://www.example.com")
link = driver.find_element_by_link_text('Example 1')
link.click()
Alert(driver).accept()
#to dismiss alert
#Alert(driver).dismiss()
I think this would have solved your query.

Based on the comment conversation, I would recommend both using an XPATH search (instead of Name or Id) and waiting for elements to be clickable or loaded. When web-driving or web-scraping, pages may intentionally or accidentally load slowly and this can cause issues if you have pauses or waits either hard coded or non-existent. This snippet of code should allow you to search Google using Selenium and Chromedriver (you can modify the driver function to use Firefox or something else if you'd like):
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import ElementNotVisibleException
from selenium.webdriver.chrome.options import Options
from time import sleep
def init_driver(drvr_path):
chrome_options = Options()
chrome_options.add_argument("--disable-extensions")
driver = webdriver.Chrome(drvr_path+'chromedriver.exe',chrome_options=chrome_options)
driver.wait = WebDriverWait(driver, 5)
return driver
def lookup(query, driver=None, drvr_path=''):
driver = None
if driver is None:
driver = init_driver(drvr_path)
driver.implicitly_wait(45) # Allow up to 45 Seconds for page to load
driver.get("http://www.google.com")
try:
box = driver.wait.until(EC.presence_of_element_located((By.XPATH, """//*[#id="lst-ib"]""")))
box.send_keys(query)
sleep(3) # Let you see the window open
button = driver.wait.until(EC.element_to_be_clickable((By.XPATH,"""//*[#id="sblsbb"]/button""")))
try:
button.click()
except ElementNotVisibleException, s:
print "Error Handled: "+str(s)
button = driver.wait.until(EC.element_to_be_clickable((By.XPATH,"""//*[#id="sblsbb"]/button""")))
try:
button.click()
except:
print "Could not search Google..."
return
resp=driver.page_source.encode('utf-8')
with open(query+'.html','wb') as f:
f.write(resp)
print 'Wrote the File...'
except:
print("Box or Button not found in google.com")
driver.quit()
For example, if your Chromedriver.exe file was located in your default Python path, you could do something like: lookup('Selenium Python XPATH Examples') and it should download an HTML file of the Google Search results. If you already have a Driver initialized, you could of course pass that to it.
Hope this helps

Try this code, hope it will help you
from selenium import webdriver
import time
driver = webdriver.Chrome('path to chromedriver\chromedriver.exe')
driver.get('https://www.example.com')
driver.maximize_window()
link = driver.find_element_by_link_text('Example 1')
link.click()
handles =driver.window_handles # this will give window handles
driver.switch_to.window(handles[1])
button = driver.find_element_by_name("'Yes I am authorized user to this site'")
button.click()

Python Selenium: Can't find the login element of costar.com

I am getting very frustrated trying to login to costar.com with python and selenium. I have tried it on the chrome browser and firefox browser, but can't figure out the correct code.
I have logged into other websites, but cannot figure out how to input text into the login boxes for this site.
Here's what I have so far:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
browser = webdriver.Chrome(executable_path="C:/Users/Gus Gabel/Anaconda/chromedriver.exe")
browser.get("http://www.costar.com")
browser.maximize_window()
#the username and password boxes are hidden until you press the login button on the home page
login = browser.find_element_by_id("loginLink")
login.click()
#now that the username and password boxes are available, I've tried finding the elements by class_name, xpath, (BY.NAME), id, etc and nothing has worked
#here are a few codes that haven't worked and the errors associated
user = browser.find_element_by_id('username')
NoSuchElementException: Message: no such element
user = browser.find_element_by_class_name('usernameNew')
NoSuchElementException: Message: no such element
#when i try to use the above code with "elements" instead of "element", no error message pops up.
user = browser.find_elements_by_class_name('usernameNew')
#but then whey i try to choose which element by doing this
user = browser.find_element_by_class_name('usernameNew')[0]
NoSuchElementException: Message: no such element
How is it possible that there can be a list of elements, but yet not have an initial element?
If anyone can figure out how to input text into the username and password text boxes of costar.com, I will be greatly appreciative. I can't figure this out for the life of me!

To enter text into an input box with Selenium (e.g, your user name into the username field), use the send_keys method of the element:
input_element.send_keys('some_text_string')
You can also try Selenium's action_chains module, as in the following code (untested), to get past the roadblock from #dm295's answer:
from selenium import webdriver
from selenium.webdriver.common import action_chains
driver = webdriver.Firefox()
driver.maximize_window()
driver.get("http://www.costar.com")
action = action_chains.ActionChains(driver)
login = driver.find_element_by_class_name("login-icon")
login.click()
driver.switch_to.frame(driver.find_element_by_id('custlogin'))
username = driver.find_element_by_id('username')
action.move_to_element(username).perform()
username.send_keys('Gus Gabel')

A couple of things:
(a) you have to wait until the page is loaded (or at least the part of the page that you are interested in)
(b) only maximized browser window works for me (depends on device/resolution)
(c) you are trying to click the wrong element
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
driver = webdriver.Firefox()
driver.maximize_window()
driver.get("http://www.costar.com")
try:
# wait 15 seconds till the login link is present
login = WebDriverWait(driver, 15).until(
EC.presence_of_element_located((By.CLASS_NAME, "login-icon"))
)
login.click()
# just sleep 5 seconds to show that the login link was clicked
time.sleep(5)
# do other stuff (probably fill in username and password) ...
finally:
driver.quit()
Read the documentation regards waiting in selenium.
EDIT
Manged to get one step further (switch to iframe), but the following code raises
selenium.common.exceptions.ElementNotVisibleException: Message:
Element is not currently visible and so may not be interacted with
from selenium import webdriver
driver = webdriver.Firefox()
driver.maximize_window()
driver.get("http://www.costar.com")
try:
login = driver.find_element_by_class_name("login-icon")
login.click()
# switch to the custlogin iframe
driver.switch_to.frame(driver.find_element_by_id('custlogin'))
username = driver.find_element_by_id('username')
# this will raise:
# selenium.common.exceptions.ElementNotVisibleException: Message:
# Element is not currently visible and so may not be interacted with
username.send_keys('username')
finally:
driver.quit()