We Keep Coding

Using Selenium to get a list of Instagram followers

I am trying to take an account of over 1,000,000 followers on instagram and add their usernames to a txt file. I am trying to use Selenium for this but my authorization for login fails every time i login. Any advice on how to get around this? I assume the site believes this is a hack but im not sure.
from selenium import webdriver as web
from selenium.webdriver.common.keys import Keys
import time
import random
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.chrome.options import Options
bot_username = 'null'
bot_password = 'Null'
profiles = ['Enter Here']
amount = 300
# 'usernames' or 'links'
result = 'usernames'
us = ''
class Instagram():
def __init__(self, username, password):
self.username = username
self.password = password
options = Options()
options.add_experimental_option("excludeSwitches", ["enable-logging"])
self.browser = web.Chrome("chromedriver",options=options)
self.browser.set_window_size(400, 900)
def close_browser(self):
self.browser.close()
self.browser.quit()
def login(self):
browser = self.browser
try:
browser.get('https://www.instagram.com')
time.sleep(random.randrange(3, 5))
# Enter username:
username_input = browser.find_element_by_name('username')
username_input.clear()
username_input.send_keys(self.username)
time.sleep(random.randrange(2, 4))
# Enter password:
password_input = browser.find_element_by_name('password')
password_input.clear()
password_input.send_keys(self.password)
time.sleep(random.randrange(1, 2))
password_input.send_keys(Keys.ENTER)
time.sleep(random.randrange(3, 5))
print(f'[{self.username}] Successfully logged on!')
except Exception as ex:
print(f'[{self.username}] Authorization fail')
self.close_browser()
def xpath_exists(self, url):
browser = self.browser
try:
browser.find_element_by_xpath(url)
exist = True
except NoSuchElementException:
exist = False
return exist
def get_followers(self, users, amount):
browser = self.browser
followers_list = []
for user in users:
browser.get('https://instagram.com/' + user)
time.sleep(random.randrange(3, 5))
followers_button = browser.find_element_by_xpath('/html/body/div[1]/section/main/div/ul/li[2]/a/span')
count = followers_button.get_attribute('title')
if ',' in count:
count = int(''.join(count.split(',')))
else:
count = int(count)
if amount > count:
print(f'You set amount = {amount} but there are {count} followers, then amount = {count}')
amount = count
followers_button.click()
loops_count = int(amount / 12)
print(f'Scraping. Total: {amount} usernames. Wait {loops_count} iterations')
time.sleep(random.randrange(8,10))
followers_ul = browser.find_element_by_xpath("/html/body/div[6]/div/div/div[2]")
time.sleep(random.randrange(5,7))
try:
for i in range(1, loops_count + 1):
browser.execute_script("arguments[0].scrollTop = arguments[0].scrollHeight", followers_ul)
time.sleep(random.randrange(8, 10))
all_div = followers_ul.find_elements_by_tag_name("li")
for us in all_div:
us = us.find_element_by_tag_name("a").get_attribute("href")
if result == 'usernames':
us1 = us.replace("https://www.instagram.com/", "")
us = us1.replace("/", "")
followers_list.append(us)
time.sleep(1)
f3 = open('userlist.txt', 'w')
for list in followers_list:
f3.write(list + '\n')
print(f'Got: {len(followers_list)} usernames of {amount}. Saved to file.')
time.sleep(random.randrange(3, 5))
except Exception as ex:
print(ex)
self.close_browser()
return followers_list
bot = Instagram(bot_username, bot_password)
bot.login()
followers = bot.get_followers(profiles, amount)

How to fix 'Options' object has no attribute 'add_experimental_option' in Selenium Python

Environment:
Python 3.10
selenium==4.3.0
Windows 10
Problem
I am trying to make a selenium script with Chrome.
chrome_options = Options()
chrome_options.add_argument(r"--user-data-dir=" + chrome_profile_path)
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-application-cache")
chrome_options.add_argument("--disable-session-crashed-bubble")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--start-maximized")
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension', False)
I execute the code from my Windows terminal. It creates this issue:
chrome_profile_path : C:\Users\gauth\AppData\Local\Google\Chrome\User Data
You don't have Chrome browser installed on your computer or we di'nt find your profile folder! : 'Options' object has no attribute 'add_experimental_option'
So in the same terminal, I executed python and typed these lines below in order to reproduce the issue:
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension', False)
And I didn't get any issue.
So why do I get the issue when I execute my script and not when I execute line by line in Python terminal?
Here is the full code:
def ChromeDriverWithProfile():
"""
This function will return the chrome driver with the Chrome profile loaded
"""
try:
if platform.system() == 'Linux':
chrome_profile_path = os.environ['HOME'] + "/.config/google-chrome"
chrome_profile_path = os.path.expandvars(chrome_profile_path)
elif platform.system() == 'Darwin':
if not os.environ['HOME'] or os.environ['HOME'] == "":
HOME_DIR = os.path.expanduser("~")
else:
HOME_DIR = os.environ['HOME']
chrome_profile_path = HOME_DIR + "/Library/Application Support/Google/Chrome/Default"
chrome_profile_path = os.path.expandvars(chrome_profile_path)
else:
chrome_profile_path = r"%LocalAppData%\Google\Chrome\User Data"
chrome_profile_path = os.path.expandvars(chrome_profile_path)
print(f"chrome_profile_path : {chrome_profile_path}")
chrome_options = Options()
chrome_options.add_argument(r"--user-data-dir=" + chrome_profile_path)
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-application-cache")
chrome_options.add_argument("--disable-session-crashed-bubble")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--start-maximized")
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"]) # !!!! THE ISSUE COME HERE AT THIS LINE !!!!
chrome_options.add_experimental_option('useAutomationExtension', False)
# overcome limited resource problems
chrome_options.add_argument("--disable-dev-shm-usage")
# Bypass OS security model
chrome_options.add_argument("--no-sandbox")
# We need to remove the bubble popup 'Restore pages' of Chrome:
# https://dev.to/cuongld2/get-rid-of-chrome-restore-bubble-popup-when-automate-gui-test-using-selenium-3pmh
if platform.system() == 'Linux':
preference_file = chrome_profile_path + "/Default/Preferences"
elif platform.system() == 'Darwin':
preference_file = chrome_profile_path
else:
preference_file = chrome_profile_path + "\\Default\\Preferences"
string_to_be_change = '"exit_type":"Crashed"'
new_string = '"exit_type": "none"'
# read input file
fin = open(preference_file, "rt")
# read file contents to string
data = fin.read()
# replace all occurrences of the required string
data = data.replace(string_to_be_change, new_string)
# close the input file
fin.close()
# open the input file in write mode
fin = open(preference_file, "wt")
# overrite the input file with the resulting data
fin.write(data)
# close the file
fin.close()
driver = webdriver.Chrome(ChromeDriverManager().install(), options=chrome_options)
return driver
except Exception as ex:
print(
f"You don't have Chrome browser installed on your computer or we di'nt find your profile folder! : {ex}")
return None
driver = ChromeDriverWithProfile()
I searched for this issue "'Options' object has no attribute 'add_experimental_option'" on the web. There isn't anything. Obviously, I am the only one on earth who has this issue. :-(
Can you help me, please?

How in python I can collect log with level Warning?

I have a code and as understand it should be collect 3 log files with loglevel Warning but its collect25 files with level info. How it possible where I stack?
mail_url = 'https://mail.yandex.ru'
caps = DesiredCapabilities.CHROME
caps['loggingPrefs'] = {'performance': 'WARNING'}
driver = webdriver.Chrome(desired_capabilities=caps)
def get_log():
for entry in driver.get_log('performance'):
date_stamp = str(datetime.datetime.now()).split('.')[0]
date_stamp = date_stamp.replace(" ", "_").replace(":", "_").replace("-", "_")
fileVar = open("log_" + date_stamp + ".txt", "w")
fileVar.write(str(entry))
fileVar.close()
def take_screenshot():
date_stamp = str(datetime.datetime.now()).split('.')[0]
date_stamp = date_stamp.replace(" ", "_").replace(":", "_").replace("-", "_")
driver.save_screenshot(date_stamp + ".png")
driver.set_window_size(360, 740) #open ya.ru in 360x740
driver.get('https://ya.ru')
time.sleep (5)
get_log()
driver.set_window_size(1920, 1080) #open ya.ru in 1920x1080
driver.get('https://ya.ru')
time.sleep (5)
get_log()
target = driver.find_element_by_xpath('//a[#href="' + mail_url + '"]')
builder = ActionChains(driver)
builder.move_to_element(target).perform()
get_log()
driver.set_window_size(800, 600)

The loggingPrefs key should be browser, not performance
caps['loggingPrefs'] = {'browser': 'WARNING'}
driver = webdriver.Chrome(desired_capabilities=caps)
for entry in driver.get_log('browser'):
# ...

web scraping with python, with navigation controller

I am new to python and I need help with web scraping code to save a dynamic map every week.
This is the site I am interested in.
The purpose is to get to the page, select season, select week, and download image to a local folder. I'll use the image to integrate for an automated weekly report using SAS.
thank you in advance!

import sys
import os
import time
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
from selenium import webdriver
import arrow
BASE_URL = 'https://gis.cdc.gov/grasp/fluview/main.html'
DOWNLOAD_PATH = "/Users/"
def closeWebDriver(driver):
if os.name == 'nt':
driver.quit()
else:
driver.close()
def getImage():
profile = FirefoxProfile()
profile.set_preference("browser.download.panel.shown", False)
profile.set_preference("browser.helperApps.neverAsk.openFile","image/png")
profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "image/png")
profile.set_preference("browser.download.folderList", 2);
profile.set_preference("browser.download.dir", DOWNLOAD_PATH)
driver = webdriver.Firefox(firefox_profile=profile)
driver.get(BASE_URL)
time.sleep(5)
if not isValidTimeFrame(driver):
print('Not the time to download yet!')
closeWebDriver(driver)
return
selectFirstWeek(driver)
print('- Consume the web.')
wrapper = driver.find_element_by_class_name('downloads-help-area')
download_img_els = wrapper.find_elements_by_class_name('downloads-button')
for el in download_img_els:
text = el.text.encode('utf-8')
# print(text)
if 'download image' == text.strip().lower():
# Process
downloadImage(el)
break
time.sleep(5)
closeWebDriver(driver)
def isValidTimeFrame(driver):
seasons_button = driver.find_element_by_class_name('seasons-button')
time_frame = seasons_button.text.encode('utf-8').strip().lower()
current_year = arrow.now().to('local')
current_year_str = current_year.format('YYYY')
next_year = current_year.shift(years=1)
next_year_str = next_year.format('YY')
print(time_frame)
compare_year = '%s-%s' % (current_year_str, next_year_str)
return time_frame == compare_year
def selectFirstWeek(driver):
prev = driver.find_element_by_id('prevMap')
week = driver.find_element_by_id('weekSlider')
while True:
print(week)
current_number = week.get_property('value')
print('- Week: ' + current_number)
prev.click()
if int(current_number) < 2:
break;
time.sleep(1)
def downloadImage(el):
print('- Click on ' + el.text)
el.click()
getImage()

How to set timeout for PhantomJS?

Here's the code I have for setting it:
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.resourceTimeout"] = ("5000")
driver = webdriver.PhantomJS(desired_capabilities=dcap)
However in my super long script, it doesn't seem to timeout when my internet is slow and a page takes longer than 5 seconds to load.
There is so little documentation on PhantomJS time outs, and even less of it is for Python, so I figure maybe this isn't even the way to do it.
Has anyone successfully set a timeout with PhantomJS in Python?
Thanks!

instead of targeting phantomjs resource timeout, you can set a timeout for Driver like below example of Firefox driver:
browser = webdriver.Firefox()
browser.set_page_load_timeout(30)
you can change 30 to any numeric
Thanks

You could use the default Python sleep method to retry a few times.
import time
time.sleep()

FIREFOX = 'firefox'
PHANTOM = 'phantom'
NO_IMAGES = False
NEED_IMAGES = True
opened_pages_counter = 0
driver = None
details = {}
user_agent_mozilla = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'
print("[x] UNIVERSAL_DRIVER module loaded")
if platform.system() == 'Windows':
PHANTOMJS_PATH = 'd:/bin/phantomjs/bin/phantomjs.exe'
else:
PHANTOMJS_PATH = './phantomjs'
#atexit.register
def cleanup():
print("universal_driver:: on_exit")
closeDriver()
# PHANTOMJS_PATH = 'd:/bin/phantomjs/bin/phantomjs.exe'
# driver_test = buildDriver(FIREFOX, NO_IMAGES )
# timeout need to be in seconds
# driver_test = buildDriver(FIREFOX, NEED_IMAGES, timeout=100, width=100, height=50)
def buildDriver(driverType, needImages, **kwargs):
global driver, details
closeDriver()
timeout = 60
width=800
height = 600
x=0
y=0
for key in kwargs:
print("another keyword arg: %s: %s" % (key, kwargs[key]))
if key=="timeout": timeout = int(float(kwargs[key]))
if key == "width": width = int(float(kwargs[key]))
if key == "height": height = int(float(kwargs[key]))
if key == "x": x = int(float(kwargs[key]))
if key == "y": y = int(float(kwargs[key]))
details['driverType'] = driverType
details['needImages'] = needImages
if driverType == FIREFOX:
if driver == None:
firefox_profile = webdriver.FirefoxProfile()
if needImages == False:
firefox_profile.set_preference('permissions.default.image', 2)
firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
firefox_profile.set_preference("http.response.timeout", timeout)
firefox_profile.set_preference("dom.max_script_run_time", timeout)
driver = webdriver.Firefox(firefox_profile=firefox_profile)
driver.set_window_size(width, height)
return driver
if driverType == PHANTOM:
if driver == None:
dcap = dict(DesiredCapabilities.PHANTOMJS)
dcap["phantomjs.page.settings.userAgent"] = user_agent_mozilla
dcap["phantomjs.page.settings.resourceTimeout"] = timeout*1000 # in mls
# dcap["phantomjs.page.settings.loadImages"] = needImages
# driver = webdriver.PhantomJS(desired_capabilities=dcap)
service_argsA = []
if needImages == False:
service_argsA = ['--load-images=no']
driver = webdriver.PhantomJS(PHANTOMJS_PATH, desired_capabilities=dcap, service_args=service_argsA)
driver.set_window_size(width, height)
return driver
def openPage(url): # need to prevent of opening pages twice
global driver
global opened_pages_counter
if driver == None:
driver = buildDriver(PHANTOM, NO_IMAGES)
if driver.current_url != url:
driver.get(url)
opened_pages_counter = opened_pages_counter + 1
uprint("universal_driver::", details['driverType'], ", needImages:", details['needImages'], " ; page opened:", url)
def closeDriver():
global driver
if driver == None:
return
driver.close()
driver.quit()
driver = None
uprint("universal_driver:: driver closed")
def uprint(*objects, sep=' ', end='\n', file=sys.stdout):
enc = file.encoding
if enc == 'UTF-8':
print(*objects, sep=sep, end=end, file=file)
else:
f = lambda obj: str(obj).encode(enc, errors='backslashreplace').decode(enc)
print(*map(f, objects), sep=sep, end=end, file=file)