We Keep Coding

Run python class only once and save output for later use

So I have this craigslist scraper project I am working on and I am running into a potential problem. I have this file url.py that has a UrlObj class and getters and setters. In my main.py, I am instantiating that object and getting the completed url back to be sent to the Job class in my main.py to do its scraping stuff.
I would like to deploy this in the cloud in the future and have it run on a time interval (i.e. every day, 4 hours, etc), but I have noticed a problem. Every time this program is ran, the UrlObj class will be called, prompting the user to enter the relevant data to construct the URL. Since this will be in the cloud running in the background, no one will be able to input the prompts every time its built and ran.
What I want is for url.py and UrlObj to be called only once, in the beginning to allow the user to input and populate the necessary fields to construct the url. Then, every time the program is built and ran, the url the user made in the beginning should be used, not calling url.py and UrlObj to prompt the user again to type in inputs since it will be running in the cloud and on a time interval.
Is it too naive to think to set conditions around url = UrlObj().url to make sure it runs once. Like an if statement or while loop?
url.py:
class UrlObj:
def __init__(self):
self.location = self.get_location() # Location(i.e. City) being searched
self.postal_code = self.get_postal_code() # Postal code of location being searched
self.query = self.get_query() # Search for the type of items that will be searched
self.max_price = self.get_max_price() # Max price of the items that will be searched
self.radius = self.get_radius() # Radius of the area searched derived from the postal code given previously
self.url = f"https://{self.location}.craigslist.org/search/sss?&max_price={self.max_price}&postal={self.postal_code}&query={self.query}&20card&search_distance={self.radius}"
def get_location(self):
location = input("Please enter the location: ")
return location
def get_postal_code(self):
postal_code = input("Please enter the postal code: ")
return postal_code
def get_query(self):
query = input("Please enter the item: ")
return query
def get_max_price(self):
max_price = input("Please enter the max price: ")
return max_price
def get_radius(self):
radius = input("Please enter the radius: ")
return radius
main.py:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import pandas as pd
from url import *
class Job():
def __init__(self):
self.driver = webdriver.Chrome(r"C:\Program Files\chromedriver") # Path of Chrome web driver
self.delay = 5 # The delay the driver gives when loading the web page
# Load up the web page
# Gets all relevant data on the page
# Goes to next page until we are at the last page
def load_craigslist_url(self, url):
data = []
self.driver.get(url)
while True:
try:
wait = WebDriverWait(self.driver, self.delay)
wait.until(EC.presence_of_element_located((By.ID, "searchform")))
data.append(self.extract_post_titles())
WebDriverWait(self.driver, 2).until(
EC.element_to_be_clickable((By.XPATH, '//*[#id="searchform"]/div[3]/div[3]/span[2]/a[3]'))).click()
except:
break
return data
# # Extracts all relevant information from the web-page and returns them as individual lists
def extract_post_titles(self):
all_posts = self.driver.find_elements_by_class_name("result-row")
dates_list = []
titles_list = []
prices_list = []
distance_list = []
for post in all_posts:
title = post.text.split("$")
if title[0] == '':
title = title[1]
else:
title = title[0]
title = title.split("\n")
price = title[0]
title = title[-1]
title = title.split(" ")
month = title[0]
day = title[1]
title = ' '.join(title[2:])
date = month + " " + day
if not price[:1].isdigit():
price = "0"
int(price)
raw_distance = post.find_element_by_class_name(
'maptag').text
distance = raw_distance[:-2]
titles_list.append(title)
prices_list.append(price)
dates_list.append(date)
distance_list.append(distance)
return titles_list, prices_list, dates_list, distance_list
# # Kills browser
def kill(self):
self.driver.close()
#staticmethod
def organizeResults(results):
titles_list = results[0][0]
prices_list = list(map(int, results[0][1]))
dates_list = results[0][2]
distance_list = list(map(float, results[0][3]))
list_of_attributes = []
for i in range(len(titles_list)):
content = {'Listing': titles_list[i], 'Price': prices_list[i], 'Date posted': dates_list[i],
'Distance from zip': distance_list[i]}
list_of_attributes.append(content)
list_of_attributes.sort(key=lambda x: x['Distance from zip'])
return list_of_attributes
#staticmethod
def to_csv(dictionary):
df = pd.DataFrame(dictionary)
df.to_csv('data.csv', index=False)
if __name__ == '__main__':
# This should be called only once!!!
# Then the 'url' should be used every time main.py is built and ran, and not be constructed again by calling 'UrlObj().url'
url = UrlObj().url
scraper = Job()
results = scraper.load_craigslist_url(url)
scraper.kill()
dictionary_of_listings = scraper.organizeResults(results)
scraper.to_csv(dictionary_of_listings)

Choose file name depend on parameter in command line running pytest

I have a project with structure the same as this:
https://github.com/RomanBaggins/Test_automatisation_final_project.
In conftest.py i get parameters from cmd.
In one of the functions in mane_page.py I need to assign a value(= parameter in cmd) to a variable.
For example:
cmd: pytest -s -v --language=en --status=empty test_main_page.py
import pytest
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
def pytest_addoption(parser):
parser.addoption('--language', action='store', default=None, help="Choose language")
parser.addoption('--status', action='store', default=None, help="Choose status")
#pytest.fixture(scope="function")
def browser(request):
user_language = request.config.getoption("language")
if user_language:
options = Options()
options.add_experimental_option('prefs', {'intl.accept_languages': user_language})
browser = webdriver.Chrome(options=options)
browser.implicitly_wait(5)
yield browser
else:
raise pytest.UsageError("--choose your language again")
browser.quit()
#pytest.fixture
def choose_status(request):
return request.config.getoption("status")
Then in main_page.py I want to do this:
from .base_page import BasePage
from selenium.webdriver.common.by import By
from .locators import MainPageLocators
class MainPage(BasePage):
def go_to_login_page(self):
login_link = self.browser.find_element(By.CSS_SELECTOR, "#login_link")
login_link.click()
**chosen status = status** <-----------
def should_be_login_link(self):
assert self.is_element_present(*MainPageLocators.LOGIN_LINK)
Finally I need this result: chosen_satus = "empty"
How can I do it?

You can send it as a parameter to go_to_login_page() from the test
#pytest.fixture
def choose_status(request):
return request.config.getoption("status")
def test_main_page(choose_status):
MainPage().go_to_login_page(choose_status)
class MainPage(BasePage):
def go_to_login_page(self, status):
# status = empty

Here is the final code, that solve my problem:
conftest.py
#pytest.fixture
def choose_status(request):
return request.config.getoption("status")
def test_main_page(choose_status):
MainPage().go_to_login_page(choose_status)
In main_page.py
class MainPage(BasePage):
def go_to_login_page(self,choose_status):
if choose_status == "clear":
#do smth
In test_main_page:
def test_guest_can_go_to_login_page(browser,choose_status):
link = "someurl"
page = MainPage(browser, link)
page.open()
page.go_to_login_page(choose_status)
page.should_be_login_link()

How to define a global method whose variables can be used in other methods in Python

I'm working on an automation project. I am automating the login page. There is a problem with the login page in the website that I'm working on, i.e, it fails the first time I log in, even if the credentials are correct, and then successfully logs in if I log in the second time. So, to overcome this I have included a try except for method. The code:
driver=webdriver.Chrome("C:\\temp\\chromedriver.exe")
url = 'site-address'
driver.get(url)
driver.maximize_window()
wait = WebDriverWait(driver, 120)
class Demo
def func1()
user=driver.find_element_by_xpath("//")
user.send_keys('Name')
pas = driver.find_element_by_xpath("//")
password.send_keys('pas')
btn = driver .find_element_by_xpath("//")
btn.click()
try:
chk=wait.until(EC.presence_of_element_located((By.ID, 'lop')))
assert check.text == "Some Text"
print('PAGE VERIFIED : ', check.text)
except:
print('LOGIN UN-SUCCESSFUL')
print('Re-Trying...')
user=driver.find_element_by_xpath("//")
user.send_keys('Name')
pas = driver.find_element_by_xpath("//")
pas.send_keys('pas')
btn = driver .find_element_by_xpath("//")
btn.click()
So, as you can see I am repeating the process of entering username, password and click twice, once in the beginning and then the same process in the except block. My question is, how to create a method, call it setup() such that I can initialize the driver, URL, username, password in setup(). And then call the method in func1() and then in the except block. I'm new to Python, not familiar how to return the values.

Python classes have a constructor called __init__ which you can use to pass information to when creating an object of this class. Something like this, even if the separation of logic is not the best. But gives you an idea of the constructor.
class Demo:
def __init__(self, driver, url, username, password):
""" Constructor of our demo class, set local variables"""
self.driver = driver
self.username = username
self.password = password
self.driver.get(url)
self.driver.maximize_window()
self.wait = WebDriverWait(self.driver, 120)
def func1(self, attempt=1):
""" Find elements in the opened page and send username and pasword,
then clikc the button (submit?) """
user = self.driver.find_element_by_xpath("//")
user.send_keys(self.username)
pas = driver.find_element_by_xpath("//")
pas.send_keys(self.password)
btn = driver .find_element_by_xpath("//")
btn.click()
try:
# Wait for element
chk = self.wait.until(EC.presence_of_element_located((By.ID, 'lop')))
assert chk.text == "Some Text"
print('PAGE VERIFIED : ', chk.text)
except:
# General exception ..
if attempt >= 3: # Start with 1.. so 1, 2, 3.
print("FAILED THREE TIMES, GIVING UP")
else:
print('LOGIN UN-SUCCESSFUL')
print('Re-Trying...')
self.func1(attempt + 1)
if __name__ == '__main__':
driver = webdriver.Chrome("C:\\temp\\chromedriver.exe")
url = 'site-address'
TEST = Demo(driver=driver, url=url, username='Name', password='pas')
TEST.func1()

Python Appium implementing Page Object Model

I'm trying to implement 'By' and 'Keys' with appium just like how I do it on selenium.
On selenium i could do this:
Locators
from selenium.webdriver.common.by import By
class LoginPageLocators(object):
HEADING = (By.CSS_SELECTOR, 'h3[class="panel-title"]')
USERNAME = (By.NAME, 'username')
PASSWORD = (By.NAME, 'password')
LOGIN_BTN = (By.CSS_SELECTOR, 'input[value="Login"]')
functions
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from base import Page
from locators.locators import *
class LoginPage(Page):
def __init__(self, context):
Page.__init__(
self,
context)
def goto_login_page(self, url):
self.open(url)
def enter_username(self, username):
uname = self.find_element(*LoginPageLocators.USERNAME)
uname.send_keys(username)
def enter_password(self, password):
pword = self.find_element(*LoginPageLocators.PASSWORD)
pword.send_keys(password)
def click_login(self):
login = self.find_element(*LoginPageLocators.LOGIN_BTN)
login.click()
def verify_dashboard_page(self, page):
self.verify_page(page)
Is there a way to this in appium? there is no module if i do this:
from appium.webdriver.common.by import By
from appium.webdriver.common.keys import Keys

from appium.webdriver.common.mobileby import By
from appium.webdriver.common.mobileby import MobileBy
class FirstPageLocators(object):
LOCATOR_ONE = (MobileBy.ACCESSIBILITY_ID, 'id')
LOCATOR_TWO = (MobileBy.XPATH, 'xpath_value')

I use another way for locators.
I have page_object directory, which contains files for the app pages and general file - base_page. In the Base_page.py I include general action methods for another pages. Example:
from appium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as ex_cond
from src.platform import PLATFORM, IS_IOS
class BasePage:
def __init__(self, driver: webdriver) -> None:
self._driver = driver
def get_element(self, locator: str, timeout=10):
by = get_locator_by_string(locator)
return WebDriverWait(self._driver, timeout).until(
ex_cond.visibility_of_element_located(by), ' : '.join(by))
def get_no_element(self, locator: str, timeout=10):
by = get_locator_by_string(locator)
element = WebDriverWait(self._driver, timeout).until(
ex_cond.invisibility_of_element_located(by), ' : '.join(by))
if element is None:
return 'No element found'
def get_element_text(self, locator: str, timeout=10):
by = get_locator_by_string(locator)
element = WebDriverWait(self._driver, timeout).until(
ex_cond.visibility_of_element_located(by), ' : '.join(by))
if IS_IOS:
return element.get_attribute('label')
else:
return element.text
def get_element_and_click(self, locator: str, timeout=10):
by = get_locator_by_string(locator)
element = WebDriverWait(self._driver, timeout).until(
ex_cond.visibility_of_element_located(by), ' : '.join(by))
return element.click()
To find the right way for locators, I created the custom method in the same file - base_page.py. Example:
def get_locator_by_string(locator_with_type):
exploided_locator = locator_with_type.split(':', 1)
by_type = exploided_locator[0]
locator = exploided_locator[1]
if by_type == 'xpath':
return (MobileBy.XPATH, locator)
elif by_type == 'css':
return (MobileBy.CSS_SELECTOR, locator)
elif by_type == 'id':
return (MobileBy.ID, locator)
elif by_type == 'accessibility_id':
return (MobileBy.ACCESSIBILITY_ID, locator)
elif by_type == 'android_uiautomator':
return (MobileBy.ANDROID_UIAUTOMATOR, locator)
elif by_type == 'ios_uiautomation':
return (MobileBy.IOS_UIAUTOMATION, locator)
elif by_type == 'ios_predicate':
return (MobileBy.IOS_PREDICATE, locator)
elif by_type == 'class':
return (MobileBy.CLASS_NAME, locator)
else:
raise Exception(f'Cannot get type of locator. Locator
{locator_with_type}')
Get_locator_by_string doesn't consist all search methods. There is methods I need.
In the locators file I have two ways - for Android and for iOS. Example:
import allure
from src.config import BUNDLE_APP
from src.ui.base_page import BasePage, locator_for_platform
class AuthorPage(BasePage):
_author_title = locator_for_platform({
'ANDROID': 'id:%s:id/authorName' % BUNDLE_APP,
'IOS': 'accessibility_id:author_name'
})
_subscribe_button = locator_for_platform({
'ANDROID': 'id:%s:id/subscribeBackground' % BUNDLE_APP,
'IOS': 'accessibility_id:author_subscribe_button'
})
#allure.step('Press subscribe button')
def press_subscribe_button(self):
super().get_element_and_click(self._subscribe_button)
#allure.step('Get subscribe button text')
def get_subscribe_button_text(self):
return super().get_element_text(self._subscribe_button_text)
#allure.step('Get author\'s name text')
def get_author_name_text(self):
return super().get_element_text(self._author_title)
To choose correct locators for platform I use methods for platform's checking. Example:
def locator_for_platform(selectors):
return selectors.get(PLATFORM, 'Undefined Selector')
To check current platform this file:
import os
from appium import webdriver
from src import config
def get_env(key, default=None):
return os.environ.get(key=key, default=default)
def get():
if IS_ANDROID:
return webdriver.Remote(
command_executor=config.APPIUM_HOST,
desired_capabilities=config.DESIRED_CAPS_ANDROID_RESET
)
else:
return webdriver.Remote(
command_executor=config.APPIUM_HOST,
desired_capabilities=config.DESIRED_CAPS_ANDROID_RESET
)
PLATFORM = get_env('PLATFORM', 'IOS')
IS_ANDROID = PLATFORM == 'ANDROID'
IS_IOS = PLATFORM == 'IOS'
And the last file for setup current platform for test:
#!/usr/bin/env bash
export PLATFORM=IOS
export PLATFORM=ANDROID
APPIUM_HOST and DESIRED_CAPS_ANDROID_RESET / DESIRED_CAPS_ANDROID_RESET in other file. Example:
APPIUM_HOST = 'http://localhost:4445/wd/hub'
DESIRED_CAPS_IOS = {
'platformName': 'iOS',
'platformVersion': '13.3',
'deviceName': 'iPhone 8',
'automationNam': 'XCUITest',
'app': MY_APP_IOS
}
DESIRED_CAPS_ANDROID_RESET = {
'platformName': 'Android',
'platformVersion': '10',
'automationName': 'uiautomator2',
'deviceName': DEVICES['Pixel Emulator (10.0)'],
'app': MY_APP_ANDROID,
'unicodeKeyboard': 'true',
'resetKeyboard': 'true',
'disableWindowAnimation': 'true',
'autoWebviewTimeout': '2000',
'clearDeviceLogsOnStart': 'true'
}
DEVICES and MY_APP_ANDROID is another dictionaries.
I hope my experience will be helpful for you.

Click on Button does not work properly

I'm trying to make a little testing script which can post something to my testing facebook group. I think that the best way is to use Selenium webdriver if I don't use Graph API.
Login works correctly. Then I get the group (self.driver.get(group_url)). Now, I locate an textarea element and send my text there - this works (according to self.driver.save_screenshot..).
Now, I'm going to locate a submit/post button. I think it is located correctly because I've tested it by copying XPATH by inspect element in Chrome.
So I do click(). Everything seems work good but there is no new post.
# -*- coding: utf-8 -*-
from selenium import webdriver
import mLib
from selenium.webdriver.common.keys import Keys
import time
GROUPS = ['https://www.facebook.com/groups/1467884653516334/']
TEST = 'TESTING TEXT'
class base():
def __init__(self):
self.driver = webdriver.PhantomJS()
self.settings = {}
self.user = None
self.password = None
self.logged = False
self.groups = []
self.set_settings()
self.groups = self.get_groups()
def get_post_button(self):
print self.driver.find_elements_by_xpath("//*[contains(text(), 'Post')]")
def get_groups(self):
# if 'groups' in self.settings.keys():
# groups = self.settings['groups'].split('*')
# return groups if groups is not None else []
return GROUPS
def set_settings(self):
with open('settings.txt') as f:
for line in f:
splitted = line.split('::')
self.settings[splitted[0]]=splitted[1]
def login(self,username,password):
self.driver.get('http://www.facebook.com')
user_form = self.driver.find_element_by_id('email')
user_form.send_keys(username)
pass_form = self.driver.find_element_by_id('pass')
pass_form.send_keys(password)
pass_form.send_keys(Keys.RETURN)
time.sleep(5)
if 'You must log in to continue' in self.driver.page_source:
self.login()
self.logged = True
def send_post(self,text,group):
assert self.logged == True
self.driver.get(group)
mLib.printToFile('source.txt',self.driver.page_source.encode('utf-8'))
inner_wraps = self.driver.find_elements_by_css_selector('div.innerWrap')
for iw in inner_wraps:
try:
text_field = iw.find_element_by_css_selector('textarea')
text_field.send_keys(text)
self.driver.save_screenshot('screen.png')
except:
continue
button = self.driver.find_element_by_xpath('//*[#id="u_0_1w"]/div/div[5]/div/ul/li[2]/button')
print button.get_attribute('outerHTML').encode('utf-8')
webdriver.ActionChains(self.driver).move_to_element(button).click(button).perform()
bs = base()
bs.login('email#gmail.com','password')
bs.send_post(TEST,GROUPS[0])
print 'OK'
bs.driver.quit()
Why is that so? Do you have any advice?