We Keep Coding

Linux NoHup fails for Streaming API IG Markets where file is python

This is quite a specific question regarding nohup in linux, which runs a python file.
Back-story, I am trying to save down streaming data (from IG markets broadcast signal). And, as I am trying to run it via a remote-server (so I don't have to keep my own local desktop up 24/7),
somehow, the nohup will not engage when it 'listen's to a broadcast signal.
Below, is the example python code
#!/usr/bin/env python
#-*- coding:utf-8 -*-
"""
IG Markets Stream API sample with Python
"""
user_ = 'xxx'
password_ = 'xxx'
api_key_ = 'xxx' # this is the 1st api key
account_ = 'xxx'
acc_type_ = 'xxx'
fileLoc = 'marketdata_IG_spx_5min.csv'
list_ = ["CHART:IX.D.SPTRD.DAILY.IP:5MINUTE"]
fields_ = ["UTM", "LTV", "TTV", "BID_OPEN", "BID_HIGH", \
"BID_LOW", "BID_CLOSE",]
import time
import sys
import traceback
import logging
import warnings
warnings.filterwarnings('ignore')
from trading_ig import (IGService, IGStreamService)
from trading_ig.lightstreamer import Subscription
cols_ = ['timestamp', 'data']
# A simple function acting as a Subscription listener
def on_prices_update(item_update):
# print("price: %s " % item_update)
print("xxxxxxxx
))
# A simple function acting as a Subscription listener
def on_charts_update(item_update):
# print("price: %s " % item_update)
print(xxxxxx"\
.format(
stock_name=item_update["name"], **item_update["values"]
))
res_ = [xxxxx"\
.format(
stock_name=item_update["name"], **item_update["values"]
).split(' '))]
# display(pd.DataFrame(res_))
try:
data_ = pd.read_csv(fileLoc)[cols_]
data_ = data_.append(pd.DataFrame(res_, columns = cols_))
data_.to_csv(fileLoc)
print('there is data and we are reading it')
# display(data_)
except:
pd.DataFrame(res_, columns = cols_).to_csv(fileLoc)
print('there is no data and we are saving first time')
time.sleep(60) # sleep for 1 min
def main():
logging.basicConfig(level=logging.INFO)
# logging.basicConfig(level=logging.DEBUG)
ig_service = IGService(
user_, password_, api_key_, acc_type_
)
ig_stream_service = IGStreamService(ig_service)
ig_session = ig_stream_service.create_session()
accountId = account_
################ my code to set sleep function to sleep/read at only certain time intervals
s_time = time.time()
############################
# Making a new Subscription in MERGE mode
subscription_prices = Subscription(
mode="MERGE",
# make sure to put L1 in front of the instrument name
items= list_,
fields= fields_
)
# adapter="QUOTE_ADAPTER")
# Adding the "on_price_update" function to Subscription
subscription_prices.addlistener(on_charts_update)
# Registering the Subscription
sub_key_prices = ig_stream_service.ls_client.subscribe(subscription_prices)
print('this is the line here')
input("{0:-^80}\n".format("HIT CR TO UNSUBSCRIBE AND DISCONNECT FROM \
LIGHTSTREAMER"))
# Disconnecting
ig_stream_service.disconnect()
if __name__ == '__main__':
main()
#######
Then, I try to run it on linux using this command : nohup python marketdata.py
where marketdata.py is basically the python code above.
Somehow, the nohup will not engage....... Any experts/guru who might see what I am missing in my code?

Python unittest: mock open specific paths (don't mock others)

The use-case is that I want to mock the opening of two files ~/.myconf and ./.myconf but not the other ones.
I'm testing the setup of a complex object which reads multiple files in its __init__ and so I'd like to mock some data for some of them, not mock at all for some others.
As an example here is how I mock the conditional opening of those two files, but it feels complex and I find it odd that there's no easy way already built-in that I'm missing.
import builtins
import configparser
import unittest
from textwrap import dedent
from pathlib import Path
from unittest.mock import mock_open
OPEN = builtins.open
def get_hierarchical_config():
cwd = Path.cwd()
global_config = configparser.ConfigParser()
local_config = configparser.ConfigParser()
global_config.read(Path("~/.myconf").expanduser().resolve())
local_config.read((cwd / ".myconf").expanduser().resolve())
full_config.read_dict(global_config)
full_config.read_dict(local_config)
return full_config["mysection"]
def get_custom_mock_open(global_conf_str, local_conf_str) -> callable:
def mocked_open():
def conditional_open_func(path, *args, **kwargs):
p = Path(path).expanduser().resolve()
if p.name == ".myconfig":
if p.parent == Path.home():
return mock_open(read_data=global_conf_str)()
return mock_open(read_data=local_conf_str)()
return OPEN(path, *args, **kwargs)
return conditional_open_func
return mocked_open
[...]
class TestConfig(unittest.TestCase):
def test_read_confs(self):
global_conf = dedent(
"""\
[mysection]
no_overwrite=path/to/somewhere
local_overwrite=ERROR:not overwritten
syntax_test_key= no/space= problem2
"""
)
local_conf = dedent(
"""\
[mysection]
local_overwrite=SUCCESS:overwritten
local_new_key=cool value
"""
)
with patch(
"builtins.open",
new_callable=get_custom_mock_open(global_conf, local_conf),
):
conf = dict(get_hierarchical_config()) # reads the config files
target = {
"no_overwrite": "path/to/somewhere",
"local_overwrite": "SUCCESS:overwritten",
"syntax_test_key": "no/space= problem2",
"local_new_key": "cool value",
}
self.assertDictEqual(conf, target)

configparser.NoSectionError: No section: 'database'

Getting "configparser.NoSectionError: No section: 'database'" error while running from unnitest file.
.py file
import psycopg2 as pg2
from configparser import ConfigParser
parser = ConfigParser()
configFilePath = r'dev.ini'
parser.read(configFilePath)
print('1')
conn = pg2.connect(user=parser.get('database', 'user'),
password=parser.get('database', 'password'),
host=parser.get('database', 'host'),
port=parser.get('database', 'port'),
database=parser.get('database', 'database'))
print(conn.get_dsn_parameters(), "\n")
print('2')
def get_data_from_query(query):
"""return query output"""
cur = conn.cursor()
cur.execute(query)
data_from_query = cur.fetchall()
return data_from_query
def economical_bowlers(year,data_table_1="matches",data_table_2="deliveries"):
query = """select c.bowler , sum(c.sum*6/c.count)
from (select a.bowler ,sum(a.total_runs - a.bye_runs - a.legbye_runs ),count(a.ball)
from {2} a inner join {1} b
on a.match_id=b.id
where b.season='{0}'
group by a.bowler)
c group by c.bowler;""".format(year,data_table_1,data_table_2)
data = get_data_from_query(query)
return dict(data)
.py unittest file
import unittest
import os
import sys
sys.path.insert(2, os.path.join(os.getcwd(), '..'))
import economical_bowlers
from Extractor import extractor
import ipl_project_sql
class Economical_bowlers(unittest.TestCase):
def test_economical_bowlers(self):
expected_matches_data_of_all_teams = {'AD Russell': 10, 'P Kumar': 6}
calculated_matches_data_of_all_teams = ipl_project_sql.economical_bowlers('2015', data_table_1='matches',
data_table_2='deliveries')
self.assertEqual(expected_matches_data_of_all_teams,
calculated_matches_data_of_all_teams)
if __name__ == '__main__':
unittest.main()
Both files work perfectly when are independent.The second file just test the first files output.I just want to remove error.

I had one config file for encryption which needed to be in safe folder as that of test folder.

How to mock googleapiclient.discovery.build

I am trying to mock the result of API call made to compute engine to list VMs. But unfortunately couldn't mock an exact function.
I've tried using PATCH and MOCK methods to mock specific calls made, still unsuccessful
code.py file looks likes this
import googleapiclient.discovery
import logging
class Service:
def __init__(self, project, event):
self.project_id = project
self.compute = googleapiclient.discovery.build('compute', 'v1',
cache_discovery=False)
self.event = event
self.zones = self._validate_event()
def _validate_event(self):
if "jsonPayload" not in self.event:
zones = self.compute.zones().list(
project=self.project_id).execute()['items']
else:
zones = self.compute.zones().get(project=self.project_id,
zone=self.event["jsonPayload"]
["resource"]["zone"]).execute()
logging.debug(f"Identified Zones are {zones}")
return [zone["name"] for zone in zones]
My test file looks like this
# in-built
from unittest import TestCase
from unittest.mock import patch
# custom
import code
class TestServiceModule(TestCase):
def setUp(self):
self.project_id = "sample-project-id"
#patch('code.googleapiclient.discovery')
def test__validate_event_with_empty_inputs(self, mock_discovery):
mock_discovery.build.zones.list.execute.return_value = {"items": [
{
"name": "eu-west-1"
}
]}
obj = code.Service(event={}, project=self.project_id)
print(obj.zones)
In the above test case, I Expected to see "eu-west-1" as the value when I print obj.zones

You didn't mock the googleapiclient.discovery.build method correctly. Here is the unit test solution:
E.g.
code.py:
import googleapiclient.discovery
import logging
class Service:
def __init__(self, project, event):
self.project_id = project
self.compute = googleapiclient.discovery.build('compute', 'v1', cache_discovery=False)
self.event = event
self.zones = self._validate_event()
def _validate_event(self):
if "jsonPayload" not in self.event:
zones = self.compute.zones().list(project=self.project_id).execute()['items']
else:
zones = self.compute.zones().get(project=self.project_id,
zone=self.event["jsonPayload"]["resource"]["zone"]).execute()
logging.debug(f"Identified Zones are {zones}")
return [zone["name"] for zone in zones]
test_code.py:
from unittest import TestCase, main
from unittest.mock import patch
import code
class TestService(TestCase):
def setUp(self):
self.project_id = "sample-project-id"
#patch('code.googleapiclient.discovery')
def test__validate_event_with_empty_inputs(self, mock_discovery):
# Arrange
mock_discovery.build.return_value.zones.return_value.list.return_value.execute.return_value = {
"items": [{"name": "eu-west-1"}]}
# Act
obj = code.Service(event={}, project=self.project_id)
# Assert
mock_discovery.build.assert_called_once_with('compute', 'v1', cache_discovery=False)
mock_discovery.build.return_value.zones.assert_called_once()
mock_discovery.build.return_value.zones.return_value.list.assert_called_once_with(project='sample-project-id')
mock_discovery.build.return_value.zones.return_value.list.return_value.execute.assert_called_once()
self.assertEqual(obj.zones, ["eu-west-1"])
if __name__ == '__main__':
main()
unit test result with coverage report:
.
----------------------------------------------------------------------
Ran 1 test in 0.002s
OK
Name Stmts Miss Cover Missing
-----------------------------------------------------------------------
src/stackoverflow/56794377/code.py 14 1 93% 16
src/stackoverflow/56794377/test_code.py 16 0 100%
-----------------------------------------------------------------------
TOTAL 30 1 97%
Versions:
google-api-python-client==1.12.3
Python 3.7.5

How to mock a function, in a function map/dictionary?

I am trying to patch the fun_1 function from the worker_functions dictionary and I seem to be struggling:
cli.py:
import sys
from worker_functions import (
fun_1,
fun_2,
fun_3,
)
FUNCTION_MAP = {
'run_1': fun_1,
'run_2': fun_2,
'run_3': fun_3,
}
def main():
command = sys.argv[1]
tag = sys.argv[2]
action = FUNCTION_MAP[command]
action(tag)
I've tried mocking cli.fun_1 and cli.main.action and cli.action but this is leading to failure.
test_cli.py:
from mock import patch
from cli import main
def make_test_args(tup):
sample_args = ['cli.py']
sample_args.extend(tup)
return sample_args
def test_fun_1_command():
test_args = make_test_args(['run_1', 'fake_tag'])
with patch('sys.argv', test_args),\
patch('cli.fun_1') as mock_action:
main()
mock_action.assert_called_once()
Do I seem to be missing something?

You'll need to patch the references in the FUNCTION_MAP dictionary itself. Use the patch.dict() callable to do so:
from unittest.mock import patch, MagicMock
mock_action = MagicMock()
with patch('sys.argv', test_args),\
patch.dict('cli.FUNCTION_MAP', {'run_1': mock_action}):
# ...
That's because the FUNCTION_MAP dictionary is the location that the function reference is looked up.