We Keep Coding

Python SQS_Extended_Client giving - Message must be shorter than 1024 bytes error

I am using python sqs_extended_client to process data and using S3 bucket for large payload processing but despite using, I am getting the following error
ERROR - An error occurred (InvalidParameterValue) when calling the SendMessage operation: One or more parameters are invalid. Reason: Message must be shorter than 1024 bytes.
Following is my code
# boto resource
resource = boto3.resource("sqs")
queue = resource.Queue(AWS_SQS_QUEUE_URL)
queue.large_payload_support = SQS_BUCKET
queue.message_size_threshold = 65536
#Send message code
def send_message(message_body):
response = queue.send_message(
QueueUrl=AWS_SQS_QUEUE_URL, MessageBody=json.dumps(message_body)
)
return response
message = {'topic_name': 'report', 'report': {'header': {'stamp': {'sec': 166515321, 'nanosec': 313451487}, 'x_id': ''}, 'b_id': 'a1', 'c_id': 1, 'd_id': 2, 'e_id': 'STOPPED', 'f_id': '', 'd_id': False, 'e_id': False, 'g_id': 1, 'id': 0, 'h_id': 0, 'i_id': 1.0, 'j_id': {'pose': {'k_id': 12040, 'm_id': 180}, 'lyf_id': False}, 'pose': {'x': 27, 'y': -12, 'thaw': 3}, 'next_pose': {'x': 27, 'y': -12, 'yaw': 3}, 'current_goal': {'pose': {'node_id': 0, 'heading': 0}, 'lyf_id': False}, 'goals': '', 'goal_id': '', 'next_goal': {'pose': {'node_id': 0, 'heading': 0}, 'lift': False}, 'next_goal_type': '', 'next_goal_id': ''}, 'filename': 'filename.ext'}
status = send_message(json.dumps(message))

How to convert string from variable in table for csv-export?

I am an absolute beginner with Python (the title probably says it already).I usually look for the answers with googling, but here I don't even know what term to look for.... I have a long string in a variable, which I suppose will be easily converted to a table, only I can't figure out how to do it myself
This is my example:
#pip install pycoingecko
from pycoingecko import CoinGeckoAPI
cg = CoinGeckoAPI()
output = cg.get_search_trending()
print(output)
This is the output:
{'coins': [{'item': {'id': 'metavpad', 'coin_id': 21397, 'name': 'MetaVPad', 'symbol': 'METAV', 'market_cap_rank': 511, 'thumb': 'https://assets.coingecko.com/coins/images/21397/thumb/metav.png?1639044315', 'small': 'https://assets.coingecko.com/coins/images/21397/small/metav.png?1639044315', 'large': 'https://assets.coingecko.com/coins/images/21397/large/metav.png?1639044315', 'slug': 'metavpad', 'price_btc': 7.777707600278187e-06, 'score': 0}}, {'item': {'id': 'syscoin', 'coin_id': 119, 'name': 'Syscoin', 'symbol': 'SYS', 'market_cap_rank': 189, 'thumb': 'https://assets.coingecko.com/coins/images/119/thumb/Syscoin.png?1560401261', 'small': 'https://assets.coingecko.com/coins/images/119/small/Syscoin.png?1560401261', 'large': 'https://assets.coingecko.com/coins/images/119/large/Syscoin.png?1560401261', 'slug': 'syscoin', 'price_btc': 1.3905286168359925e-05, 'score': 1}}, {'item': {'id': 'rainbowtoken', 'coin_id': 17828, 'name': 'RainbowToken', 'symbol': 'RAINBOWTOKEN', 'market_cap_rank': 907, 'thumb': 'https://assets.coingecko.com/coins/images/17828/thumb/WsLiOeJ.png?1637337787', 'small': 'https://assets.coingecko.com/coins/images/17828/small/WsLiOeJ.png?1637337787', 'large': 'https://assets.coingecko.com/coins/images/17828/large/WsLiOeJ.png?1637337787', 'slug': 'rainbowtoken', 'price_btc': 5.831112758941096e-13, 'score': 2}}, {'item': {'id': 'railgun', 'coin_id': 16840, 'name': 'Railgun', 'symbol': 'RAIL', 'market_cap_rank': 534, 'thumb': 'https://assets.coingecko.com/coins/images/16840/thumb/railgun.jpeg?1625322775', 'small': 'https://assets.coingecko.com/coins/images/16840/small/railgun.jpeg?1625322775', 'large': 'https://assets.coingecko.com/coins/images/16840/large/railgun.jpeg?1625322775', 'slug': 'railgun', 'price_btc': 3.1094468809624446e-05, 'score': 3}}, {'item': {'id': 'wonderland', 'coin_id': 18126, 'name': 'Wonderland', 'symbol': 'TIME', 'market_cap_rank': 113, 'thumb': 'https://assets.coingecko.com/coins/images/18126/thumb/time.PNG?1630621941', 'small': 'https://assets.coingecko.com/coins/images/18126/small/time.PNG?1630621941', 'large': 'https://assets.coingecko.com/coins/images/18126/large/time.PNG?1630621941', 'slug': 'wonderland', 'price_btc': 0.08713452772286424, 'score': 4}}, {'item': {'id': 'gods-unchained', 'coin_id': 17139, 'name': 'Gods Unchained', 'symbol': 'GODS', 'market_cap_rank': 274, 'thumb': 'https://assets.coingecko.com/coins/images/17139/thumb/10631.png?1635718182', 'small': 'https://assets.coingecko.com/coins/images/17139/small/10631.png?1635718182', 'large': 'https://assets.coingecko.com/coins/images/17139/large/10631.png?1635718182', 'slug': 'gods-unchained', 'price_btc': 0.00014524078849750436, 'score': 5}}, {'item': {'id': 'altura',
'coin_id': 15127, 'name': 'Altura', 'symbol': 'ALU', 'market_cap_rank': 456, 'thumb': 'https://assets.coingecko.com/coins/images/15127/thumb/ALU_logo_200x200.png?1626868890', 'small': 'https://assets.coingecko.com/coins/images/15127/small/ALU_logo_200x200.png?1626868890', 'large': 'https://assets.coingecko.com/coins/images/15127/large/ALU_logo_200x200.png?1626868890', 'slug': 'altura', 'price_btc': 3.302478861283615e-06, 'score': 6}}], 'exchanges': []}
Now how can I convert this to export it to a CSV? Or what keywords do I have to search for?

You can try something like this, to put everything in 'coins' to a dataframe:
import pandas as pd
from pycoingecko import CoinGeckoAPI
cg = CoinGeckoAPI()
output = cg.get_search_trending()
tab = pd.concat([pd.DataFrame(i) for i in output['coins']],axis=1).T
tab.index = range(tab.shape[0])
The data frame looks like this:
coin_id id ... symbol thumb
0 21397 metavpad ... METAV https://assets.coingecko.com/coins/images/2139...
1 119 syscoin ... SYS https://assets.coingecko.com/coins/images/119/...
2 17828 rainbowtoken ... RAINBOWTOKEN https://assets.coingecko.com/coins/images/1782...
3 16840 railgun ... RAIL https://assets.coingecko.com/coins/images/1684...
4 10354 insure ... SURE https://assets.coingecko.com/coins/images/1035...
5 18126 wonderland ... TIME https://assets.coingecko.com/coins/images/1812...
6 17139 gods-unchained ... GODS https://assets.coingecko.com/coins/images/1713...
You can write it to a csv:
tab.to_csv("results.csv")

Dynamic add list in to json-dict as float

Im running a python 3.7 script where im trying to remove quotes.
Im trying to add a dynamic list (x and y coordinates) to a json-dictionary but i keep getting quotes surrounding the list. I need the list to be inserted without these quotes.
Here is my code that im running:
#The dynamic list
polygon = ['0.124912491249125', '0.683368336833683', '0.128112811281128', '0.472147214721472', '-0.096909690969097', '0.444344434443444', '-0.196919691969197', '-0.533353335333533', '-0.64996499649965', '-0.427742774277428', '-0.290529052905291', '-0.266726672667267', '-0.237523752375237', '0.64996499649965']
list_polygon = []
for i,k in zip(polygon[0::2], polygon[1::2]):
data_polygon = ('['+str(i), str(k)+']')
data_polygon = str(data_polygon)
list_polygon.append(data_polygon)
list_polygon = str(list_polygon)
list_polygon = list_polygon.replace("'",'').replace("(",'').replace(")",'').replace("[[",'[').replace("]]",']')
cord_data = {'apiVersion': '1.3',
'context': '<client context>',
'params': {'cameras': [{'active': True, 'id': 1, 'rotation': 0}],
'configurationStatus': 5,
'profiles': [{'camera': 1,
'filters': [{'active': True,
'data': 1,
'type': 'timeShortLivedLimit'},
{'active': True,
'data': 5,
'type': 'distanceSwayingObject'},
{'active': True,
'data': [5, 5],
'type': 'sizePercentage'}],
'name': 'Profile 1',
'triggers': [{'data': [list_polygon],
'type': 'includeArea'}],
'uid': 1}]},
'method': 'setConfiguration'}
And i get the answer:
{'apiVersion': '1.3', 'context': '<client context>', 'params': {'cameras': [{'active': True, 'id': 1, 'rotation': 0}], 'configurationStatus': 5, 'profiles': [{'camera': 1, 'filters': [{'active': True, 'data': 1, 'type': 'timeShortLivedLimit'}, {'active': True, 'data': 5, 'type': 'distanceSwayingObject'}, {'active': True, 'data': [5, 5], 'type': 'sizePercentage'}], 'name': 'Profile 1', 'triggers': [{'data': ['[0.124912491249125, 0.683368336833683], [0.128112811281128, 0.472147214721472], [-0.096909690969097, 0.444344434443444], [-0.196919691969197, -0.533353335333533], [-0.64996499649965, -0.427742774277428], [-0.290529052905291, -0.266726672667267], [-0.237523752375237, 0.64996499649965]'], 'type': 'includeArea'}], 'uid': 1}]}, 'method': 'setConfiguration'}
As you see it adds a quote between the brackets (at the start and the end) 'triggers': [{'data': ['[ 0.124912491249125, 0.683368336833683], [0.128112811281128, 0.472147214721472], [-0.096909690969097, 0.444344434443444], [-0.196919691969197, -0.533353335333533], [-0.64996499649965, -0.427742774277428], [-0.290529052905291, -0.266726672667267], [-0.237523752375237, 0.64996499649965 ]']
Instead i want it to look like this:
'triggers': [{'data': [[ 0.124912491249125, 0.683368336833683], [0.128112811281128, 0.472147214721472], [-0.096909690969097, 0.444344434443444], [-0.196919691969197, -0.533353335333533], [-0.64996499649965, -0.427742774277428], [-0.290529052905291, -0.266726672667267], [-0.237523752375237, 0.64996499649965 ]]
The first and second must be paired.
The post i want to send should look like this:
cord_data = {'apiVersion': '1.3',
'context': '<client context>',
'params': {'cameras': [{'active': True, 'id': 1, 'rotation': 0}],
'configurationStatus': 5,
'profiles': [{'camera': 1,
'filters': [{'active': True,
'data': 1,
'type': 'timeShortLivedLimit'},
{'active': True,
'data': 5,
'type': 'distanceSwayingObject'},
{'active': True,
'data': [5, 5],
'type': 'sizePercentage'}],
'name': 'Profile 1',
'triggers': [{'data': [[0.124912491249125, 0.683368336833683],
[0.128112811281128, 0.472147214721472],
[-0.096909690969097, 0.444344434443444]
and so on],
'type': 'includeArea'}],
'uid': 1}]},
'method': 'setConfiguration'}
What im i doing wrong?

You make every effort to make it str first, when you need list of lists of floats.
polygon = ['0.124912491249125', '0.683368336833683', '0.128112811281128', '0.472147214721472', '-0.096909690969097', '0.444344434443444', '-0.196919691969197', '-0.533353335333533', '-0.64996499649965', '-0.427742774277428', '-0.290529052905291', '-0.266726672667267', '-0.237523752375237', '0.64996499649965']
list_polygon = [[float(i), float(k)] for i,k in zip(polygon[0::2], polygon[1::2])]
cord_data = {'apiVersion': '1.3',
'context': '<client context>',
'params': {'cameras': [{'active': True, 'id': 1, 'rotation': 0}],
'configurationStatus': 5,
'profiles': [{'camera': 1,
'filters': [{'active': True,
'data': 1,
'type': 'timeShortLivedLimit'},
{'active': True,
'data': 5,
'type': 'distanceSwayingObject'},
{'active': True,
'data': [5, 5],
'type': 'sizePercentage'}],
'name': 'Profile 1',
'triggers': [{'data': list_polygon,
'type': 'includeArea'}],
'uid': 1}]},
'method': 'setConfiguration'}
print(cord_data)
And a word of advise - don't forget Floating Point Arithmetic: Issues and Limitations

I guess you'd want something like this:
from pprint import pprint
#The dynamic list
polygon = ['0.124912491249125', '0.683368336833683', '0.128112811281128', '0.472147214721472', '-0.096909690969097', '0.444344434443444', '-0.196919691969197', '-0.533353335333533', '-0.64996499649965', '-0.427742774277428', '-0.290529052905291', '-0.266726672667267', '-0.237523752375237', '0.64996499649965']
list_polygon = []
for i,k in zip(polygon[0::2], polygon[1::2]):
# Don't duplicate convert to `str`, please. Data is already a string.
data_polygon = ([i, k])
# Remove this line, it messes it up if retained
# data_polygon = str(data_polygon)
list_polygon.append(data_polygon)
# Why are we converting all lists to string anyway??
# list_polygon = str(list_polygon)
# list_polygon = list_polygon.replace("'",'').replace("(",'').replace(")",'').replace("[[",'[').replace("]]",']')
# Convert all nested strings to floats
list_polygon = [list(map(float, data_polygon)) for data_polygon in list_polygon]
cord_data = {'apiVersion': '1.3',
'context': '<client context>',
'params': {'cameras': [{'active': True, 'id': 1, 'rotation': 0}],
'configurationStatus': 5,
'profiles': [{'camera': 1,
'filters': [{'active': True,
'data': 1,
'type': 'timeShortLivedLimit'},
{'active': True,
'data': 5,
'type': 'distanceSwayingObject'},
{'active': True,
'data': [5, 5],
'type': 'sizePercentage'}],
'name': 'Profile 1',
# Don't nest it within another list (unless needed)
# 'triggers': [{'data': [list_polygon],
'triggers': [{'data': list_polygon,
'type': 'includeArea'}],
'uid': 1}]},
'method': 'setConfiguration'}
pprint(cord_data)

Json Decoder Error while trying to load using json.load inPython

I am trying to load a text file formatted as JSON using the below code
with open('orderdata/ETHUSDTorder.txt') as json_file:
data = json.load(json_file)
actualdata=data['average']
print(actualdata)
print("read complete")
but it gives me below error
Traceback (most recent call last):
File "C:\Users\pavan.alur\PycharmProjects\pythonProject\main.py", line 30, in <module>
data = json.load(json_file)
File "C:\Users\pavan.alur\AppData\Local\Programs\Python\Python39\lib\json\__init__.py", line 293, in load
return loads(fp.read(),
File "C:\Users\pavan.alur\AppData\Local\Programs\Python\Python39\lib\json\__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "C:\Users\pavan.alur\AppData\Local\Programs\Python\Python39\lib\json\decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "C:\Users\pavan.alur\AppData\Local\Programs\Python\Python39\lib\json\decoder.py", line 353, in raw_decode
obj, end = self.scan_once(s, idx)
json.decoder.JSONDecodeError: Expecting property name enclosed in double quotes: line 2 column 5 (char 6)
Below is my text file
{
'info': {
'symbol': 'ETHUSD',
'orderid': '20556566',
'orderlistid': '-1',
'clientorderid': 'TXgguEidjfF9eLKF',
'transactiontime': '1619882734',
'price': '0.00',
'origqty': '0.00500',
'executedQty': '0.005000',
'commutativeQuoteQty': '143.06',
'status': 'Filled',
'timeInforce': 'GTC',
'type': 'MARKET',
'side': 'buy',
'fills': [{
'price': '2861.34',
'qty': '0.030',
'comission': '0.0001',
'comissionAsset': 'BNB',
'tradeid': '5722834'
}, {
'price': '2761.23',
'qty': '0.019',
'comission': '0.0001',
'comissionAsset': 'BNB',
'tradeid': '5722834'
}]
},
'id': '2044343',
'clientorderid': 'TXgguEidjfF9eLKF',
'timestamp': '1619882734',
'datetime': '2021-06-10T20:21:234',
'lasttradetimestamp': None,
'symbol': 'ETHUSD',
'type': 'MARKET',
'timeInforce': 'GTC',
'postonly': 'false',
'side': 'buy',
'price': '2861.23',
'stopprice': 'None',
'amount': 0.05,
'cost': 143.06,
'average': 2861.23,
'filled': 0.05,
'remaining': 0.0,
'status': 'closed',
'fee': {
'cost': 0.00013,
'currency': 'BNB'
},
'trades': [{
'info': {
'price': '2861.17',
'qty': '0.0330',
'comission': '0.0001',
'commissionAsset': 'BNB',
'tradeid': '2383487'
},
'timestamp': None,
'datetime': None,
'symbol': 'ETHUSD',
'id': None,
'order': None,
'type': None,
'side': None,
'takerOrMaker': None,
'price': 2861.23,
'amount': 0.034,
'cost': 87.534,
'fee': {
'cost': 0.0034,
'currency': 'BNB'
}
}, {
'info': {
'price': '2861.23',
'qty': '0.0019',
'comission': '0.0023',
'comissionAsset': 'BNB',
'tradeid': '453233'
},
'timestamp': None,
'datetime': None,
'symbol': 'ETHUSD',
'id': None,
'order': None,
'type': None,
'side': None,
'takerOrMaker': None,
'price': 2861.39,
'amount': 0.01941,
'cost': 55.53,
'fee': {
'cost': 6.737,
'currency': 'BNB'
}
}]
}
I am trying to get the "average" value and the JSON response is received from binance api
The response is received using ccxt library and I am trying to print the order that got executed and save the response in json format in text file Any help would be appreciated
I am a bit new to the python so please bear with me here
Thanks in advance. Have a great day :)

You need double-quotes for it to be valid JSON.
There are JSON validator and beautification web sites you can try.
jsonlint.com is one of them, for example.

I got it. Thanks, Ben for helping I just had to use dump to format everything and then load works perfectly. Silly me Thanks all for the help and have a great day ahead :)

Why is this weird ID returned when using todoist python api?

This question is regarding the use of todoist python api.
After adding an item (that's what a task is called in the api) I get these weird looking IDs. I say weird because a regular id is just an integer. But these IDs are not usable in anyway, I can't do api.items.get_by_id() with this id. What is going on? How do I get out from this weird state?
'reply email': 'b5b4eb2c-b28f-11e9-bd8d-80e6500af142',
I printed out a few more IDs, and all the integer ones work well with all API calls, the UUID ones throw an exception.
3318771761
3318771783
3318771807
3318771823
3318771843
61c30a10-b2a0-11e9-98d7-80e6500af142
62326586-b2a0-11e9-98d7-80e6500af142
62a3ea9e-b2a0-11e9-98d7-80e6500af142
631222ac-b2a0-11e9-98d7-80e6500af142
63816338-b2a0-11e9-98d7-80e6500af142
63efd14c-b2a0-11e9-98d7-80e6500af142

You have to use api.commit() to do the Sync and have the final id. The id you're trying to use is just a temporary id while the sync is not done.
>>> import todoist; import os; token = os.environ.get('token'); api = todoist.TodoistAPI(token); item=api.items.add('test');
>>> item
Item({'content': 'test',
'id': '3b4d77c0-b891-11e9-a080-2afbabeedbe3',
'project_id': 1490600000})
>>> api.commit()
{'full_sync': False, 'sync_status': {'3b4d793c-b891-11e9-a080-2afbabeaeaef': 'ok'}, 'temp_id_mapping': {'3b4d77c0-b891-11e9-a080-2afbabeaeaef': 3331113774}, 'labels': [], 'project_notes': [], 'filters': [], 'sync_token': 'EEVprctG1E39VDqJfu_cwyhpO6rkOaavyU5r70Eu0nY1ZjsWSjssGr4qLLLikucJAu_Zakld7DniBsEyZ7i820dqcZNcOAbUcbzHFpMpSjzr-GALTA', 'day_orders': {}, 'projects': [], 'collaborators': [], 'day_orders_timestamp': '1564672738.25', 'live_notifications_last_read_id': 2259500000, 'items': [{'legacy_project_id': 1484800000, 'day_order': -1, 'assigned_by_uid': 5050000, 'labels': [], 'sync_id': None, 'section_id': None, 'in_history': 0, 'child_order': 3, 'date_added': '2019-08-06T21:29:18Z', 'id': 3331113774, 'content': 'test2', 'checked': 0, 'user_id': 5050000, 'due': None, 'priority': 1, 'parent_id': None, 'is_deleted': 0, 'responsible_uid': None, 'project_id': 1490600000, 'date_completed': None, 'collapsed': 0}], 'notes': [], 'reminders': [], 'due_exceptions': [], 'live_notifications': [], 'sections': [], 'collaborator_states': []}
>>> item
Item({'assigned_by_uid': 5050000,
'checked': 0,
'child_order': 3,
'collapsed': 0,
'content': 'test',
'date_added': '2019-08-06T21:29:18Z',
'date_completed': None,
'day_order': -1,
'due': None,
'id': 3331100000,
'in_history': 0,
'is_deleted': 0,
'labels': [],
'legacy_project_id': 148400000,
'parent_id': None,
'priority': 1,
'project_id': 1490660000,
'responsible_uid': None,
'section_id': None,
'sync_id': None,
'user_id': 5050000})