Related
I'm using Python wrappers for CDK to create a Glue job. The command attribute requires an object of type IResolvable | JobCommandProperty. I tried to put a JobCommandProperty object here but I'm getting an exception.
I created a JobCommandProperty object. I was looking for a .builder()function somewhere (similar than in the Java API), but couldn't find one.
from aws_cdk import (
aws_glue as glue,
aws_iam as iam,
core
)
class ScheduledGlueJob (core.Stack):
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
policy_statement = iam.PolicyStatement(
actions=['logs:*','s3:*','ec2:*','iam:*','cloudwatch:*','dynamodb:*','glue:*']
)
policy_statement.add_all_resources()
glue_job_role = iam.Role(
self,
'Glue-Job-Role',
assumed_by=iam.ServicePrincipal('glue.amazonaws.com')
).add_to_policy(
policy_statement
)
job = glue.CfnJob(
self,
'glue-test-job',
role=glue_job_role,
allocated_capacity=10,
command=glue.CfnJob.JobCommandProperty(
name='glueetl',
script_location='s3://my-bucket/glue-scripts/job.scala'
))
The error message is this:
$cdk synth
Traceback (most recent call last):
File "app.py", line 30, in <module>
glue_job = ScheduledGlueJob(app, 'Cronned-Glue-Job')
File "/Users/d439087/IdeaProjects/ds/test_cdk/.env/lib/python3.7/site-packages/jsii/_runtime.py", line 66, in __call__
inst = super().__call__(*args, **kwargs)
File "/Users/d439087/IdeaProjects/ds/test_cdk/glue/scheduled_job.py", line 33, in __init__
script_location='s3://my-bucket/glue-scripts/job.scala'
File "/Users/d439087/IdeaProjects/ds/test_cdk/.env/lib/python3.7/site-packages/jsii/_runtime.py", line 66, in __call__
inst = super().__call__(*args, **kwargs)
File "/Users/d439087/IdeaProjects/ds/test_cdk/.env/lib/python3.7/site-packages/aws_cdk/aws_glue/__init__.py", line 2040, in __init__
jsii.create(CfnJob, self, [scope, id, props])
File "/Users/d439087/IdeaProjects/ds/test_cdk/.env/lib/python3.7/site-packages/jsii/_kernel/__init__.py", line 208, in create
overrides=overrides,
File "/Users/d439087/IdeaProjects/ds/test_cdk/.env/lib/python3.7/site-packages/jsii/_kernel/providers/process.py", line 331, in create
return self._process.send(request, CreateResponse)
File "/Users/d439087/IdeaProjects/ds/test_cdk/.env/lib/python3.7/site-packages/jsii/_kernel/providers/process.py", line 316, in send
raise JSIIError(resp.error) from JavaScriptError(resp.stack)
jsii.errors.JSIIError: Expected 'string', got true (boolean)
Maybe someone has a working CDK (python) example to create a CfnJobobject?
Nevermind, the role attribute has to be of type string, I got confused by the JSII error message.
glue_job_role variable's type is no longer Role because you have added .add_to_policy to it. below code should work.
glue_job_role = iam.Role(
self,
'Glue-Job-Role',
assumed_by=iam.ServicePrincipal('glue.amazonaws.com')
)
glue_job_role.add_to_policy(
policy_statement
)
job = glue.CfnJob(
self,
'glue-test-job',
role=glue_job_role.arn,
allocated_capacity=10,
command=glue.CfnJob.JobCommandProperty(
name='glueetl',
script_location='s3://my-bucket/glue-scripts/job.scala'
))
Be aware that a crawler is not the same as a job, nonetheless I think the permissions are similar.
As of 16 August 2020, this is working for a crawler (and none of the previous answers unfortunately)
from aws_cdk import (
aws_iam as iam,
aws_glue as glue,
core
)
class MyDataScienceStack(core.Stack):
def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
super().__init__(scope, id, **kwargs)
statement = iam.PolicyStatement(actions=["s3:GetObject","s3:PutObject"],
resources=["arn:aws:s3:::mybucketname",
"arn:aws:s3:::mybucketname/data_warehouse/units/*"])
write_to_s3_policy = iam.PolicyDocument(statements=[statement])
glue_role = iam.Role(
self, 'GlueCrawlerFormyDataScienceRole',
role_name = 'GlueCrawlerFormyDataScienceRole',
inline_policies=[write_to_s3_policy],
assumed_by=iam.ServicePrincipal('glue.amazonaws.com'),
managed_policies=[iam.ManagedPolicy.from_aws_managed_policy_name('service-role/AWSGlueServiceRole')]
)
glue_crawler = glue.CfnCrawler(
self, 'glue-crawler-id',
description="Glue Crawler for my-data-science-s3",
name='any name',
database_name='units',
schedule={"scheduleExpression": "cron(5 * * * ? *)"},
role=glue_role.role_arn,
targets={"s3Targets": [{"path": "s3://mybucketname/data_warehouse/units"}]}
)
So to preface this: the goal is to split a sqlite database into three shards through code. I have a majority of the background of storing and accessing the databases, but seem to have run into an issue trying to query more than one of those shards in a row to find a specific row. Code more than likely speaks for itself, so please look at it below.
def get_db():
"""Opens a new database connection if there is none yet for the
current application context.
"""
top = _app_ctx_stack.top
if not hasattr(top, 'sqlite_db'):
top.sqlite_db = [ sqlite3.connect(DATABASE_1, detect_types=sqlite3.PARSE_DECLTYPES), sqlite3.connect(DATABASE_2, detect_types=sqlite3.PARSE_DECLTYPES), sqlite3.connect(DATABASE_3, detect_types=sqlite3.PARSE_DECLTYPES) ]
top.sqlite_db[0].row_factory = sqlite3.Row
top.sqlite_db[1].row_factory = sqlite3.Row
top.sqlite_db[2].row_factory = sqlite3.Row
return top.sqlite_db
def query_db(query, shard, args=(), one=False):
"""Queries the database and returns a list of dictionaries."""
db = get_db()
cur = db[shard].execute(query, args)
rv = cur.fetchall()
return (rv[0] if rv else None) if one else rv
def get_user_id(username):
"""Convenience method to look up the id for a username."""
rb = query_db('select user_id from user where username = ?', 1, [username], one=True)
rv = query_db('select user_id from user where username = ?', 2, [username], one=True)
return rv[0] if rv else None
#app.teardown_appcontext
def close_database(exception):
"""Closes the database again at the end of the request."""
top = _app_ctx_stack.top
if hasattr(top, 'sqlite_db'):
top.sqlite_db[0].close()
top.sqlite_db[1].close()
top.sqlite_db[2].close()
I tried to include everything that may be needed. get_user_id is where I'm encountering an issue; it seems that calling into query_db twice in arrow is causing the problem below (it seems that it doesn't matter what they're assigned to either, as it encounters the same problem either way).
TypeError: 'NoneType' object has no attribute '__getitem__'
If i remove either of the lines (whether it queries the right shard or not) it works. Whether there's an actual row in the database with that username shouldn't matter, since it is confirmed to return None when I remove on of the lines in get_user_id. The problem seems to arise from calling query_db twice in a row. This further leads me to believe that there's an issue with calling the db connection. Once again, I've tried to include everything that I believe to be relevant, but I can add anything else if this isn't enough.
EDIT 1:
Adding full traceback as requested.
Traceback (most recent call last):
File "/home/me/.local/bin/flask", line 9, in <module>
load_entry_point('Flask', 'console_scripts', 'flask')()
File "/home/me/Desktop/project/flask/flask/cli.py", line 881, in main
cli.main(args=args, prog_name=name)
File "/home/me/Desktop/project/flask/flask/cli.py", line 557, in main
return super(FlaskGroup, self).main(*args, **kwargs)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 697, in main
rv = self.invoke(ctx)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 1066, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 895, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/me/.local/lib/python2.7/site-packages/click/decorators.py", line 17, in new_func
return f(get_current_context(), *args, **kwargs)
File "/home/me/Desktop/project/flask/flask/cli.py", line 412, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "/home/me/.local/lib/python2.7/site-packages/click/core.py", line 535, in invoke
return callback(*args, **kwargs)
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 149, in dummy_command
dummy_db()
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 143, in dummy_db
rv = get_user_id('dbrewer5')
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 86, in get_user_id
rv = query_db('select user_id from user where username = ?', 2, [username], one=True)
File "/home/me/Desktop/project/flask/examples/minitwit/minitwit/mt_api.py", line 75, in query_db
cur = db[shard].execute(query, args)
TypeError: 'NoneType' object has no attribute '__getitem__'
I want to create a custom ListCtrl "MyListCtrlCrafting" which pulls the data from a different class called "DBInterface" (actually it is not a real database, but a sophisticated python dictionary). There are tutorials on how to do that, I followed "Python in Action". The main points are: inherit from ListCtrl an set the style parameter to wx.LC_REPORT | wx.LC_SINGLE_SEL | wx.LC_VIRTUAL, call SetItemCount() during init() and override some methods.
To test how that works I made a small App which consists only of one (main-)frame, the virtual ListCtrl and a rudimentary version of DBInterface. Everything works out fine in this setting. But when I connect the classes of the real App, I get a Traceback:
Traceback (most recent call last):
File "DesktopCrafter.py", line 198, in <module>
controller = AppCtrl()
File "DesktopCrafter.py", line 186, in __init__
self.frame = GUI.MainWindow(back_end=self.back_end, crafting_controller=self.crafting_controller, parent=None, title='Desktop Crafter')
...
self.view = MyListCtrlCrafting(name='crafting-hand', back_end=self.db, crafting_controller=self.cc, parent=self, id=wx.ID_ANY)
File "d:\workspace-fun\DesktopCrafter\dc\util\DCUI.py", line 107, in __init__
self.bindData()
File "d:\workspace-fun\DesktopCrafter\dc\util\DCUI.py", line 121, in bindData
self.SetItemCount(count)
wx._core.wxAssertionError: C++ assertion "m_count == ListView_GetItemCount(GetHwnd())" failed at ..\..\src\msw\listctrl.cpp(3120) in wxListCtrl::SetItemCount(): m_count should match ListView_GetItemCount
In contrast to the simple App, the virtualListCtrl is now deeply nested. Can this error just be produced by wrong connections inside this nesting or between DBInterface and the ListCtrl? Or do I have to understand how m_count is calculated, to solve this error? If so, how can I read the _core file? I already read about ListCtrl in the core.py file, but it doesn't contain the relevant parts.
My problem with this traceback is that I don't understand why it is raised during SetItemCount(). This method should be something like a definition and since it is dealing with rows of a list, it should accept positive integers, and maybe 0, and maybe also -1 for standard. I plug in 5, so this cannot be the real problem going on here(?)
Any help or hint is much appreciated!
The complete Traceback:
Traceback (most recent call last):
File "DesktopCrafter.py", line 198, in <module>
controller = AppCtrl()
File "DesktopCrafter.py", line 186, in __init__
self.frame = GUI.MainWindow(back_end=self.back_end, crafting_controller=self.crafting_controller, parent=None, title='Desktop Crafter')
File "d:\workspace-fun\DesktopCrafter\dc\util\DCUI.py", line 285, in __init__
self.InitUI()
File "d:\workspace-fun\DesktopCrafter\dc\util\DCUI.py", line 299, in InitUI
self.panel = MainPanel(back_end=self.db, crafting_controller=self.cc)
File "d:\workspace-fun\DesktopCrafter\dc\util\DCUI.py", line 251, in __init__
self.splitter = MySplitter(back_end=back_end, crafting_controller=crafting_controller)
File "d:\workspace-fun\DesktopCrafter\dc\util\DCUI.py", line 229, in __init__
self.l_frame = LPanel(back_end=back_end, crafting_controller=crafting_controller)
File "d:\workspace-fun\DesktopCrafter\dc\util\DCUI.py", line 188, in __init__
self.panel = CraftingPanel(back_end=back_end, crafting_controller=crafting_controller, *args, **kwargs)
File "d:\workspace-fun\DesktopCrafter\dc\util\DCUI.py", line 154, in __init__
self.view = MyListCtrlCrafting(name='crafting-hand', back_end=self.db, crafting_controller=self.cc, parent=self, id=wx.ID_ANY)
File "d:\workspace-fun\DesktopCrafter\dc\util\DCUI.py", line 107, in __init__
self.bindData()
File "d:\workspace-fun\DesktopCrafter\dc\util\DCUI.py", line 121, in bindData
self.SetItemCount(count)
wx._core.wxAssertionError: C++ assertion "m_count == ListView_GetItemCount(GetHwnd())" failed at ..\..\src\msw\listctrl.cpp(3120) in wxListCtrl::SetItemCount(): m_count should match ListView_GetItemCount
The virtual ListCtrl (both print's give me the expected results):
class MyListCtrlCrafting(wx.ListCtrl):
def __init__(self, name, back_end, crafting_controller, *args, **kwargs):
wx.ListCtrl.__init__(self, style=wx.LC_REPORT | wx.LC_SINGLE_SEL | wx.LC_VIRTUAL, *args, **kwargs)
self.name = name
self.db = back_end
self.cc = crafting_controller
#print(self.db.retrieveValue(['player', self.name]))
self.Bind(wx.EVT_LIST_CACHE_HINT, self.DoCacheItems)
self.bindData()
self.InsertColumn(0, "Slot")
self.InsertColumn(1, "Item")
self.InsertColumn(2, "Amount")
print("initialized MyListCtrl")
def bindData(self):
data = self.db.retrieveValue(['player', self.name])
count = len(data)
#print(count)
self.itemDataMap = {i: ('slot'+str(i+1), data[str(i)]['item'], data[str(i)]['amount']) for i in range(count)}
self.SetItemCount(count)
def DoCacheItems(self, e):
self.db.updateCache(e.GetCacheFrom(), e.GetCacheTo())
def OnGetItemText(self, row_no, col_no):
data = self.db.retrieveValue(['player', self.name, str(row_no)])
return data[col_no]
def OnGetItemAttr(self, item): return None
def OnGetItemImage(self, item): return -1
def OnBackEndUpdated(self):
self.bindData()
self.RefreshItems(0, self.GetItemCount())
The DBInterface:
class DBInterface(dict):
def __init__(self, path, *args, **kwargs):
super(DBInterface, self).__init__(*args, **kwargs)
self.path = path
def load(self):
with open(self.path, 'r') as f:
try:
self.update(json.loads(f.read()))
except Exception as e:
print(e); print(self.path)
def save(self):
self.path.ensure()
with open(self.path, 'w') as f:
f.write(json.dumps(self))
def retrieveValue(self, path):
a = self
for i in range(len(path)):
a = a[path[i]]
return a
That assert is basically a check to ensure that the operation (setting the count) was successful, by asking the native control how many items it has (which should have been set by the prior API call. It seems like something that really shouldn't be able to fail, although obviously it is. Perhaps something is resetting or changing the native control somehow? Are there any threads involved?
I don't have a solid answer for you, but as advice I would suggest taking the small sample that works and build it up step by step until it matches the window hierarchy and environment and basic features of that part of the full application where it is broken. Eventually it should start failing the same way and then you'll know what changed that triggered it and can look more closely at that.
I have a command as follows:
class AddChatMessages(Command):
arguments = [
('messages', AmpList([('message', Unicode()), ('type', Integer())]))]
And I have a responder for it in a controller:
def add_chat_messages(self, messages):
for i, m in enumerate(messages):
messages[i] = (m['message'], m['type'])
self.main.add_chat_messages(messages)
return {}
commands.AddChatMessages.responder(add_chat_messages)
I am writing a unit test for it. This is my code:
class AddChatMessagesTest(ProtocolTestMixin, unittest.TestCase):
command = commands.AddChatMessages
data = {'messages': [{'message': 'hi', 'type': 'None'}]}
def assert_callback(self, unused):
pass
Where ProtocolMixin is as follows:
class ProtocolTestMixin(object):
def setUp(self):
self.protocol = client.CommandProtocol()
def assert_callback(self, unused):
raise NotImplementedError("Has to be implemented!")
def test_responder(self):
responder = self.protocol.lookupFunction(
self.command.commandName)
d = responder(self.data)
d.addCallback(self.assert_callback)
return d
It works if AmpList is not involved, but when it is - I get following error:
======================================================================
ERROR: test_responder
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/<username>/Projects/space/env/lib/python2.7/site-packages/twisted/internet/defer.py", line 139, in maybeDeferred
result = f(*args, **kw)
File "/Users/<username>/Projects/space/env/lib/python2.7/site-packages/twisted/internet/utils.py", line 203, in runWithWarningsSuppressed
reraise(exc_info[1], exc_info[2])
File "/Users/<username>/Projects/space/env/lib/python2.7/site-packages/twisted/internet/utils.py", line 199, in runWithWarningsSuppressed
result = f(*a, **kw)
File "/Users/<username>/Projects/space/tests/client_test.py", line 32, in test_responder
d = responder(self.data)
File "/Users/<username>/Projects/space/env/lib/python2.7/site-packages/twisted/protocols/amp.py", line 1016, in doit
kw = command.parseArguments(box, self)
File "/Users/<username>/Projects/space/env/lib/python2.7/site-packages/twisted/protocols/amp.py", line 1717, in parseArguments
return _stringsToObjects(box, cls.arguments, protocol)
File "/Users/<username>/Projects/space/env/lib/python2.7/site-packages/twisted/protocols/amp.py", line 2510, in _stringsToObjects
argparser.fromBox(argname, myStrings, objects, proto)
File "/Users/<username>/Projects/space/env/lib/python2.7/site-packages/twisted/protocols/amp.py", line 1209, in fromBox
objects[nk] = self.fromStringProto(st, proto)
File "/Users/<username>/Projects/space/env/lib/python2.7/site-packages/twisted/protocols/amp.py", line 1465, in fromStringProto
boxes = parseString(inString)
File "/Users/<username>/Projects/space/env/lib/python2.7/site-packages/twisted/protocols/amp.py", line 2485, in parseString
return cls.parse(StringIO(data))
TypeError: must be string or buffer, not list
Which makes sense, but how do I serialize a list in AddChatMessagesTest.data?
The responder expects to be called with a serialized box. It will then deserialize it, dispatch the objects to application code, take the object the application code returns, serialize it, and then return that serialized form.
For a few AMP types. most notably String, the serialized form is the same as the deserialized form, so it's easy to overlook this.
I think that you'll want to pass your data through Command.makeArguments in order to produce an object suitable to pass to a responder.
For example:
>>> from twisted.protocols.amp import Command, Integer
>>> class Foo(Command):
... arguments = [("bar", Integer())]
...
>>> Foo.makeArguments({"bar": 17}, None)
AmpBox({'bar': '17'})
>>>
If you do this with a Command that uses AmpList I think you'll find makeArguments returns an encoded string for the value of that argument and that the responder is happy to accept and parse that kind of string.
I'm learning twisted so I can integrate it with a blackjack pygame I have. When figuring out how to pass the data from one client to server and then to other clients I was trying to see how I could manipulate the strings that normally get printed to the terminal in each clients screen in this example:
When I say manipulate, I mean to change the datatype to list, int, tuple, etc. of the 'data', print information of it (type(data)), and have conditionals for any keywords.
from twisted.internet import stdio, reactor, protocol
from twisted.protocols import basic
import re
## when client receives data from server // the client script checks then blits ##
class DataForwardingProtocol(protocol.Protocol):
def __init__(self):
self.output = None
self.normalizeNewlines = False
def dataReceived(self,data):
if self.normalizeNewlines:
## see if data is == to secret ##
## this never returns True ? ##
if data == 'secret':
print "This line isn't secure"
else:
data = re.sub(r"(\r\n|\n)","\r\n",data)
if self.output:
if data == "secret":
print "This line isn't secure"
else:
## this will return the error message below ##
## so I'm very unsure of what is going on with 'data' ##
self.output.write(type(data))
class StdioProxyProtocol(DataForwardingProtocol):
def connectionMade(self):
inputForwarder = DataForwardingProtocol()
inputForwarder.output = self.transport
inputForwarder.normalizeNewlines = True
stdioWrapper = stdio.StandardIO(inputForwarder)
self.output = stdioWrapper
class StdioProxyFactory(protocol.ClientFactory):
protocol = StdioProxyProtocol
reactor.connectTCP('192.168.1.2', 6000, StdioProxyFactory())
reactor.run()
Returns:
Unhandled Error
Traceback (most recent call last):
File "/usr/lib/python2.7/dist-packages/twisted/python/log.py", line 84, in callWithLogger
return callWithContext({"system": lp}, func, *args, **kw)
File "/usr/lib/python2.7/dist-packages/twisted/python/log.py", line 69, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "/usr/lib/python2.7/dist-packages/twisted/python/context.py", line 118, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/usr/lib/python2.7/dist-packages/twisted/python/context.py", line 81, in callWithContext
return func(*args,**kw)
--- <exception caught here> ---
File "/usr/lib/python2.7/dist-packages/twisted/internet/posixbase.py", line 614, in _doReadOrWrite
why = selectable.doRead()
File "/usr/lib/python2.7/dist-packages/twisted/internet/tcp.py", line 203, in doRead
return self._dataReceived(data)
File "/usr/lib/python2.7/dist-packages/twisted/internet/tcp.py", line 209, in _dataReceived
rval = self.protocol.dataReceived(data)
File "Downloads/trial.py", line 22, in dataReceived
self.output.write(type(data))
File "/usr/lib/python2.7/dist-packages/twisted/internet/_posixstdio.py", line 53, in write
self._writer.write(data)
File "/usr/lib/python2.7/dist-packages/twisted/internet/process.py", line 174, in write
abstract.FileDescriptor.write(self, data)
File "/usr/lib/python2.7/dist-packages/twisted/internet/abstract.py", line 335, in write
self._tempDataLen += len(data)
exceptions.TypeError: object of type 'type' has no len()
So I can never 'check' the 'data' and when I try to print out anything about the data or change its type I get an extensive error? Is there something obvious I'm missing or is this the wrong way to even go about it? If it helps here is the server script
When you call type() it returns the type of the object, and not a string representing the type of the object.
You could check the type like this instead:
>>> aString = 'abc'
>>> anInt = 123
>>> type(aString) is str
True
>>> type(aString) is int
False
>>> type(anInt) is str
False
>>> type(anInt) is int
True
The data you receive will not be a list, tuple or some other kind of object unless you serialize it. Check out the module Pickle to see examples of how you can serialize objects!