We Keep Coding

Parametrized RuleBasedStateMachine

After watching https://www.youtube.com/watch?v=zi0rHwfiX1Q I tried to port the example from C (implementation) and Erlang (testing) to Python and Hypothesis. Given this implementation (the rem function emulates %'s C behavior):
import math
def rem(x, y):
res = x % y
return int(math.copysign(res,x))
class Queue:
def __init__(self, capacity: int):
self.capacity = capacity + 1
self.data = [None] * self.capacity
self.inp = 0
self.outp = 0
def put(self, n: int):
self.data[self.inp] = n
self.inp = (self.inp + 1) % self.capacity
def get(self):
ans = self.data[self.outp]
self.outp = (self.outp + 1) % self.capacity
return ans
def size(self):
return rem((self.inp - self.outp), self.capacity)
and this test code
import unittest
from hypothesis.stateful import rule, precondition, RuleBasedStateMachine
from hypothesis.strategies import integers
from myqueue import Queue
class QueueMachine(RuleBasedStateMachine):
cap = 1
def __init__(self):
super().__init__()
self.queue = Queue(self.cap)
self.model = []
#rule(value=integers())
#precondition(lambda self: len(self.model) < self.cap)
def put(self, value):
self.queue.put(value)
self.model.append(value)
#rule()
def size(self):
expected = len(self.model)
actual = self.queue.size()
assert actual == expected
#rule()
#precondition(lambda self: self.model)
def get(self):
actual = self.queue.get()
expected = self.model[0]
self.model = self.model[1:]
assert actual == expected
TestQueue = QueueMachine.TestCase
if __name__ == "__main__":
unittest.main()
The actual question is how to use Hypothesis to also parametrize over QueueMachine.cap instead of setting it manually in the test class.

You can set self.queue in an initialize method rather than __init__, using a suitable integers strategy for the capacity.

Why does this callback run indefinitely?

I've recently stumbled upon this very useful code which I got from this link https://github.com/Douglas6/cputemp.
I've modified the code for clarity reasons and so "application" class is imported as an external class. I'm trying to understand why "set_temperature_callback" is called indefinitely in a loop. The interval of the loop is defined by NOTIFY_TIMEOUT. Here the interval is set to 1000 ms. I'm using this code as a peripheral for one of my Unity projects and it's working fine. I really appreciate any help!
import dbus
from advertisement import Advertisement
from service import Service, Characteristic, Descriptor
from gpiozero import CPUTemperature
#testing code
#------------------------------------------#
from application import Application
import inspect as i
import sys
def FindSource(method):
sys.stdout.write(i.getsource(method))
#------------------------------------------#
GATT_CHRC_IFACE = "org.bluez.GattCharacteristic1"
#the interval between sending messages, is counted in milliseconds, was 5000
NOTIFY_TIMEOUT = 1000
class ThermometerAdvertisement(Advertisement):
def __init__(self, index):
Advertisement.__init__(self, index, "peripheral")
self.add_local_name("Raspberry (thermometer)")
self.include_tx_power = True
class ThermometerService(Service):
#the service we're interested and is displayed in unity
THERMOMETER_SVC_UUID = "00000001-710e-4a5b-8d75-3e5b444bc3cf"
def __init__(self, index):
self.farenheit = True
Service.__init__(self, index, self.THERMOMETER_SVC_UUID, True)
self.add_characteristic(TempCharacteristic(self))
self.add_characteristic(UnitCharacteristic(self))
def is_farenheit(self):
return self.farenheit
def set_farenheit(self, farenheit):
self.farenheit = farenheit
class TempCharacteristic(Characteristic):
TEMP_CHARACTERISTIC_UUID = "00000002-710e-4a5b-8d75-3e5b444bc3cf"
def __init__(self, service):
self.notifying = False
Characteristic.__init__(
self, self.TEMP_CHARACTERISTIC_UUID,
["notify", "read"], service)
self.add_descriptor(TempDescriptor(self))
def get_temperature(self):
value = []
unit = "C"
# code here repeats
cpu = CPUTemperature()
temp = cpu.temperature
if self.service.is_farenheit():
temp = (temp * 1.8) + 32
unit = "F"
strtemp = str(round(temp, 1)) + " " + unit
for c in strtemp:
value.append(dbus.Byte(c.encode()))
return value
# the entire method will run on a loop when unity app hits "subscribe"
def set_temperature_callback(self):
if self.notifying:
print("running inside set_temperature_callback")
value = self.get_temperature()
self.PropertiesChanged(GATT_CHRC_IFACE, {"Value": value}, [])
return self.notifying
def StartNotify(self):
print("Start notify first time")
if self.notifying:
print("returning from start notify")
return
self.notifying = True
value = self.get_temperature()
self.PropertiesChanged(GATT_CHRC_IFACE, {"Value": value}, [])
self.add_timeout(NOTIFY_TIMEOUT, self.set_temperature_callback)
def StopNotify(self):
self.notifying = False
def ReadValue(self, options):
value = self.get_temperature()
return value
class TempDescriptor(Descriptor):
TEMP_DESCRIPTOR_UUID = "2901"
TEMP_DESCRIPTOR_VALUE = "CPU Temperature test"
def __init__(self, characteristic):
Descriptor.__init__(
self, self.TEMP_DESCRIPTOR_UUID,
["read"],
characteristic)
def ReadValue(self, options):
value = []
desc = self.TEMP_DESCRIPTOR_VALUE
for c in desc:
value.append(dbus.Byte(c.encode()))
return value
class UnitCharacteristic(Characteristic):
UNIT_CHARACTERISTIC_UUID = "00000003-710e-4a5b-8d75-3e5b444bc3cf"
def __init__(self, service):
Characteristic.__init__(
self, self.UNIT_CHARACTERISTIC_UUID,
["read", "write"], service)
self.add_descriptor(UnitDescriptor(self))
def WriteValue(self, value, options):
val = str(value[0]).upper()
if val == "C":
self.service.set_farenheit(False)
elif val == "F":
self.service.set_farenheit(True)
def ReadValue(self, options):
value = []
if self.service.is_farenheit(): val = "F"
else: val = "C"
value.append(dbus.Byte(val.encode()))
return value
class UnitDescriptor(Descriptor):
UNIT_DESCRIPTOR_UUID = "2901"
UNIT_DESCRIPTOR_VALUE = "Temperature Units (F or C)"
def __init__(self, characteristic):
Descriptor.__init__(
self, self.UNIT_DESCRIPTOR_UUID,
["read"],
characteristic)
def ReadValue(self, options):
value = []
desc = self.UNIT_DESCRIPTOR_VALUE
for c in desc:
value.append(dbus.Byte(c.encode()))
return value
app = Application()
app.add_service(ThermometerService(0))
app.register()
adv = ThermometerAdvertisement(0)
adv.register()

Kivy camera2 api get buffer

I'm looking for a way to get a buffer from camera2 API in Kivy, like with camera_android.
I am using the code from the Color blind project :https://github.com/inclement/colour-blind-camera/blob/master/camera2/camera2.py
THANKS.
camera_android
from jnius import autoclass, PythonJavaClass, java_method
from kivy.clock import Clock
from kivy.graphics.texture import Texture
from kivy.graphics import Fbo, Callback, Rectangle
from kivy.core.camera import CameraBase
import threading
Camera = autoclass('android.hardware.Camera')
SurfaceTexture = autoclass('android.graphics.SurfaceTexture')
GL_TEXTURE_EXTERNAL_OES = autoclass(
'android.opengl.GLES11Ext').GL_TEXTURE_EXTERNAL_OES
ImageFormat = autoclass('android.graphics.ImageFormat')
class PreviewCallback(PythonJavaClass):
"""
Interface used to get back the preview frame of the Android Camera
"""
__javainterfaces__ = ('android.hardware.Camera$PreviewCallback', )
def __init__(self, callback):
super(PreviewCallback, self).__init__()
self._callback = callback
#java_method('([BLandroid/hardware/Camera;)V')
def onPreviewFrame(self, data, camera):
self._callback(data, camera)
class CameraAndroid(CameraBase):
"""
Implementation of CameraBase using Android API
"""
_update_ev = None
def __init__(self, **kwargs):
self._android_camera = None
self._preview_cb = PreviewCallback(self._on_preview_frame)
self._buflock = threading.Lock()
super(CameraAndroid, self).__init__(**kwargs)
def __del__(self):
self._release_camera()
def init_camera(self):
self._release_camera()
self._android_camera = Camera.open(self._index)
params = self._android_camera.getParameters()
width, height = self._resolution
params.setPreviewSize(width, height)
params.setFocusMode('continuous-picture')
self._android_camera.setParameters(params)
# self._android_camera.setDisplayOrientation()
self.fps = 30.
pf = params.getPreviewFormat()
assert(pf == ImageFormat.NV21) # default format is NV21
self._bufsize = int(ImageFormat.getBitsPerPixel(pf) / 8. *
width * height)
self._camera_texture = Texture(width=width, height=height,
target=GL_TEXTURE_EXTERNAL_OES,
colorfmt='rgba')
self._surface_texture = SurfaceTexture(int(self._camera_texture.id))
self._android_camera.setPreviewTexture(self._surface_texture)
self._fbo = Fbo(size=self._resolution)
self._fbo['resolution'] = (float(width), float(height))
self._fbo.shader.fs = '''
#extension GL_OES_EGL_image_external : require
#ifdef GL_ES
precision highp float;
#endif
/* Outputs from the vertex shader */
varying vec4 frag_color;
varying vec2 tex_coord0;
/* uniform texture samplers */
uniform sampler2D texture0;
uniform samplerExternalOES texture1;
uniform vec2 resolution;
void main()
{
vec2 coord = vec2(tex_coord0.y * (
resolution.y / resolution.x), 1. -tex_coord0.x);
gl_FragColor = texture2D(texture1, tex_coord0);
}
'''
with self._fbo:
self._texture_cb = Callback(lambda instr:
self._camera_texture.bind)
Rectangle(size=self._resolution)
def _release_camera(self):
if self._android_camera is None:
return
self.stop()
self._android_camera.release()
self._android_camera = None
# clear texture and it'll be reset in `_update` pointing to new FBO
self._texture = None
del self._fbo, self._surface_texture, self._camera_texture
def _on_preview_frame(self, data, camera):
with self._buflock:
if self._buffer is not None:
# add buffer back for reuse
self._android_camera.addCallbackBuffer(self._buffer)
self._buffer = data
# check if frame grabbing works
# print self._buffer, len(self.frame_data)
def _refresh_fbo(self):
self._texture_cb.ask_update()
self._fbo.draw()
def start(self):
super(CameraAndroid, self).start()
with self._buflock:
self._buffer = None
for k in range(2): # double buffer
buf = b'\x00' * self._bufsize
self._android_camera.addCallbackBuffer(buf)
self._android_camera.setPreviewCallbackWithBuffer(self._preview_cb)
self._android_camera.startPreview()
if self._update_ev is not None:
self._update_ev.cancel()
self._update_ev = Clock.schedule_interval(self._update, 1 / self.fps)
def stop(self):
super(CameraAndroid, self).stop()
if self._update_ev is not None:
self._update_ev.cancel()
self._update_ev = None
self._android_camera.stopPreview()
self._android_camera.setPreviewCallbackWithBuffer(None)
# buffer queue cleared as well, to be recreated on next start
with self._buflock:
self._buffer = None
def _update(self, dt):
self._surface_texture.updateTexImage()
self._refresh_fbo()
if self._texture is None:
self._texture = self._fbo.texture
self.dispatch('on_load')
self._copy_to_gpu()
def _copy_to_gpu(self):
"""
A dummy placeholder (the image is already in GPU) to be consistent
with other providers.
"""
self.dispatch('on_texture')
def grab_frame(self):
"""
Grab current frame (thread-safe, minimal overhead)
"""
with self._buflock:
if self._buffer is None:
return None
buf = self._buffer.tostring()
return buf
def decode_frame(self, buf):
"""
Decode image data from grabbed frame.
This method depends on OpenCV and NumPy - however it is only used for
fetching the current frame as a NumPy array, and not required when
this :class:`CameraAndroid` provider is simply used by a
:class:`~kivy.uix.camera.Camera` widget.
"""
import numpy as np
from cv2 import cvtColor
w, h = self._resolution
arr = np.fromstring(buf, 'uint8').reshape((h + h / 2, w))
arr = cvtColor(arr, 93) # NV21 -> BGR
return arr
def read_frame(self):
"""
Grab and decode frame in one call
"""
return self.decode_frame(self.grab_frame())
#staticmethod
def get_camera_count():
"""
Get the number of available cameras.
"""
return Camera.getNumberOfCameras()
color blind camera
from kivy.event import EventDispatcher
from kivy.graphics.texture import Texture
from kivy.graphics import Fbo, Callback, Rectangle
from kivy.properties import (BooleanProperty, StringProperty, ObjectProperty, OptionProperty, ListProperty)
from kivy.clock import Clock
from jnius import autoclass, cast, PythonJavaClass, java_method, JavaClass, MetaJavaClass, JavaMethod
import logging
from enum import Enum
logger = logging.getLogger(__file__)
logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
logger.addHandler(handler)
CameraManager = autoclass("android.hardware.camera2.CameraManager")
PythonActivity = autoclass("org.kivy.android.PythonActivity")
Context = autoclass("android.content.Context")
context = cast("android.content.Context", PythonActivity.mActivity)
CameraDevice = autoclass("android.hardware.camera2.CameraDevice")
CaptureRequest = autoclass("android.hardware.camera2.CaptureRequest")
CameraCharacteristics = autoclass("android.hardware.camera2.CameraCharacteristics")
ArrayList = autoclass('java.util.ArrayList')
JavaArray = autoclass('java.lang.reflect.Array')
SurfaceTexture = autoclass('android.graphics.SurfaceTexture')
Surface = autoclass('android.view.Surface')
GL_TEXTURE_EXTERNAL_OES = autoclass(
'android.opengl.GLES11Ext').GL_TEXTURE_EXTERNAL_OES
ImageFormat = autoclass('android.graphics.ImageFormat')
Handler = autoclass("android.os.Handler")
Looper = autoclass("android.os.Looper")
MyStateCallback = autoclass("net.inclem.camera2.MyStateCallback")
CameraActions = autoclass("net.inclem.camera2.MyStateCallback$CameraActions")
# MyStateCallback = autoclass("org.kivy.android.MyStateCallback")
MyCaptureSessionCallback = autoclass("net.inclem.camera2.MyCaptureSessionCallback")
CameraCaptureEvents = autoclass("net.inclem.camera2.MyCaptureSessionCallback$CameraCaptureEvents")
_global_handler = Handler(Looper.getMainLooper())
class LensFacing(Enum):
"""Values copied from CameraCharacteristics api doc, as pyjnius
lookup doesn't work on some devices.
"""
LENS_FACING_FRONT = 0
LENS_FACING_BACK = 1
LENS_FACING_EXTERNAL = 2
class ControlAfMode(Enum):
CONTROL_AF_MODE_CONTINUOUS_PICTURE = 4
class ControlAeMode(Enum):
CONTROL_AE_MODE_ON = 1
class Runnable(PythonJavaClass):
__javainterfaces__ = ['java/lang/Runnable']
def __init__(self, func):
super(Runnable, self).__init__()
self.func = func
#java_method('()V')
def run(self):
try:
self.func()
except:
import traceback
traceback.print_exc()
class PyCameraInterface(EventDispatcher):
"""
Provides an API for querying details of the cameras available on Android.
"""
camera_ids = []
cameras = ListProperty()
java_camera_characteristics = {}
java_camera_manager = ObjectProperty()
def __init__(self):
super().__init__()
logger.info("Starting camera interface init")
self.java_camera_manager = cast("android.hardware.camera2.CameraManager",
context.getSystemService(Context.CAMERA_SERVICE))
self.camera_ids = self.java_camera_manager.getCameraIdList()
characteristics_dict = self.java_camera_characteristics
camera_manager = self.java_camera_manager
logger.info("Got basic java objects")
for camera_id in self.camera_ids:
logger.info(f"Getting data for camera {camera_id}")
characteristics_dict[camera_id] = camera_manager.getCameraCharacteristics(camera_id)
logger.info("Got characteristics dict")
self.cameras.append(PyCameraDevice(
camera_id=camera_id,
java_camera_manager=camera_manager,
java_camera_characteristics=characteristics_dict[camera_id],
))
logger.info(f"Finished interpreting camera {camera_id}")
def select_cameras(self, **conditions):
options = self.cameras
outputs = []
for camera in cameras:
for key, value in conditions.items():
if getattr(camera, key) != value:
break
else:
outputs.append(camera)
return outputs
class PyCameraDevice(EventDispatcher):
camera_id = StringProperty()
output_texture = ObjectProperty(None, allownone=True)
preview_active = BooleanProperty(False)
preview_texture = ObjectProperty(None, allownone=True)
preview_resolution = ListProperty()
preview_fbo = ObjectProperty(None, allownone=True)
java_preview_surface_texture = ObjectProperty(None)
java_preview_surface = ObjectProperty(None)
java_capture_request = ObjectProperty(None)
java_surface_list = ObjectProperty(None)
java_capture_session = ObjectProperty(None)
connected = BooleanProperty(False)
supported_resolutions = ListProperty()
# TODO: populate this
facing = OptionProperty("UNKNOWN", options=["UNKNOWN", "FRONT", "BACK", "EXTERNAL"])
java_camera_characteristics = ObjectProperty()
java_camera_manager = ObjectProperty()
java_camera_device = ObjectProperty()
java_stream_configuration_map = ObjectProperty()
_open_callback = ObjectProperty(None, allownone=True)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.register_event_type("on_opened")
self.register_event_type("on_closed")
self.register_event_type("on_disconnected")
self.register_event_type("on_error")
self._java_state_callback_runnable = Runnable(self._java_state_callback)
self._java_state_java_callback = MyStateCallback(self._java_state_callback_runnable)
self._java_capture_session_callback_runnable = Runnable(self._java_capture_session_callback)
self._java_capture_session_java_callback = MyCaptureSessionCallback(
self._java_capture_session_callback_runnable)
self._populate_camera_characteristics()
def on_opened(self, instance):
pass
def on_closed(self, instance):
pass
def on_disconnected(self, instance):
pass
def on_error(self, instance, error):
pass
def close(self):
self.java_camera_device.close()
def _populate_camera_characteristics(self):
logger.info("Populating camera characteristics")
self.java_stream_configuration_map = self.java_camera_characteristics.get(
CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)
logger.info("Got stream configuration map")
self.supported_resolutions = [
(size.getWidth(), size.getHeight()) for size in
self.java_stream_configuration_map.getOutputSizes(SurfaceTexture(0).getClass())]
logger.info("Got supported resolutions")
facing = self.java_camera_characteristics.get(
CameraCharacteristics.LENS_FACING)
logger.info(f"Got facing: {facing}")
if facing == LensFacing.LENS_FACING_BACK.value: # CameraCharacteristics.LENS_FACING_BACK:
self.facing = "BACK"
elif facing == LensFacing.LENS_FACING_FRONT.value: # CameraCharacteristics.LENS_FACING_FRONT:
self.facing = "FRONT"
elif facing == LensFacing.LENS_FACING_EXTERNAL.value: # CameraCharacteristics.LENS_FACING_EXTERNAL:
self.facing = "EXTERNAL"
else:
raise ValueError("Camera id {} LENS_FACING is unknown value {}".format(self.camera_id, facing))
logger.info(f"Finished initing camera {self.camera_id}")
def __str__(self):
return "<PyCameraDevice facing={}>".format(self.facing)
def __repr__(self):
return str(self)
def open(self, callback=None):
self._open_callback = callback
self.java_camera_manager.openCamera(
self.camera_id,
self._java_state_java_callback,
_global_handler
)
def _java_state_callback(self, *args, **kwargs):
action = MyStateCallback.camera_action.toString()
camera_device = MyStateCallback.camera_device
self.java_camera_device = camera_device
logger.info("CALLBACK: camera event {}".format(action))
if action == "OPENED":
self.dispatch("on_opened", self)
self.connected = True
elif action == "DISCONNECTED":
self.dispatch("on_disconnected", self)
self.connected = False
elif action == "CLOSED":
self.dispatch("on_closed", self)
self.connected = False
elif action == "ERROR":
error = MyStateCallback.camera_error
self.dispatch("on_error", self, error)
self.connected = False
elif action == "UNKNOWN":
print("UNKNOWN camera state callback item")
self.connected = False
else:
raise ValueError("Received unknown camera action {}".format(action))
if self._open_callback is not None:
self._open_callback(self, action)
def start_preview(self, resolution):
if self.java_camera_device is None:
raise ValueError("Camera device not yet opened, cannot create preview stream")
if resolution not in self.supported_resolutions:
raise ValueError(
"Tried to open preview with resolution {}, not in supported resolutions {}".format(
resolution, self.supported_resolutions))
if self.preview_active:
raise ValueError("Preview already active, can't start again without stopping first")
logger.info("Creating capture stream with resolution {}".format(resolution))
self.preview_resolution = resolution
self._prepare_preview_fbo(resolution)
self.preview_texture = Texture(
width=resolution[0], height=resolution[1], target=GL_TEXTURE_EXTERNAL_OES, colorfmt="rgba")
logger.info("Texture id is {}".format(self.preview_texture.id))
self.java_preview_surface_texture = SurfaceTexture(int(self.preview_texture.id))
self.java_preview_surface_texture.setDefaultBufferSize(*resolution)
self.java_preview_surface = Surface(self.java_preview_surface_texture)
self.java_capture_request = self.java_camera_device.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW)
self.java_capture_request.addTarget(self.java_preview_surface)
self.java_capture_request.set(
CaptureRequest.CONTROL_AF_MODE, ControlAfMode.CONTROL_AF_MODE_CONTINUOUS_PICTURE.value) # CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE)
self.java_capture_request.set(
CaptureRequest.CONTROL_AE_MODE, ControlAeMode.CONTROL_AE_MODE_ON.value) # CaptureRequest.CONTROL_AE_MODE_ON)
self.java_surface_list = ArrayList()
self.java_surface_list.add(self.java_preview_surface)
self.java_camera_device.createCaptureSession(
self.java_surface_list,
self._java_capture_session_java_callback,
_global_handler,
)
return self.preview_fbo.texture
def _prepare_preview_fbo(self, resolution):
self.preview_fbo = Fbo(size=resolution)
self.preview_fbo['resolution'] = [float(f) for f in resolution]
self.preview_fbo.shader.fs = """
#extension GL_OES_EGL_image_external : require
#ifdef GL_ES
precision highp float;
#endif
/* Outputs from the vertex shader */
varying vec4 frag_color;
varying vec2 tex_coord0;
/* uniform texture samplers */
uniform sampler2D texture0;
uniform samplerExternalOES texture1;
uniform vec2 resolution;
void main()
{
gl_FragColor = texture2D(texture1, tex_coord0);
}
"""
with self.preview_fbo:
Rectangle(size=resolution)
def _java_capture_session_callback(self, *args, **kwargs):
event = MyCaptureSessionCallback.camera_capture_event.toString()
logger.info("CALLBACK: capture event {}".format(event))
self.java_capture_session = MyCaptureSessionCallback.camera_capture_session
if event == "READY":
logger.info("Doing READY actions")
self.java_capture_session.setRepeatingRequest(self.java_capture_request.build(), None, None)
Clock.schedule_interval(self._update_preview, 0.)
def _update_preview(self, dt):
self.java_preview_surface_texture.updateTexImage()
self.preview_fbo.ask_update()
self.preview_fbo.draw()
self.output_texture = self.preview_fbo.texture
The code i want implement from camera_android to color blind
def grab_frame(self):
"""
Grab current frame (thread-safe, minimal overhead)
"""
with self._buflock:
if self._buffer is None:
return None
buf = self._buffer.tostring()
return buf

Attempt to pickle unknown type while creating a deepcopy

I have the following two classes:
class QPolygonModel(QtGui.QPolygon):
_idx = None
_selected = None
def __init__(self, idx, polygon: QtGui.QPolygon = None):
# Call default constructor
if polygon is None:
super().__init__()
# Call copy constructor
else:
super().__init__(polygon)
self._idx = idx
self._selected = False
#property
def idx(self):
return self._idx
#property
def is_selected(self):
return self._selected
#is_selected.setter
def is_selected(self, flag):
self._selected = flag
def get_points(self):
res = []
for i in range(0, self.size()):
res.append(self.point(i))
return res
This is a custom polygon class that inherits from QPolygon. Objects of this class are stored in a list in the "Scene" class:
class ImageAnnotatorState:
points = None
radius = None
image = None
polygons = None
_finished = None
multiselect = None
def __init__(self, image):
super().__init__()
self.points = QtGui.QPolygon()
self.radius = 8
self.image = image
self.polygons = self._init_polygons()
self.is_finished = False
self.multiselect = False
def _init_polygons(self):
result = []
for annotation in self.image.annotations:
polyline = QPolygonModel(annotation.get_id())
for point in annotation.points:
q_point = QPoint(point.x, point.y)
polyline.append(q_point)
result.append(polyline)
return result
#property
def is_finished(self):
return self._finished
#is_finished.setter
def is_finished(self, flag):
self._finished = flag
Now for the purpose of creating an undo function, I need to create a deepcopy of this scene class so I can store the state that was active before a scene change was made.
So in a QDialog form, I try to do the following:
class ImageAnnotator(QDialog):
_state = None
_previous_state = None
def __init__(self, image):
super().__init__()
self._state = ImageAnnotatorState(image)
self._previous_state = copy.deepcopy(self._state)
self.show()
The deepcopy call here fails with the following exception:
SystemError: attempt to pickle unknown type 'QPolygonModel'
What am I doing wrong?
EDIT:
Reproducible example:
from PyQt5 import QtCore, QtGui
from PyQt5.QtWidgets import QDialog
from PyQt5.QtWidgets import QApplication
import copy
import sys
class Test(QtGui.QPolygon):
idx = None
def __init__(self, z = None):
if z is None:
super().__init__()
else:
super().__init__(z)
class State:
test = None
def __init__(self):
self.test = [Test(), Test()]
print(self.test)
class Main(QDialog):
state = None
prev = None
def __init__(self):
super().__init__()
self.state = State()
prev = copy.deepcopy(self.state)
print(prev)
app = QApplication(sys.argv)
Main()

It seems that it is a bug similar to the one that ekhumoro points out in this answer. A workaround is to implement the __deepcopy__ method.
On the other hand if you want to set the default value in the case of QPolygon do not use None but an empty QPolygon.
With the above, I have implemented the following:
import copy
import random
from PyQt5 import QtCore, QtGui
class QPolygonModel(QtGui.QPolygon):
def __init__(self, idx, polygon=QtGui.QPolygon()):
super().__init__(polygon)
self._idx = idx
self._selected = False
#property
def idx(self):
return self._idx
#property
def is_selected(self):
return self._selected
#is_selected.setter
def is_selected(self, flag):
self._selected = flag
def get_points(self):
res = []
for i in range(0, self.size()):
res.append(self.point(i))
return res
# https://stackoverflow.com/a/10622689
def __deepcopy__(self, memo):
o = QPolygonModel(self.idx)
o.__dict__.update(self.__dict__)
ba = QtCore.QByteArray()
stream_w = QtCore.QDataStream(ba, QtCore.QIODevice.WriteOnly)
stream_w << self
stream_r = QtCore.QDataStream(ba, QtCore.QIODevice.ReadOnly)
stream_r >> o
return o
class State:
def __init__(self):
self.polylines = []
for _ in range(4):
poly = QtGui.QPolygon(
[QtCore.QPoint(*random.sample(range(10), 2)) for _ in range(4)]
)
polyline = QPolygonModel(random.randint(0, 10), poly)
self.polylines.append(polyline)
if __name__ == "__main__":
curr = State()
prev = copy.deepcopy(curr)
assert len(curr.polylines) == len(prev.polylines)
for polyline1, polyline2 in zip(curr.polylines, prev.polylines):
assert id(polyline1) != id(polyline2)
assert polyline1.size() == polyline2.size()
assert polyline1.is_selected == polyline2.is_selected
assert polyline1.idx == polyline2.idx
for i, j in zip(range(polyline1.size()), range(polyline2.size())):
assert polyline1.point(i) == polyline2.point(j)

Class inheritance type checking after pickling in Python

Is there a sure-fire way to check that the class of an object is a sub-class of the desired super?
For Example, in a migration script that I'm writing, I have to convert objects of a given type to dictionaries in a given manner to ensure two-way compatability of the data.
This is best summed up like so:
Serializable
User
Status
Issue
Test
Set
Step
Cycle
However, when I'm recursively checking objects after depickling, I receive a Test object that yields the following results:
Testing data object type:
type(data)
{type}< class'__main.Test' >
Testing Class type:
type(Test())
{type}< class'__main.Test' >
Testing object type against class type:
type(Test()) == type(data)
{bool}False
Testing if object isinstance() of Class:
isinstance(data, Test)
{bool}False
Testing if Class isinstance() of Super Class:
isinstance(Test(), Serializable)
{bool}True
Testing isinstance() of Super Class::
isinstance(data, Serializable)
{bool}False
Interestingly, it doesn't appear to have any such problem prior to pickling as it handles the creation of dictionary and integrity hash just fine.
This only crops up with depickled objects in both Pickle and Dill.
For Context, here's the code in it's native environment - the DataCache object that is pickled:
class DataCache(object):
_hash=""
_data = None
#staticmethod
def genHash(data):
dataDict = DataCache.dictify(data)
datahash = json.dumps(dataDict, sort_keys=True)
return hashlib.sha256(datahash).digest()
#staticmethod
def dictify(data):
if isinstance(data,list):
datahash = []
for item in data:
datahash.append(DataCache.dictify(item))
elif isinstance(data,(dict, collections.OrderedDict)):
datahash = collections.OrderedDict()
for key,value in datahash.iteritems():
datahash[key]= DataCache.dictify(value)
elif isinstance(data, Serializable):
datahash = data.toDict()
else:
datahash = data
return datahash
def __init__(self, restoreDict = {}):
if restoreDict:
self.__dict__.update(restoreDict)
def __getinitargs__(self):
return (self.__dict__)
def set(self, data):
self._hash = DataCache.genHash(data)
self._data = data
def verify(self):
dataHash = DataCache.genHash(self._data)
return (self._hash == dataHash)
def get(self):
return self._data
Finally, I know there's arguments for using JSON for readability in storage, I needed Pickle's ability to convert straight to and from Objects without specifying the object type myself. (thanks to the nesting, it's not really feasible)
Am I going mad here or does pickling do something to the class definitions?
EDIT:
Minimal Implementation:
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import requests
from aenum import Enum
import json # _tricks
import base64
import argparse
import os
import sys
import datetime
import dill
import hashlib
import collections
class Serializable(object):
def __init__(self, initDict={}):
if initDict:
self.__dict__.update(initDict)
def __str__(self):
return str(self.sortSelf())
def sortSelf(self):
return collections.OrderedDict(sorted(self.__dict__.items()))
def toDict(self):
return self.__dict__
def fromDict(self, dict):
# Not using __dict__.update(...) to avoid polluting objects with the excess data
varMap = self.__dict__
if dict and varMap:
for key in varMap:
if (key in dict):
varMap[key] = dict[key]
self.__dict__.update(varMap)
return self
return None
class Issue(Serializable):
def __init__(self, initDict={}):
self.id = 0
self.key = ""
self.fields = {}
if initDict:
self.__dict__.update(initDict)
Serializable.__init__(self)
def fieldToDict(self, obj, key, type):
if key in obj:
result = obj[key]
else:
return None
if result is None:
return None
if isinstance(result, type):
return result.toDict()
return result
def fromDict(self, jsonDict):
super(Issue, self).fromDict(jsonDict)
self.fields["issuetype"] = IssueType().fromDict(self.fields["issuetype"])
self.fields["assignee"] = User().fromDict(self.fields["assignee"])
self.fields["creator"] = User().fromDict(self.fields["creator"])
self.fields["reporter"] = User().fromDict(self.fields["reporter"])
return self
def toDict(self):
result = super(Issue, self).toDict()
blankKeys = []
for fieldName, fieldValue in self.fields.iteritems():
if fieldValue is None:
blankKeys.append(fieldName)
if blankKeys:
for key in blankKeys:
self.fields.pop(key, None)
result["fields"]["issuetype"] = self.fieldToDict(result["fields"], "issuetype", IssueType)
result["fields"]["creator"] = self.fieldToDict(result["fields"], "creator", User)
result["fields"]["reporter"] = self.fieldToDict(result["fields"], "reporter", User)
result["fields"]["assignee"] = self.fieldToDict(result["fields"], "assignee", User)
return result
class IssueType(Serializable):
def __init__(self):
self.id = 0
self.name = ""
def toDict(self):
return {"id": str(self.id)}
class Project(Serializable):
def __init__(self):
Serializable.__init__(self)
self.id = 0
self.name = ""
self.key = ""
class Cycle(Serializable):
def __init__(self):
self.id = 0
self.name = ""
self.totalExecutions = 0
self.endDate = ""
self.description = ""
self.totalExecuted = 0
self.started = ""
self.versionName = ""
self.projectKey = ""
self.versionId = 0
self.environment = ""
self.totalCycleExecutions = 0
self.build = ""
self.ended = ""
self.name = ""
self.modifiedBy = ""
self.projectId = 0
self.startDate = ""
self.executionSummaries = {'executionSummary': []}
class Step(Serializable):
def __init__(self):
self.id = ""
self.orderId = 0
self.step = ""
self.data = ""
self.result = ""
self.attachmentsMap = {}
def toDict(self):
dict = {}
dict["step"] = self.step
dict["data"] = self.data
dict["result"] = self.result
dict["attachments"] = []
return dict
class Status(Serializable):
def __init__(self):
self.id = 0
self.name = ""
self.description = ""
self.isFinal = True
self.color = ""
self.isNative = True
self.statusCount = 0
self.statusPercent = 0.0
class User(Serializable):
def __init__(self):
self.displayName = ""
self.name = ""
self.emailAddress = ""
self.key = ""
self.active = False
self.timeZone = ""
class Execution(Serializable):
def __init__(self):
self.id = 0
self.orderId = 0
self.cycleId = -1
self.cycleName = ""
self.issueId = 0
self.issueKey = 0
self.projectKey = ""
self.comment = ""
self.versionId = 0,
self.versionName = "",
self.executedOn = ""
self.creationDate = ""
self.executedByUserName = ""
self.assigneeUserName = ""
self.status = {}
self.executionStatus = ""
def fromDict(self, jsonDict):
super(Execution, self).fromDict(jsonDict)
self.status = Status().fromDict(self.status)
# This is already listed as Execution Status, need to associate and convert!
return self
def toDict(self):
result = super(Execution, self).toDict()
result['status'] = result['status'].toDict()
return result
class ExecutionContainer(Serializable):
def __init__(self):
self.executions = []
def fromDict(self, jsonDict):
super(ExecutionContainer, self).fromDict(jsonDict)
self.executions = []
for executionDict in jsonDict["executions"]:
self.executions.append(Execution().fromDict(executionDict))
return self
class Test(Issue):
def __init__(self, initDict={}):
if initDict:
self.__dict__.update(initDict)
Issue.__init__(self)
def toDict(self):
result = super(Test, self).toDict()
stepField = "CustomField_0001"
if result["fields"][stepField]:
steps = []
for step in result["fields"][stepField]["steps"]:
steps.append(step.toDict())
result["fields"][stepField] = steps
return result
def fromDict(self, jsonDict):
super(Test, self).fromDict(jsonDict)
stepField = "CustomField_0001"
steps = []
if stepField in self.fields:
for step in self.fields[stepField]["steps"]:
steps.append(Step().fromDict(step))
self.fields[stepField] = {"steps": steps}
return self
class Set(Issue):
def __init__(self, initDict={}):
self.__dict__.update(initDict)
Issue.__init__(self)
class DataCache(object):
_hash = ""
_data = None
#staticmethod
def genHash(data):
dataDict = DataCache.dictify(data)
datahash = json.dumps(dataDict, sort_keys=True)
return hashlib.sha256(datahash).digest()
#staticmethod
def dictify(data):
if isinstance(data, list):
datahash = []
for item in data:
datahash.append(DataCache.dictify(item))
elif isinstance(data, (dict, collections.OrderedDict)):
datahash = collections.OrderedDict()
for key, value in datahash.iteritems():
datahash[key] = DataCache.dictify(value)
elif isinstance(data, Serializable):
datahash = data.toDict()
else:
datahash = data
return datahash
def __init__(self, restoreDict={}):
if restoreDict:
self.__dict__.update(restoreDict)
def __getinitargs__(self):
return (self.__dict__)
def set(self, data):
self._hash = DataCache.genHash(data)
self._data = data
def verify(self):
dataHash = DataCache.genHash(self._data)
return (self._hash == dataHash)
def get(self):
return self._data
def saveCache(name, projectKey, object):
filePath = "migration_caches/{projectKey}".format(projectKey=projectKey)
if not os.path.exists(path=filePath):
os.makedirs(filePath)
cache = DataCache()
cache.set(object)
targetFile = open("{path}/{name}".format(name=name, path=filePath), 'wb')
dill.dump(obj=cache, file=targetFile)
targetFile.close()
def loadCache(name, projectKey):
filePath = "migration_caches/{projectKey}/{name}".format(name=name, projectKey=projectKey)
result = False
try:
targetFile = open(filePath, 'rb')
try:
cache = dill.load(targetFile)
if isinstance(cache, DataCache):
if cache.verify():
result = cache.get()
except EOFError:
# except BaseException:
print ("Failed to load cache from file: {filePath}\n".format(filePath=filePath))
except IOError:
("Failed to load cache file at: {filePath}\n".format(filePath=filePath))
targetFile.close()
return result
testIssue = Test().fromDict({"id": 1000,
"key": "TEST",
"fields": {
"issuetype": {
"id": 1,
"name": "TestIssue"
},
"assignee": "Minothor",
"reporter": "Minothor",
"creator": "Minothor",
}
})
saveCache("Test", "TestProj", testIssue)
result = loadCache("Test", "TestProj")
EDIT 2
The script in it's current form, now seems to work correctly with vanilla Pickle, (initially switched to Dill due to a similar issue, which was solved by the switch).
However, if you are here with this issue and require Dill's features, then as Mike noted in the comments - it's possible to change the settings in dill.settings to have Dill behave pickle referenced items only with joblib mode, effectively mirroring pickle's standard pickling behaviour.