We Keep Coding

Optimize byte array escaping performance python

I need to perform custom escaping over a byte array in python. However, during escaping python converts bytes to integers, making performance optimization very difficult. How can I speed up my escaping function?
ESCAPE_DICT={
0x00: [0x5C,0x7A], # null -> \z 0x5c 0x7a
0x22: [0x5C,0x71], # " -> \q 0x5c 0x71
0x3B: [0x5C,0x73], # ; -> \s 0x5c 0x73
0x5C: [0x5C,0x5C], # \ -> \\ 0x5c 0x5c
0x0A: [0x5C,0x6E], # line-feed -> \n 0x5c 0x6e
0x0C: [0x5C,0x66], # form-feed -> \f 0x5c 0x66
0x0D: [0x5C,0x63], # carr-return -> \c 0x5c 0x63
}
def escape(string: bytes):
str_len=string.__len__()
escaped_list=[]
for i in range(0,str_len):
curr_byte=string[i]
escape = ESCAPE_DICT.get(curr_byte)
if escape is None:
# Don't escape current byte
escaped_list.append(curr_byte)
else:
# Escape current byte
escaped_list.extend(escape)
return bytes(escaped_array)

import re
ESCAPE_DICT = {
b'\x00': rb'\z', # null
b'"': rb'\q',
b';': rb'\s',
b'\\': rb'\\',
b'\n': rb'\n', # linefeed
b'\f': rb'\f', # formfeed
b'\r': rb'\c', # carriage return
}
ESCAPE_CLASS = '[' + ''.join(r'\x' + e.hex() for e in ESCAPE_DICT) + ']'
ESCAPE_REGEX = re.compile(ESCAPE_CLASS.encode())
def escape(string: bytes) -> bytes:
return re.sub(ESCAPE_REGEX, lambda m: ESCAPE_DICT[m.group(0)], string)
x = b'"abc\ndef\rpqr\x00stu\\xyz"'
y = escape(x)
from pprint import pprint
pprint(ESCAPE_CLASS)
pprint(ESCAPE_REGEX)
pprint(x)
pprint(y)
# =>
# '[\\x00\\x22\\x3b\\x5c\\x0a\\x0c\\x0d]'
# re.compile(b'[\\x00\\x22\\x3b\\x5c\\x0a\\x0c\\x0d]')
# b'"abc\ndef\rpqr\x00stu\\xyz"'
# b'\\qabc\\ndef\\cpqr\\zstu\\\\xyz\\q'
You can read the rb prefix as “raw bytes”.
Your escapes are a bit strange, though. E.g., the carriage return is normally \r, not \c, and \s normally stands for generic whitespace.

TMF8701 ToF Distance Sensor from AMS with Python

I am trying to send a command to the command register from my TMF8701 sensor from AMS with Python code from a Raspberry Pi. I have to of these sensors connected to the I2C Bus of the Raspberry Pi. One for channel 1 and one for channel 0. I am writing the 0x10 register with 0x0A and trying to do the factory calibration. But after this command is written, the 0x1E register does not have the correct value. I also tried to write another command, for example 0x0B, but also the 0x1E register does not return the correct value. Can anyone help me?
import smbus
from time import sleep
CMD_DATA0=0x0F
CMD_DATA1=0x0E
CMD_DATA2=0x0D
CMD_DATA3=0x0C
CMD_DATA4=0x0B
CMD_DATA5=0x0A
CMD_DATA6=0x09
CMD_DATA7=0x08
CMD_DATA8=0x07
CMD_DATA9=0x06
#14 Byte
FACTORY_CALIBRATION=0x28
class tmf8701_distanceSensor():
# by default, this assumes that the device is at 0x41
def __init__(self, channel, address=0x41):
self.address = address
self.channel = channel
self.bus = smbus.SMBus(self.channel)
#self.reset()
sleep(1) # wait 1 sec
self.setup()
#def reset(self):
#0x10 - command
#factoryCalibrationValue=""
def factoryCalibration(self):
sleep(2)
self.bus.write_i2c_block_data(self.address, 0x10, [0x0A])
#while (self.bus.read_byte_data(self.address, 0x1E))!=0x0A:
#sleep(1)
#print("while")
sleep(10)
#print("factory calibration")
#print(str(int(self.bus.read_byte_data(self.address, 0x1E))), str(int(0x0A)))
#Factory Calibration begins 0x20 und dann 14 byte
block=self.bus.read_i2c_block_data(self.address, 0x20, 14)
print(block)
print(self.bus.read_byte_data(self.address, 0x2c))
print(self.bus.read_byte_data(self.address, 0x2d))
print(self.bus.read_byte_data(self.address, 0x01))
"""def bootloaderReadyForNewCommand(self):
#CMD_STAT register - 0x0 - READY
#0x10-0xFF - busy - davor error und gut !
read0x10Register=self.bus.read_byte_data(self.address, 0x10)
if read0x10Register == 0:
return("ready")
elif read0x10Register < int(0x10):
return read0x10Register
else:
print("try again later??? ")
return "busy"""""
def readResults(self):
print("erst 0x20")
print(self.bus.read_byte_data(self.address, 0x20))
print("dann 0x1D")
print(self.bus.read_byte_data(self.address, 0x1D))
print("dann 0x20")
print(self.bus.read_byte_data(self.address, 0x20))
print("zuletzt 0x1E")
print(self.bus.read_byte_data(self.address, 0x1E))
def setDeviceInStandBy(self):
self.bus.write_i2c_block_data(self.address, 0xE0, [0x00])
def setup(self):
###bootloader settings:
if not self.bus.read_byte_data(self.address, 0x00)==0x80:
self.bus.write_i2c_block_data(self.address, 0x00, [0x80])
#wake up sensor
self.bus.write_i2c_block_data(self.address, 0xE0, [0x01])
sleep(3)
###
read0xE0Register=self.bus.read_byte_data(self.address, 0xE0)
print("register 0xE0, if 65 => perfect!")
print(read0xE0Register)
### app0 settings:
if not self.bus.read_byte_data(self.address, 0x00)==0xC0:
self.bus.write_i2c_block_data(self.address, 0x00, [0xC0])
self.bus.write_i2c_block_data(self.address, 0x10, [0x0B])
#read serial number
self.bus.write_i2c_block_data(self.address, 0x10, [0x47])
#while (self.bus.read_byte_data(self.address, 0x1E))!=0x0A:
#sleep(1)
#print("while")
sleep(10)
print("serialdata")
print(self.bus.read_byte_data(self.address, 0x1E))
block=self.bus.read_i2c_block_data(self.address, 0x28, 3)
print(block)
#print("factory calibration")
#print(str(int(self.bus.read_byte_data(self.address, 0x1E))), str(int(0x0A)))
#Factory Calibration begins 0x20 und dann 14 byte
#block=self.bus.read_i2c_block_data(self.address, 0x1E, )
#print(block)
self.factoryCalibration()
print("currently running SW register: 128 = 0x80 - bootloader, 192 = 0xC0 - App0 ")
print(self.bus.read_byte_data(self.address, 0x00))
###algorithm state and factory calibration is provided
self.bus.write_i2c_block_data(self.address, CMD_DATA7, [0x03])
###
self.bus.write_i2c_block_data(self.address, CMD_DATA6, [0x23])
### no GPIO control used
self.bus.write_i2c_block_data(self.address, CMD_DATA5, [0x00])
self.bus.write_i2c_block_data(self.address, CMD_DATA4, [0x00])
### needs to be always 00
self.bus.write_i2c_block_data(self.address, CMD_DATA3, [0x00])
### repetition period in ms 64 hex = 100ms
self.bus.write_i2c_block_data(self.address, CMD_DATA2, [0x64])
### needs to be always ff
self.bus.write_i2c_block_data(self.address, CMD_DATA1, [0xFF])
self.bus.write_i2c_block_data(self.address, CMD_DATA0, [0xFF])
import csv,os
import numpy as np
import threading
#import max30102
import TMF8701_distanceSensor
import datetime
import time
from numpy import number
print("start")
print("sensor finger")
sensorTMF_finger=TMF8701_distanceSensor.tmf8701_distanceSensor(0)
#sensorTMF_finger.factoryCalibration()
print("sensor hand")
sensorTMF_hand=TMF8701_distanceSensor.tmf8701_distanceSensor(1)
#sensorTMF_hand.factoryCalibration()
number=0
while number<10:
print("hand")
#sensorTMF_hand.readResults()
print("finger")
sensorTMF_finger.readResults()
number=number+1
The output I get on the Raspberry Pi looks like that:
start
sensor finger
register 0xE0, if 65 => perfect!
65
serialdata
0
[0, 0, 0]
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
0
0
16
currently running SW register: 128 = 0x80 - bootloader, 192 = 0xC0 - App0
192
sensor hand
register 0xE0, if 65 => perfect!
65
serialdata
0
[0, 0, 0]
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
0
0
16
currently running SW register: 128 = 0x80 - bootloader, 192 = 0xC0 - App0
192
hand
finger
erst 0x20
0
dann 0x1D
0
dann 0x20
0
zuletzt 0x1E
0
hand
finger
erst 0x20
0
dann 0x1D
0
dann 0x20
0
zuletzt 0x1E
0
hand
finger
erst 0x20
0
dann 0x1D
0
dann 0x20
0
zuletzt 0x1E
0
hand
finger
erst 0x20
0
dann 0x1D
0
dann 0x20
0
zuletzt 0x1E
0
hand
finger
erst 0x20
0
dann 0x1D
0
dann 0x20
0
zuletzt 0x1E
0
hand
finger
erst 0x20
0
dann 0x1D
0
dann 0x20
0
zuletzt 0x1E
0
hand
finger
erst 0x20
0
dann 0x1D
0
dann 0x20
0
zuletzt 0x1E
0
hand
finger
erst 0x20
0
dann 0x1D
0
dann 0x20
0
zuletzt 0x1E
0
hand
finger
erst 0x20
0
dann 0x1D
0
dann 0x20
0
zuletzt 0x1E
0
hand
finger
erst 0x20
0
dann 0x1D
0
dann 0x20
0
zuletzt 0x1E
0

How to create EddyStone Beacon using BlueZ and Python?

I'm working on BLE(BlueTooth Low Energy) on an Embedded Linux board. We use BlueZ and Python. I need to create EddyStone Beacon. I found there is a way to create iBeacon: https://scribles.net/creating-ibeacon-using-bluez-example-code-on-raspberry-pi/. I tried it. It worked.
But we need to create EddyStone Beacon. So I use the Beacon data format from here(https://ukbaz.github.io/howto/beacon_scan_cmd_line.html) to create the manufacturer data.
But my code doesn't work. What is wrong with my code? Here is my code:
def __init__(self, bus, index):
eddystone_id = 0xAAFE
beacon_type = [0x14, 0x16] # Length = 0x14, EddyStone type = 0x16
uuid = [0xAA, 0xFE] # EddyStone UUID = 0xAAFE
frame_type = [0x10] # Frame Type = 0x10
power = [0x00] # Power = 0x00
prefix = [0x02] # URL scheme = 0x02 (http://)
url = [0x73, 0x61, 0x6D, 0x70, 0x6C, 0x65, 0x77, 0x65, 0x62, 0x73, 0x69, 0x74, 0x65, 0x07]
Advertisement.__init__(self, bus, index, 'peripheral')
self.add_manufacturer_data(eddystone_id, beacon_type + uuid + frame_type + power + prefix + url)
However, if I use this command, the EddyStone Beacon is created. I can see it shows EddyStone Beacon in nRF mobile app:
sudo hcitool -i hci0 cmd 0x08 0x0008 1c 02 01 06 03 03 aa fe 14 16 aa fe 10 00 02 73 61 6d 70 6c 65 77 65 62 73 69 74 65 07 00 00 00
As you can see, the data I put in the add_manufacturer_data() function is the same as the data in the command. But why the Python code doesn't work?

iBeacon uses manufacturer_data while Eddystone beacons use service_data so I would expect your code to look more like this:
def __init__(self, bus, index):
Advertisement.__init__(self, bus, index, 'broadcast')
self.add_service_uuid('FEAA')
frame_type = [0x10] # Eddystone frame Type = 0x10
power = [0x00] # Beacon broadcast power = 0x00
prefix = [0x02] # URL scheme = 0x02 (http://)
url = [0x73, 0x61, 0x6D, 0x70, 0x6C, 0x65, 0x77, 0x65, 0x62, 0x73, 0x69, 0x74, 0x65, 0x07]
eddystone_data = frame_type + power + prefix + url
self.add_service_data('FEAA', eddystone_data)
As a side note, hcitool is one of the tools that has been deprecated by the BlueZ developers. The currently supported way to create the Eddystone beacon from the command line would be with bluetoothctl. The sequence of commands would be:
bluetoothctl
menu advertise
uuids 0xFEAA
service 0xFEAA 0x10 0x00 0x02 0x73 0x61 0x6D 0x70 0x6C 0x65 0x77 0x65 0x62 0x73 0x69 0x74 0x65 0x07
back
advertise broadcast
discoverable on
I changed the advertisement type from peripheral to broadcast because typically people don't want beacons to be connectable, but it depends on your application.

OEM non printable characters in Python strings

I´m trying to port some Delphi code that sends data to a Universe database. In order to make the text legible by the DB we need to encode it in OEM.
In Delphi is done this way:
procedure TForm1.GenerarTablasNLS;
var
i: integer;
begin
for i := 0 to 255 do
begin
TablaUV_NLS[i] := AnsiChar(i);
TablaNLS_UV[i] := AnsiChar(i);
end;
// Nulo final
TablaUV_NLS[256] := #0;
TablaNLS_UV[256] := #0;
OemToCharA(#TablaUV_NLS[1], #TablaUV_NLS[1]);
CharToOemA(#TablaNLS_UV[1], #TablaNLS_UV[1]);
And then we translate our text simply like this
function StringToUniverse(const Value: string): AnsiString;
var
p: PChar;
q: PAnsiChar;
begin
SetLength(Result, Length(Value));
if Value = '' then Exit;
p := Pointer(Value);
q := Pointer(Result);
while p^ <> #0 do
begin
q^ := TablaNLS_UV[Ord(AnsiChar(p^))];
Inc(p);
Inc(q);
end;
end;
I follow the same logic in Python using a dictionary that stores each character translation
class StringUniverseDict(dict):
def __missing__(self, key):
return key
TablaString2UV = StringUniverseDict()
def rellenar_tablas_codificacion():
TablaString2UV['á'] = ' ' # chr(225) = chr(160)
TablaString2UV['é'] = '‚' # chr(233) = chr(130)
TablaString2UV['í'] = '¡' # chr(237) = chr(161)
TablaString2UV['ó'] = '¢' # chr(243) = chr(162)
TablaString2UV['ú'] = '£' # chr(250) = chr(163)
TablaString2UV['ñ'] = '¤' # chr(241) = chr(164)
TablaString2UV['ç'] = '‡' # chr(231) = chr(135)
TablaString2UV['Á'] = 'µ' # chr(193) = chr(181)
TablaString2UV['É'] = chr(144) # chr(201) = chr(144)
TablaString2UV['Í'] = 'Ö' # chr(205) = chr(214)
TablaString2UV['Ó'] = 'à' # chr(211) = chr(224)
TablaString2UV['Ñ'] = '¥' # chr(209) = chr(165)
TablaString2UV['Ç'] = '€' # chr(199) = chr(128)
TablaString2UV['ü'] = chr(129) # chr(252) = chr(129)
TablaString2UV[chr(129)] = '_' # chr(129) = chr(095)
TablaString2UV[chr(141)] = '_' # chr(141) = chr(095)
TablaString2UV['•'] = chr(007) # chr(149) = chr(007)
TablaString2UV['Å'] = chr(143) # chr(197) = chr(143)
TablaString2UV['Ø'] = chr(157) # chr(216) = chr(157)
TablaString2UV['ì'] = chr(141) # chr(236) = chr(141)
This works "fine" as long as I translate using printable characters. For example, the string
"á é í ó ú ñ ç Á Í Ó Ú Ñ Ç"
is translated, in Delphi, to the following bytes:
0xa0 0x20 0x82 0x20 0xa1 0x20 0xa2 0x20 0xa3 0x20 0xa4 0x20 0x87 0x20 0xb5 0x20 0xd6 0x20 0xe0 0x20 0xe9 0x20 0xa5 0x20 0x80 0xfe 0x73 0x64 0x73
(á translates to ' ', which is chr(160) or 0xA0 in hexa. é is '‚' or chr(130), 0x82 in hexa, í is '¡', char(161) or 0xA1 in hexa and so on)
In Python, when I try to encode this to OEM I do the following:
def convertir_string_a_universe(cadena_python):
resultado = ''
for letra in cadena_python:
resultado += TablaString2UV[letra]
return resultado
And then, to get the bytes
txt_registro = convertir_string_a_universe(txt_orig)
datos = bytes(txt_registro, 'cp1252')
With this I get the following bytes:
b'\xa0 \x82 \xa1 \xa2 \xa3 \xa4 \x87 \xb5 \xd6 \xe0 \xe9 \xa5 \x80 \x9a'
My problem is that this OEM encoding uses non-printable characters, like in 'É' = chr(144) (0x90 in hexa). If I try to call bytes(txt_registro, 'cp1252') with an array where I hava translated 'É' into chr(0x90) I get this error:
caracteres_mal = 'Éü'
txt_registro = convertir_string_a_universe(txt_orig)
datos = bytes(txt_registro, 'cp1252')
File "C:\Users\Hector\PyCharmProjects\pyuniverse\pyuniverse\UniverseRegister.py", line 138, in reconstruir_registro_universe
datos = bytes(txt_registro, 'cp1252')
File "C:\Users\Hector\AppData\Local\Programs\Python\Python36-32\lib\encodings\cp1252.py", line 12, in encode
return codecs.charmap_encode(input,errors,encoding_table)
UnicodeEncodeError: 'charmap' codec can't encode character '\x90' in position 0: character maps to <undefined>
How can I do this OEM encoding without raising this UnicodeEncodeError?

This is because cp1252 does not know about chr(0x90). If you try with utf-8 instead, it will work.
>>> chr(0x90).encode("utf8")
b'\xc2\x90'
I don't understand why you are trying to convert to cp1252 though: you have applied a custom conversion map and then, with bytes(txt_registro, 'cp1252'), you are converting your result again to cp1552.
I think what you are looking for is something like:
datos = bytes(txt_orig, 'uv')
where uv is your cutom codec.
So you would have to write an encoder and a decoder for it (which is basically what you have done already). Take a look at https://docs.python.org/3/library/codecs.html#codecs.register
to register a new codec. The function you will register with it should return a CodecInfo object described upper in the documentation.
import codecs
def buscar_a_uv(codec):
if codec == "uv":
return codecs.CodecInfo(
convertir_string_a_universe, convertir_universe_a_string, name="uv")
else:
return None
codecs.register(buscar_a_uv)
datos = bytes(txt_orig, 'uv')
EDIT
The encoder/decoder functions should return bytes, so you would need to update convertir_string_a_universe a bit.

Encoding issues (i think) when reading serial interface

I am trying to read out my smart meter through a P1 interface.
Some python scripts are posted on the web to do so, and I tried them, but seem to be getting some partial garbage out of my serial interface (not completely - so my assumption is that the serial settings are correct though. Also since I am copying this from other scripts that seem to be working).
When trying to readout the interface using cu command it does show the right output, so my hardware seems to work.
I am running this on: Linux version 3.10.25+ (dc4#dc4-arm-01) (gcc version 4.7.2 20120731 (prerelease) (crosstool-NG linaro-1.13.1+bzr2458 - Linaro GCC 2012.08) ) #622 PREEMPT Fri Jan 3 18:41:00 GMT 2014
Here is what the output should look like - retrieved through:
command:
cu -l /dev/ttyUSB0 -s 9600 --parity=none
output
/INTENTIONALLY ALTERED
0-0:96.1.1(39373936353532302020202020202020)
1-0:1.8.1(05090.742*kWh)
1-0:1.8.2(06618.743*kWh)
1-0:2.8.1(00000.000*kWh)
1-0:2.8.2(00000.000*kWh)
0-0:96.14.0(0001)
1-0:1.7.0(0000.71*kW)
1-0:2.7.0(0000.00*kW)
0-0:17.0.0(999*A)
0-0:96.3.10(1)
0-0:96.13.1()
0-0:96.13.0()
0-1:96.1.0(3238303131303038323033333632313132)
0-1:24.1.0(03)
0-1:24.3.0(140806220000)(2C)(60)(1)(0-1:24.2.0)(m3)
(03447.404)
0-1:24.4.0(1)
!
When I use the following Python code:
# DSMR P1 uitlezen
# (c) 10-2012 - GJ - gratis te kopieren en te plakken
versie = "1.0"
import sys
import serial
##############################################################################
#Main program
##############################################################################
print ("DSMR P1 uitlezen", versie)
print ("Control-C om te stoppen")
print ("Pas eventueel de waarde ser.port aan in het python script")
#Set COM port config
ser = serial.Serial()
ser.baudrate = 9600
ser.bytesize=serial.SEVENBITS
ser.parity=serial.PARITY_EVEN
ser.stopbits=serial.STOPBITS_ONE
ser.xonxoff=0
ser.rtscts=0
ser.timeout=20
ser.port="/dev/ttyUSB0"
#Open COM port
try:
ser.open()
except:
sys.exit ("Fout bij het openen van %s. Aaaaarch." % ser.name)
#Initialize
#p1_teller is mijn tellertje voor van 0 tot 20 te tellen
p1_teller=0
while p1_teller < 20:
p1_line=''
#Read 1 line van de seriele poort
try:
p1_raw = ser.readline()
print str(p1_teller),':', p1_raw
except:
sys.exit ("Seriele poort %s kan niet gelezen worden. Aaaaaaaaarch." % ser.name )
#p1_str=str(p1_raw)
#p1_line=p1_str.strip()
# als je alles wil zien moet je de volgende line uncommenten
#print (p1_line.encode('ascii','ignore'))
p1_teller = p1_teller +1
#Close port and show status
try:
ser.close()
except:
sys.exit ("Oops %s. Programma afgebroken. Kon de seriele poort niet sluiten." % ser.name )
The output turns into this:
('DSMR P1 uitlezen', '1.0')
Control-C om te stoppen
Pas eventueel de waarde ser.port aan in het python script
0 : INTENTIONALLY ALTERED BUT ALSO WITH THE ? SYMBOLS IN THE ORIGINAL OUTPUT
1 : �
2 : 0-0:96.�.�(393�393635353�30�0�0�0�0�0�0�0�0��
3 : �-0:�.�.�(05090.�9����詍
4 : �-0:�.�.�(066��.��3���詍
5 : �-0:�.�.�(00000.000���詍
6 : �-0:�.�.�(00000.000���詍
7 : 0-0:96.��.0(000���
8 : �-0:�.�.0(0000.�0��ש�
9 : �-0:�.�.0(0000.00��ש�
10 : 0-0:��.0.0(999�A��
11 : 0-0:96.3.�0(���
12 : 0-0:96.�3.�(��
13 : 0-0:96.�3.0(��
14 : 0-�:96.�.0(3�3�303�3�30303�3�303333363�3�3�3���
15 : 0-�:��.�.0(03��
16 : 0-�:��.3.0(��0�06��0000�(�é(60�(��(0-�:��.�.0�(�3��
17 : (03���.�0���
18 : 0-�:��.�.0(���
19 : !�
so there are a lot of � characters in my python output, that I expect to be some encoding issue.. but not sure and not sure how to fix this... so any help is appreciated.

It looks like the bytes you read may still contain the parity bit, making them invalid characters. Try this to remove the 8th bit:
p1_raw = ''.join(chr(ord(ch) & 0x7f) for ch in p1_raw)

This is how I in the end fixed it - Will try Ldevries method too.... looks pretty clean:
#
# MBSolget P1 Telegram Catch
#
version = "v1.00"
import sys
import os
import stat
import serial
import datetime
import locale
###############################################################################################################
# Main program
###############################################################################################################
#Initialize
p1_telegram = False
p1_timestamp = ""
p1_teller = 0
p1_log = True
#Set COM port config
ser = serial.Serial()
ser.baudrate = 9600
ser.bytesize = serial.EIGHTBITS
ser.parity = serial.PARITY_NONE
ser.stopbits = serial.STOPBITS_ONE
ser.xonxoff = 1
ser.rtscts = 0
ser.timeout = 30
ser.port = "/dev/ttyUSB0"
#Show startup arguments
print ("MBSolget P1 Telegram Catch %s" % version)
print ("Control-C om af te breken")
print ("Poort: (%s)" % (ser.name) )
#Open COM port
try:
ser.open()
except:
sys.exit ("Fout bij het openen van poort %s. " % ser.name)
while p1_log:
p1_line = ''
try:
p1_raw = ser.readline()
except:
sys.exit ("Fout bij het lezen van poort %s. " % ser.name )
ser.close()
p1_raw = ''.join(chr(ch & 0x7f) for ch in p1_raw)
# for ch in p1_raw:
# print(chr(ch & 0x7f))
# print( chr(ord(ch) & 0x7f))
# print(p1_raw)
p1_str = p1_raw #str(p1_raw ,"utf-8",errors="ignore")
p1_line = p1_str.strip()
print (p1_line)
if p1_line[0:1] == "/":
p1_telegram = True
p1_teller = p1_teller + 1
f=open("/home/geoffrey/p1_temp.log", "w")
elif p1_line[0:1] == "!":
if p1_telegram:
p1_teller = 0
p1_telegram = False
p1_log = False
f.write (p1_line)
f.write ('\r\n')
f.close()
os.chmod("/home/geoffrey/p1_temp.log", stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
if p1_telegram:
f.write (p1_line)
f.write ('\r\n')
#Close port and show status
try:
ser.close()
except:
sys.exit ("Fout bij het sluiten van %s. Programma afgebroken." % ser.name )

I guess it has to do with some decoding of data. Using textIOWrapper the decoding works fine for me. Hopefully you can do anything with my view on the case:
To make the readout of the p1 protocol as easy as possible I'd suggest to use TextIOWrapper as I mentioned before, this way you can still read the serial port with the readline method. The "!" always ends the P1 telegram, so that can be used to detect the end of the message instead of using a counter. When a full telegram has been received, the telegram can be processed. Example:
import io
import serial
serialport = serial.Serial( # Configure Serial communication port
port = "/dev/ttyUSB0"
baudrate = 9600,
timeout = 11,
bytesize = serial.SEVENBITS,
parity = serial.PARITY_EVEN,
stopbits = serial.STOPBITS_ONE )
p1port = io.TextIOWrapper(io.BufferedReader(serialport, buffer_size=1), newline='\n', encoding='ascii')
P1Message = []
while True:
try:
rawline = p1port.readline()
except UnicodeDecodeError:
print "Encode error on readline"
if '!' in rawline:
# Process your P1Message here
P1Message = [] # Clear message, wait for new one
else:
P1Message.append(rawline)
Good luck!

In the code, change the byte size to EIGHTBITS and parity to PARITY_NONE. Those match what cu is using (and are also the defaults for PySerial).

Thanks to Geoffrey I finally figured out how to setup my C# SerialPort.
private static SerialPort CreateP1SerialPort(string name)
{
return new SerialPort
{
BaudRate = 115200,
DataBits = 8,
Handshake = Handshake.XOnXOff,
Parity = Parity.None,
PortName = name,
RtsEnable = false,
StopBits = StopBits.One,
ReadTimeout = 10 * 1000,
WriteTimeout = 1000
};
}
simple example:
public static void Main()
{
// this will help you find your com port name
var portNames = SerialPort.GetPortNames().OrderBy(x => x).ToArray();
// Create a new SerialPort object with default settings.
var serialPort = CreateP1SerialPort("COM7");
serialPort.Open();
while (true)
{
var line = serialPort.ReadExisting();
if (string.IsNullOrEmpty(line))
{
Thread.Sleep(10000); // one message every 10 seconds
} else
Console.WriteLine(line);
}
}