We Keep Coding

Converting unicode to gsm encoding in golang

I am working on migrating my project in python to golang and I have a use case for converting utf-8 encoding to corresponding gsm ones if possible. I am very new to go, it will be really helpful to get some documentation or examples around it.
For example: Python snippet
ằ as unicode -> a after gsm encoding
for character in text:
if is_gsm(character):
transliterated_text += character.encode('utf-8')
continue
if is_nonascii_utf8(character):
transliterated_char = unidecode.unidecode(character)
if transliterated_char == '?' or transliterated_char == '':
gsm = False
break
if transliterated_char != rc:
character = transliterated_char
transliterated_text += character
else:
transliterated_text += character.encode('utf-8')
if gsm and is_gsm(transliterated_text.decode('utf-8')):
text = transliterated_text.decode('utf-8')
Thanks

You can do it in this way:
package main
import (
"fmt"
"regexp"
"strings"
)
var utf8GsmChars = map[string]string{
`#`: "\x00", `£`: "\x01", `$`: "\x02",
`¥`: "\x03", `è`: "\x04", `é`: "\x05",
`ù`: "\x06", `ì`: "\x07", `ò`: "\x08",
`Ç`: "\x09", `Ø`: "\x0B", `ø`: "\x0C",
`Å`: "\x0E", `Δ`: "\x10", `_`: "\x11",
`Φ`: "\x12", `Γ`: "\x13", `Λ`: "\x14",
`Ω`: "\x15", `Π`: "\x16", `Ψ`: "\x17",
`Σ`: "\x18", `Θ`: "\x19", `Ξ`: "\x1A",
`Æ`: "\x1C", `æ`: "\x1D", `ß`: "\x1E",
`É`: "\x1F", `Ä`: "\x5B", `Ö`: "\x5C",
`Ñ`: "\x5D", `Ü`: "\x5E", `§`: "\x5F",
`¿`: "\x60", `ä`: "\x7B", `ö`: "\x7C",
`ñ`: "\x7D", `ü`: "\x7E", `à`: "\x7F",
`^`: "\x1B\x14", `{`: "\x1B\x28",
`}`: "\x1B\x29", `\`: "\x1B\x2F",
`[`: "\x1B\x3C", `~`: "\x1B\x3D",
`]`: "\x1B\x3E", `|`: "\x1B\x40",
`€`: "\x1B\x65",
}
var gsmUtf8Chars = map[string]string{
"\x00": "\x40",
"\x01": "\xC2\xA3",
"\x02": "\x24",
"\x03": "\xC2\xA5",
"\x04": "\xC3\xA8",
"\x05": "\xC3\xA9",
"\x06": "\xC3\xB9",
"\x07": "\xC3\xAC",
"\x08": "\xC3\xB2",
"\x09": "\xC3\x87",
"\x0B": "\xC3\x98",
"\x0C": "\xC3\xB8",
"\x0E": "\xC3\xB8",
"\x0F": "\xC3\xA5",
"\x10": "\xCE\x94",
"\x11": "\x5F",
"\x12": "\xCE\xA6",
"\x13": "\xCE\x93",
"\x14": "\xCE\xA0",
"\x15": "\xCE\xA9",
"\x16": "\xCE\xA0",
"\x17": "\xCE\xA8",
"\x18": "\xCE\xA3",
"\x19": "\xCE\x98",
"\x1A": "\xCE\x9E",
"\x1C": "\xC3\x86",
"\x1D": "\xC3\xA6",
"\x1E": "\xC3\x9F",
"\x1F": "\xC3\x89",
"\x20": "\x20",
"\x24": "\xC2\xA4",
"\x40": "\xC2\xA1",
"\x5B": "\xC3\x84",
"\x5C": "\xC3\x96",
"\x5D": "\xC3\x91",
"\x5E": "\xC3\x9C",
"\x5F": "\xC2\xA7",
"\x60": "\xC2\xBF",
"\x7B": "\xC3\xA8",
"\x7C": "\xC3\xB6",
"\x7D": "\xC3\xB1",
"\x7E": "\xC3\xBC",
"\x7F": "\xC3\xA0",
}
func UTF8ToGsm0338(text string) string {
var s string = text
for k, v := range utf8GsmChars {
s = strings.Replace(s, k, v, -1)
}
re := regexp.MustCompile("[\\x{0080}-\\x{10FFFF}]")
s = re.ReplaceAllString(s, "?")
return s
}
func GSM0338ToUTF8(text string) string {
var s string = text
for k, v := range gsmUtf8Chars {
s = strings.Replace(s, k, v, -1)
}
return s
}
func main() {
s := "Hello World"
gsm := UTF8ToGsm0338(s)
utf8 := GSM0338ToUTF8(gsm)
fmt.Printf("word before: %s\nword after gsm: %s\nword after utf8: %s\n", s, gsm, utf8)
}

Optimize byte array escaping performance python

I need to perform custom escaping over a byte array in python. However, during escaping python converts bytes to integers, making performance optimization very difficult. How can I speed up my escaping function?
ESCAPE_DICT={
0x00: [0x5C,0x7A], # null -> \z 0x5c 0x7a
0x22: [0x5C,0x71], # " -> \q 0x5c 0x71
0x3B: [0x5C,0x73], # ; -> \s 0x5c 0x73
0x5C: [0x5C,0x5C], # \ -> \\ 0x5c 0x5c
0x0A: [0x5C,0x6E], # line-feed -> \n 0x5c 0x6e
0x0C: [0x5C,0x66], # form-feed -> \f 0x5c 0x66
0x0D: [0x5C,0x63], # carr-return -> \c 0x5c 0x63
}
def escape(string: bytes):
str_len=string.__len__()
escaped_list=[]
for i in range(0,str_len):
curr_byte=string[i]
escape = ESCAPE_DICT.get(curr_byte)
if escape is None:
# Don't escape current byte
escaped_list.append(curr_byte)
else:
# Escape current byte
escaped_list.extend(escape)
return bytes(escaped_array)

import re
ESCAPE_DICT = {
b'\x00': rb'\z', # null
b'"': rb'\q',
b';': rb'\s',
b'\\': rb'\\',
b'\n': rb'\n', # linefeed
b'\f': rb'\f', # formfeed
b'\r': rb'\c', # carriage return
}
ESCAPE_CLASS = '[' + ''.join(r'\x' + e.hex() for e in ESCAPE_DICT) + ']'
ESCAPE_REGEX = re.compile(ESCAPE_CLASS.encode())
def escape(string: bytes) -> bytes:
return re.sub(ESCAPE_REGEX, lambda m: ESCAPE_DICT[m.group(0)], string)
x = b'"abc\ndef\rpqr\x00stu\\xyz"'
y = escape(x)
from pprint import pprint
pprint(ESCAPE_CLASS)
pprint(ESCAPE_REGEX)
pprint(x)
pprint(y)
# =>
# '[\\x00\\x22\\x3b\\x5c\\x0a\\x0c\\x0d]'
# re.compile(b'[\\x00\\x22\\x3b\\x5c\\x0a\\x0c\\x0d]')
# b'"abc\ndef\rpqr\x00stu\\xyz"'
# b'\\qabc\\ndef\\cpqr\\zstu\\\\xyz\\q'
You can read the rb prefix as “raw bytes”.
Your escapes are a bit strange, though. E.g., the carriage return is normally \r, not \c, and \s normally stands for generic whitespace.

Cyrillic symbols decode in numpy array

I need to get pieplot with labels in Cyrillic symbols, that is in df.index
plt.pie(df['reg_created'], labels = df.index)
So, it's return error:
UnicodeDecodeError: 'ascii' codec can't decode byte 0xd0 in position 0: ordinal not in range(128)
df.index:
Index([u'Бизнес', u'Вечеринки', u'Выставки', u'Гражданские проекты',
u'Для детей', u'Другие развлечения', u'Другие события', u'Еда',
u'ИТ и интернет', u'Иностранные языки', u'Интеллектуальные игры',
u'Искусство и культура', u'Кино', u'Концерты', u'Красота и здоровье',
u'Наука', u'Образование за рубежом', u'Психология и самопознание',
u'Спорт', u'Театры', u'Хобби и творчество', u'Экскурсии и путешествия'],
dtype='object', name=u'name')
matplotlib.pyplot.pie label parameter shoud be a list, so if I try:
df.index.tolist()
returns:
['\xd0\x91\xd0\xb8\xd0\xb7\xd0\xbd\xd0\xb5\xd1\x81', '\xd0\x92\xd0\xb5\xd1\x87\xd0\xb5\xd1\x80\xd0\xb8\xd0\xbd\xd0\xba\xd0\xb8', '\xd0\x92\xd1\x8b\xd1\x81\xd1\x82\xd0\xb0\xd0\xb2\xd0\xba\xd0\xb8', '\xd0\x93\xd1\x80\xd0\xb0\xd0\xb6\xd0\xb4\xd0\xb0\xd0\xbd\xd1\x81\xd0\xba\xd0\xb8\xd0\xb5 \xd0\xbf\xd1\x80\xd0\xbe\xd0\xb5\xd0\xba\xd1\x82\xd1\x8b', '\xd0\x94\xd0\xbb\xd1\x8f \xd0\xb4\xd0\xb5\xd1\x82\xd0\xb5\xd0\xb9', '\xd0\x94\xd1\x80\xd1\x83\xd0\xb3\xd0\xb8\xd0\xb5 \xd1\x80\xd0\xb0\xd0\xb7\xd0\xb2\xd0\xbb\xd0\xb5\xd1\x87\xd0\xb5\xd0\xbd\xd0\xb8\xd1\x8f', '\xd0\x94\xd1\x80\xd1\x83\xd0\xb3\xd0\xb8\xd0\xb5 \xd1\x81\xd0\xbe\xd0\xb1\xd1\x8b\xd1\x82\xd0\xb8\xd1\x8f', '\xd0\x95\xd0\xb4\xd0\xb0', '\xd0\x98\xd0\xa2 \xd0\xb8 \xd0\xb8\xd0\xbd\xd1\x82\xd0\xb5\xd1\x80\xd0\xbd\xd0\xb5\xd1\x82', '\xd0\x98\xd0\xbd\xd0\xbe\xd1\x81\xd1\x82\xd1\x80\xd0\xb0\xd0\xbd\xd0\xbd\xd1\x8b\xd0\xb5 \xd1\x8f\xd0\xb7\xd1\x8b\xd0\xba\xd0\xb8', '\xd0\x98\xd0\xbd\xd1\x82\xd0\xb5\xd0\xbb\xd0\xbb\xd0\xb5\xd0\xba\xd1\x82\xd1\x83\xd0\xb0\xd0\xbb\xd1\x8c\xd0\xbd\xd1\x8b\xd0\xb5 \xd0\xb8\xd0\xb3\xd1\x80\xd1\x8b', '\xd0\x98\xd1\x81\xd0\xba\xd1\x83\xd1\x81\xd1\x81\xd1\x82\xd0\xb2\xd0\xbe \xd0\xb8 \xd0\xba\xd1\x83\xd0\xbb\xd1\x8c\xd1\x82\xd1\x83\xd1\x80\xd0\xb0', '\xd0\x9a\xd0\xb8\xd0\xbd\xd0\xbe', '\xd0\x9a\xd0\xbe\xd0\xbd\xd1\x86\xd0\xb5\xd1\x80\xd1\x82\xd1\x8b', '\xd0\x9a\xd1\x80\xd0\xb0\xd1\x81\xd0\xbe\xd1\x82\xd0\xb0 \xd0\xb8 \xd0\xb7\xd0\xb4\xd0\xbe\xd1\x80\xd0\xbe\xd0\xb2\xd1\x8c\xd0\xb5', '\xd0\x9d\xd0\xb0\xd1\x83\xd0\xba\xd0\xb0', '\xd0\x9e\xd0\xb1\xd1\x80\xd0\xb0\xd0\xb7\xd0\xbe\xd0\xb2\xd0\xb0\xd0\xbd\xd0\xb8\xd0\xb5 \xd0\xb7\xd0\xb0 \xd1\x80\xd1\x83\xd0\xb1\xd0\xb5\xd0\xb6\xd0\xbe\xd0\xbc', '\xd0\x9f\xd1\x81\xd0\xb8\xd1\x85\xd0\xbe\xd0\xbb\xd0\xbe\xd0\xb3\xd0\xb8\xd1\x8f \xd0\xb8 \xd1\x81\xd0\xb0\xd0\xbc\xd0\xbe\xd0\xbf\xd0\xbe\xd0\xb7\xd0\xbd\xd0\xb0\xd0\xbd\xd0\xb8\xd0\xb5', '\xd0\xa1\xd0\xbf\xd0\xbe\xd1\x80\xd1\x82', '\xd0\xa2\xd0\xb5\xd0\xb0\xd1\x82\xd1\x80\xd1\x8b', '\xd0\xa5\xd0\xbe\xd0\xb1\xd0\xb1\xd0\xb8 \xd0\xb8 \xd1\x82\xd0\xb2\xd0\xbe\xd1\x80\xd1\x87\xd0\xb5\xd1\x81\xd1\x82\xd0\xb2\xd0\xbe', '\xd0\xad\xd0\xba\xd1\x81\xd0\xba\xd1\x83\xd1\x80\xd1\x81\xd0\xb8\xd0\xb8 \xd0\xb8 \xd0\xbf\xd1\x83\xd1\x82\xd0\xb5\xd1\x88\xd0\xb5\xd1\x81\xd1\x82\xd0\xb2\xd0\xb8\xd1\x8f']
if I print list by element:
for i in df.index.tolist():
print i
returns Cyrillic text
Бизнес
Вечеринки
Выставки
Гражданские проекты
...
Why I have difference in print list of Cyrillic text and print that list by element?
And what I shoud get to pyplot.pie label param for Cyrillic labels?

You got your answer in the error message, the charters are decoded as ASCII and not as UTF-8
https://stackoverflow.com/a/10406161
https://stackoverflow.com/a/36454865

nrf24l01 communication raspberry pi not able to connect

I am new to Stackoverflow. I have searched for answer, but didn't find anything.
I have two Raspberry Pi 2B+, each with nRF24l01 connected. I found few libraries to make this connect, only one give any results, but not connections. This one: Github BLavery
I write script to send and to recv:
send.py:
import RPi.GPIO as GPIO
from lib_nrf24 import NRF24
import time
import spidev
GPIO.setmode(GPIO.BCM)
pipes = [[0xe7, 0xe7, 0xe7, 0xe7, 0xe7], [0xc2, 0xc2, 0xc2, 0xc2, 0xc2]]
radio = NRF24(GPIO, spidev.SpiDev())
radio.begin(0, 17)
radio.setPayloadSize(32)
radio.setChannel(0x60)
radio.setDataRate(NRF24.BR_2MBPS)
radio.setPALevel(NRF24.PA_MIN)
radio.setAutoAck(True)
radio.enableDynamicPayloads()
radio.enableAckPayload()
radio.openWritingPipe(pipes[1])
radio.printDetails()
while True:
message = list("Hello World")
radio.write(message)
print("We sent the message of {}".format(message))
# Check if it returned a ackPL
if radio.isAckPayloadAvailable():
returnedPL = []
radio.read(returnedPL, radio.getDynamicPayloadSize())
print("Our returned payload was {}".format(returnedPL))
else:
print("No payload received")
time.sleep(1)
recv.py:
import RPi.GPIO as GPIO
from lib_nrf24 import NRF24
import time
import spidev
GPIO.setmode(GPIO.BCM)
pipes = [[0xe7, 0xe7, 0xe7, 0xe7, 0xe7], [0xc2, 0xc2, 0xc2, 0xc2, 0xc2]]
radio = NRF24(GPIO, spidev.SpiDev())
radio.begin(0, 17)
radio.setPayloadSize(32)
radio.setChannel(0x60)
radio.setDataRate(NRF24.BR_2MBPS)
radio.setPAlevel(NRF24.PA_MIN)
radio.setAutoAck(True)
radio.enableDynamicPayloads()
radio.enableAckPayload()
radio.openReadingPipe(1, pipes[1])
radio.printDetails()
radio.startListening()
while True:
ackPL = [1]
while not radio.available (0):
time.sleep(1/100)
receivedMessage = []
radio.read(receivedMessage, radio.getDynamicPayloadSize())
print("Received: {}".format(receivedMessage))
print("Translating the receivedMessage into unicode characters...")
string = ""
for n in receivedMessage:
# Decode into standard i=unicode set
if (n >=32 and n <= 126):
string += chr(n)
print(string)
radio.writeAckPayload(1, ackPL, len(ackPL))
print("Loaded payload reply of {}".format(ackPL))
Everything seems to be alright, below are code returned by both scripts:
send:
STATUS = 0x03 RX_DR=0 TX_DS=0 MAX_RT=0 RX_P_NO=1 TX_FULL=1
RX_ADDR_P0-1 =
0xf8f8f8f8f8 0xf8f8f8f8f8
RX_ADDR_P2-5 =
0xf8
0xf9
0xf9
0xf9
TX_ADDR =
0xf8f8f8f8f8
RX_PW_P0-6 =
0x0c
0x00
0x00
0x00
0x00
0x00
EN_AA =
0x0f
EN_RXADDR =
0x00
RF_CH =
0x1c
RF_SETUP =
0x00
CONFIG =
0x03
DYNPD/FEATURE =
0x03
0x01
Data Rate = 1MBPS
Model = nRF24L01
CRC Length = Disabled
PA Power = PA_MIN
We sent the message of ['H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd']
No payload received
recv.py:
STATUS = 0x03 RX_DR=0 TX_DS=0 MAX_RT=0 RX_P_NO=1 TX_FULL=1
RX_ADDR_P0-1 =
0xf8f8f8f8f8 0xf8f8f8f8f8
RX_ADDR_P2-5 =
0xf8
0xf9
0xf9
0xf9
TX_ADDR =
0xf8f8f8f8f8
RX_PW_P0-6 =
0x0c
0x0c
0x00
0x00
0x00
0x00
EN_AA =
0x0f
EN_RXADDR =
0x00
RF_CH =
0x1c
RF_SETUP =
0x00
CONFIG =
0x03
DYNPD/FEATURE =
0x03
0x01
Data Rate = 1MBPS
Model = nRF24L01
CRC Length = Disabled
PA Power = PA_MIN
Received: []
Translating the receivedMessage into unicode characters...
Loaded payload reply of [1]
I don't really understand why it won't connect one to other,
Both have the same wiring:
nRF24L01-Raspberry Pi (Pin#)
GND - GND (6)
VCC - 3,3V (1)
CE - GPIO17 (11)
CSN - GPIO08(24)
SCK - GPIO11 (23)
MOSI - GPIO10 (19)
MISO - GPIO25 (22)
IRQ - unconnected
I need to send information from one RPi to second to control engine via PWM.
Can i ask for help

UnicodeWarning: special characters in Tkinter

I have written a program in Tkinter (Python 2.7), a scrabblehelper in Norwegian which contains some special characters (æøå), which means my wordlist (ordliste) contains words with special characters.
When I run my function finnord(c*), it returns 'cd'. I am using an entry.get() to get the word to put in my function.
My problem is with the encoding of entry.get(). I have local coding UTF-8, but I get an UniCodeError when I am writing any special characters in my entrybox and matching them to my wordliste.
Here is my output.
Warning (from warnings module):
File "C:\pythonprog\scrabble\feud.py", line 46
if s not in liste and s in ordliste:
UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode -
interpreting them as being unequal
When i write in my shell:
> ordinn.get()
u'k\xf8**e'
> ordinn.get().encode('utf-8')
'k\xc3\xb8**e'
> print ordinn.get()
kø**e
> print ordinn.get().encode('utf-8')
kø**e
Anyone knows why I can't match ordinn.get() (entry) to my wordlist ?

I can reproduce the error this way:
% python
Python 2.7.2+ (default, Oct 4 2011, 20:03:08)
[GCC 4.6.1] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> 'k\xf8**e' in [u'k\xf8**e']
__main__:1: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal
False
So perhaps s is a str object, and liste or ordliste contains unicode, or (as eryksun points out in the comments) vice versa. The solution is to decode the str objects (most likely with the utf-8 codec) to make them unicode.
If that does not help, please print out and post the output of
print(repr(s))
print(repr(liste))
print(repr(ordliste))
I believe the problem can be avoided by converting all strings to unicode.
When you generate ordliste from norsk.txt, use
codecs.open('norsk.txt','r','utf-8'):
encoding = sys.stdin.encoding
with codecs.open('norsk.txt','r','utf-8') as fil:
ordliste = [line.rstrip(u'\n') for line in fil]
Convert all user input to unicode as soon as possible:
def get_unicode(widget):
streng = widget.get()
try:
streng = streng.decode('utf-8')
except UnicodeEncodeError:
pass
return streng
So perhaps try this:
import Tkinter as tk
import tkMessageBox
import codecs
import itertools
import sys
alfabetet = (u"abcdefghijklmnopqrstuvwxyz"
u"\N{LATIN SMALL LETTER AE}"
u"\N{LATIN SMALL LETTER O WITH STROKE}"
u"\N{LATIN SMALL LETTER A WITH RING ABOVE}")
encoding = sys.stdin.encoding
with codecs.open('norsk.txt','r',encoding) as fil:
ordliste = set(line.rstrip(u'\n') for line in fil)
def get_unicode(widget):
streng = widget.get()
if isinstance(streng,str):
streng = streng.decode('latin-1')
return streng
def siord():
alfa=lagtabell()
try:
streng = get_unicode(ordinn)
ordene=finnord(streng,alfa)
if len(ordene) == 0:
# There are no words that match
tkMessageBox.showinfo('Dessverre..','Det er ingen ord som passer...')
else:
# Done: The words that fit the pattern
tkMessageBox.showinfo('Ferdig',
'Ordene som passer er:\n'+ordene.encode('utf-8'))
except Exception as err:
# There has been a mistake .. Check your word
print(repr(err))
tkMessageBox.showerror('ERROR','Det har skjedd en feil.. Sjekk ordet ditt.')
def finnord(streng,alfa):
liste = set()
for substitution in itertools.permutations(alfa,streng.count(u'*')):
s = streng
for ch in substitution:
s = s.replace(u'*',ch,1)
if s in ordliste:
liste.add(s)
liste = [streng]+list(liste)
return u','.join(liste)+u'.'
def lagtabell():
tinbox = get_unicode(bokstinn)
if not tinbox.isalpha():
alfa = alfabetet
else:
alfa = tinbox.lower()
return alfa
root = tk.Tk()
root.title('FeudHjelper av Martin Skow Røed')
root.geometry('400x250+450+200')
# root.iconbitmap('data/ikon.ico')
skrift1 = tk.Label(root,
text = '''\
Velkommen til FeudHjelper. Skriv inn de bokstavene du har, og erstatt ukjente med *.
F. eks: sl**ge
Det er kun lov til å bruke tre stjerner, altså tre ukjente bokstaver.''',
font = ('Verdana',8), wraplength=350)
skrift1.pack(pady = 5)
ordinn = tk.StringVar(None)
tekstboks = tk.Entry(root, textvariable = ordinn)
tekstboks.pack(pady = 5)
# What letters do you have? Eg "ahneki". Leave blank here if you want all the words.
skrift2 = tk.Label(root, text = '''Hvilke bokstaver har du? F. eks "ahneki". La det være blankt her hvis du vil ha alle ordene.''',
font = ('Verdana',8), wraplength=350)
skrift2.pack(pady = 10)
bokstinn = tk.StringVar(None)
tekstboks2 = tk.Entry(root, textvariable = bokstinn)
tekstboks2.pack()
knapp = tk.Button(text = 'Finn ord!', command = siord)
knapp.pack(pady = 10)
root.mainloop()