We Keep Coding

Converting unicode to gsm encoding in golang

I am working on migrating my project in python to golang and I have a use case for converting utf-8 encoding to corresponding gsm ones if possible. I am very new to go, it will be really helpful to get some documentation or examples around it.
For example: Python snippet
ằ as unicode -> a after gsm encoding
for character in text:
if is_gsm(character):
transliterated_text += character.encode('utf-8')
continue
if is_nonascii_utf8(character):
transliterated_char = unidecode.unidecode(character)
if transliterated_char == '?' or transliterated_char == '':
gsm = False
break
if transliterated_char != rc:
character = transliterated_char
transliterated_text += character
else:
transliterated_text += character.encode('utf-8')
if gsm and is_gsm(transliterated_text.decode('utf-8')):
text = transliterated_text.decode('utf-8')
Thanks

You can do it in this way:
package main
import (
"fmt"
"regexp"
"strings"
)
var utf8GsmChars = map[string]string{
`#`: "\x00", `£`: "\x01", `$`: "\x02",
`¥`: "\x03", `è`: "\x04", `é`: "\x05",
`ù`: "\x06", `ì`: "\x07", `ò`: "\x08",
`Ç`: "\x09", `Ø`: "\x0B", `ø`: "\x0C",
`Å`: "\x0E", `Δ`: "\x10", `_`: "\x11",
`Φ`: "\x12", `Γ`: "\x13", `Λ`: "\x14",
`Ω`: "\x15", `Π`: "\x16", `Ψ`: "\x17",
`Σ`: "\x18", `Θ`: "\x19", `Ξ`: "\x1A",
`Æ`: "\x1C", `æ`: "\x1D", `ß`: "\x1E",
`É`: "\x1F", `Ä`: "\x5B", `Ö`: "\x5C",
`Ñ`: "\x5D", `Ü`: "\x5E", `§`: "\x5F",
`¿`: "\x60", `ä`: "\x7B", `ö`: "\x7C",
`ñ`: "\x7D", `ü`: "\x7E", `à`: "\x7F",
`^`: "\x1B\x14", `{`: "\x1B\x28",
`}`: "\x1B\x29", `\`: "\x1B\x2F",
`[`: "\x1B\x3C", `~`: "\x1B\x3D",
`]`: "\x1B\x3E", `|`: "\x1B\x40",
`€`: "\x1B\x65",
}
var gsmUtf8Chars = map[string]string{
"\x00": "\x40",
"\x01": "\xC2\xA3",
"\x02": "\x24",
"\x03": "\xC2\xA5",
"\x04": "\xC3\xA8",
"\x05": "\xC3\xA9",
"\x06": "\xC3\xB9",
"\x07": "\xC3\xAC",
"\x08": "\xC3\xB2",
"\x09": "\xC3\x87",
"\x0B": "\xC3\x98",
"\x0C": "\xC3\xB8",
"\x0E": "\xC3\xB8",
"\x0F": "\xC3\xA5",
"\x10": "\xCE\x94",
"\x11": "\x5F",
"\x12": "\xCE\xA6",
"\x13": "\xCE\x93",
"\x14": "\xCE\xA0",
"\x15": "\xCE\xA9",
"\x16": "\xCE\xA0",
"\x17": "\xCE\xA8",
"\x18": "\xCE\xA3",
"\x19": "\xCE\x98",
"\x1A": "\xCE\x9E",
"\x1C": "\xC3\x86",
"\x1D": "\xC3\xA6",
"\x1E": "\xC3\x9F",
"\x1F": "\xC3\x89",
"\x20": "\x20",
"\x24": "\xC2\xA4",
"\x40": "\xC2\xA1",
"\x5B": "\xC3\x84",
"\x5C": "\xC3\x96",
"\x5D": "\xC3\x91",
"\x5E": "\xC3\x9C",
"\x5F": "\xC2\xA7",
"\x60": "\xC2\xBF",
"\x7B": "\xC3\xA8",
"\x7C": "\xC3\xB6",
"\x7D": "\xC3\xB1",
"\x7E": "\xC3\xBC",
"\x7F": "\xC3\xA0",
}
func UTF8ToGsm0338(text string) string {
var s string = text
for k, v := range utf8GsmChars {
s = strings.Replace(s, k, v, -1)
}
re := regexp.MustCompile("[\\x{0080}-\\x{10FFFF}]")
s = re.ReplaceAllString(s, "?")
return s
}
func GSM0338ToUTF8(text string) string {
var s string = text
for k, v := range gsmUtf8Chars {
s = strings.Replace(s, k, v, -1)
}
return s
}
func main() {
s := "Hello World"
gsm := UTF8ToGsm0338(s)
utf8 := GSM0338ToUTF8(gsm)
fmt.Printf("word before: %s\nword after gsm: %s\nword after utf8: %s\n", s, gsm, utf8)
}

Custom encoding and decoding UTF special characters

Just for fun I've been embedding text in images. The following code is a distillation and demonstration of the encoding and decoding mechanism I am using.
class encChar:
def __init__(self,char):
self.p = self.enc(char)
def enc(self,char):
d = bin(ord(char)).split('b')[1]
while len(d)<8:
d = "0"+d
rdif = int(d[0])*4 + int(d[1])*2 + int(d[2])*1
gdif = int(d[3])*2 + int(d[4])*1
bdif = int(d[5])*4 + int(d[6])*2 + int(d[7])*1
return (rdif,gdif,bdif)
def dec(self):
dmap = {0:"000",1:"001",2:"010",3:"011",4:"100",5:"101",6:"110",7:"111"}
r = dmap[self.p[0]]
g = dmap[self.p[1]][1:]
b = dmap[self.p[2]]
n = int(r+g+b,2)
return chr(int(r+g+b,2))
testStr = """Languages
Deutsch
Español
Français
한국어
Italiano
Русский
Tagalog
Tiếng Việt
中文"""
result= ""
for line in testStr.split("\n"):
result+=line+"\n"
print(line)
print("".join([encChar(k).dec() for k in line]))
result+="".join([encChar(k).dec() for k in line])+"\n"
print()
result+="\n"
with open("errorop.txt","w",encoding="utf8") as op:
op.write(result)
Which produces the following document:
Languages
Languages
Deutsch
Deutsch
Español
Español
Français
Français
한국어
Õ­Å
Italiano
Italiano
Русский
Tagalog
Tagalog
Tiếng Việt
Tiõng Viöt
中文
Ë
As you can see several runes are altered by the process and I'm wondering how I can preserve them through this process.

python regex gives 1 instead of 01

Taken reference Why doesn't [01-12] range work as expected?
m = re.search(r"(\w+)\[([0-9]+)\:([0-9]+)\]", DUNESX[01:44])
or
m = re.search(r"(\w+)\[(0[1-9]|1[0-9]|2[0-9]|3[0-9]|4[0-9])\:([0-9]+)\]", DUNESX[01:44])
or
m = re.search(r"(\w+)\[(0?[1-9]|1[0-9]|2[0-9]|3[0-9]|4[0-9])\:([0-9]+)\]", DUNESX[01:44])
or
m = re.search(r"(\w+)\[([0-1][0-9]+)\:([0-9]+)\]", DUNESX[01:44])
But output from above expressions are
['DUNESX1', 'DUNESX2', 'DUNESX3', 'DUNESX4', 'DUNESX5', 'DUNESX6', 'DUNESX7', 'DUNESX8', 'DUNESX9', 'DUNESX10', 'DUNESX11', 'DUNESX12', 'DUNESX13', 'DUNESX14', 'DUNESX15', 'DUNESX16', 'DUNESX17', 'DUNESX18', 'DUNESX19', 'DUNESX20', 'DUNESX21', 'DUNESX22', 'DUNESX23', 'DUNESX24', 'DUNESX25', 'DUNESX26', 'DUNESX27', 'DUNESX28', 'DUNESX29', 'DUNESX30', 'DUNESX31', 'DUNESX32', 'DUNESX33', 'DUNESX34', 'DUNESX35', 'DUNESX36', 'DUNESX37', 'DUNESX38', 'DUNESX39', 'DUNESX40', 'DUNESX41', 'DUNESX42', 'DUNESX43', 'DUNESX44']
Doesn't provide desired output like
['DUNESX01', 'DUNESX02', 'DUNESX03', 'DUNESX04', 'DUNESX05', 'DUNESX06', 'DUNESX07', 'DUNESX08', 'DUNESX09', 'DUNESX10', 'DUNESX11', 'DUNESX12', 'DUNESX13', 'DUNESX14', 'DUNESX15', 'DUNESX16', 'DUNESX17', 'DUNESX18', 'DUNESX19', 'DUNESX20', 'DUNESX21', 'DUNESX22', 'DUNESX23', 'DUNESX24', 'DUNESX25', 'DUNESX26', 'DUNESX27', 'DUNESX28', 'DUNESX29', 'DUNESX30', 'DUNESX31', 'DUNESX32', 'DUNESX33', 'DUNESX34', 'DUNESX35', 'DUNESX36', 'DUNESX37', 'DUNESX38', 'DUNESX39', 'DUNESX40', 'DUNESX41', 'DUNESX42', 'DUNESX43', 'DUNESX44']
Complete code is
import re
group_list = ['DUNESX[01:44]']
host_list = getgrandchild(group_list)
def getgrandchild(child):
nodelist = []
if child is None:
return
for nodes in child:
print(nodes)
if re.match(r".*(\[[0-1][0-9]+\:[0-9]+\])", nodes):
m = re.search(r"(\w+)\[([0-9]+)\:([0-9]+)\]", nodes)
lb = int(m.group(2))
ub = int(m.group(3))
for i in range(lb, ub+1):
nodelist.append(m.group(1)+str(i))
elif re.match(r"(\w+)", nodes):
m = re.search(r"(\w+)", nodes)
nodelist.append(m.group(1))

I think I understand what you are trying to do. Here is a code that helps:
def getgrandchild(child):
nodelist = []
for nodes in child:
m = re.search("(\w+)\[([0-9]+)\:([0-9]+)\]",nodes)
for i in range(int(m.group(2)),int(m.group(3))+1):
nodelist.append(m.group(1)+str(i).zfill(len(m.group(2))))
return nodelist
You can see i've skipped some steps, but you can have them. I've focused on the main part.
We use zfill to add numbers of the form '001' or '01', which is explained here.
So, for this code, if you give:
getgrandchild(['DUNESX[01:44]'])
you get:
['DUNESX01', 'DUNESX02', 'DUNESX03', 'DUNESX04', 'DUNESX05', 'DUNESX06', 'DUNESX07', 'DUNESX08', 'DUNESX09', 'DUNESX10', 'DUNESX11', 'DUNESX12', 'DUNESX13', 'DUNESX14', 'DUNESX15', 'DUNESX16', 'DUNESX17', 'DUNESX18', 'DUNESX19', 'DUNESX20', 'DUNESX21', 'DUNESX22', 'DUNESX23', 'DUNESX24', 'DUNESX25', 'DUNESX26', 'DUNESX27', 'DUNESX28', 'DUNESX29', 'DUNESX30', 'DUNESX31', 'DUNESX32', 'DUNESX33', 'DUNESX34', 'DUNESX35', 'DUNESX36', 'DUNESX37', 'DUNESX38', 'DUNESX39', 'DUNESX40', 'DUNESX41', 'DUNESX42', 'DUNESX43', 'DUNESX44']
Also, if you give:
getgrandchild(['PYTHON[001:025]'])
you get:
['PYTHON001', 'PYTHON002', 'PYTHON003', 'PYTHON004', 'PYTHON005', 'PYTHON006', 'PYTHON007', 'PYTHON008', 'PYTHON009', 'PYTHON010', 'PYTHON011', 'PYTHON012', 'PYTHON013', 'PYTHON014', 'PYTHON015', 'PYTHON016', 'PYTHON017', 'PYTHON018', 'PYTHON019', 'PYTHON020', 'PYTHON021', 'PYTHON022', 'PYTHON023', 'PYTHON024', 'PYTHON025']

Iterating through a yaml file

I am trying to iterate through a yaml file. I want to extract the contents
ipv6: "2031:31:31:31:: 2032:32:32:32:: 2033:33:33:33:: 2034:34:34:34:: 2035:35:35:35::"
Below is my code. I get the below error :
for x in self.dhcp_dict['subnets'][sub]['ipv4'].split():
TypeError: string indices must be integers, not str
Can anyone tell em where i am going wrong? Thanks
Jessi
Code:
dict = yaml.load(fd)
self.server_dict = dict['server_configs']
self.interface_dict = self.server_dict['interface']
self.dhcp_dict = self.server_dict['dhcp_config']
def configureDhcpv6(self):
pdb.set_trace()
log.info ("Writing the dhcp.conf file")
infile = open('v6.txt', 'w+')
self.lease_time = self.dhcp_dict['lease_time']
infile.write("default-lease-time %s; \n" %(self.lease_time))
infile.write("preferred-lifetime 604800;\noption dhcp-renewal-time 604800;\noption dhcp-rebinding-time 7200;\noption dhcp6.domain-search cisco.com;\noption dhcp6.preference 255;\noption dhcp6.rapid-commit;\noption dhcp6.info-refresh-time 21600;\ndhcpv6-lease-file-name /var/lib/dhcpd/dhcpd6.leases;\nauthoritative;\nlog-facility local7;\n\n")
for sub in self.dhcp_dict['subnets']:
if (sub == 'relay'):
for x in self.dhcp_dict['subnets'][sub]['ipv6'].split():
range6 = sub + "11" + " " + sub + "254"
infile.write("Subnet 6 %s/64 {\n" % (sub))
infile.write(" range6 %s;\n}\n\n" % (range6))
infile.close()
Yaml file:
dhcp_config:
lease_time: "300"
relay_server: "5.5.5.0"
subnets:
relay:
ipv4: "30.30.30.0 31.31.31.0 32.32.32.0 33.33.33.0 34.34.34.0 35.35.35.0"
ipv6: "2031:31:31:31:: 2032:32:32:32:: 2033:33:33:33:: 2034:34:34:34:: 2035:35:35:35::"
smart_relay: "31.1.1.0 32.1.1.0 33.1.1.0 34.1.1.0 35.1.1.0"
snoop: "36.36.36.0 37.37.37.0 38.38.38.0 39.39.39.0 30.30.30.0"

Disassemble Python code to dictionary

I'd like to develop a small debugging tool for Python programs. For the "Dynamic Slicing" feature, I need to find the variables that are accessed in a statement, and find the type of access (read or write) for those variables.
But the only disassembly feature that's built into Python is dis.disassemble, and that just prints the disassembly to standard output:
>>> dis.disassemble(compile('x = a + b', '', 'single'))
1 0 LOAD_NAME 0 (a)
3 LOAD_NAME 1 (b)
6 BINARY_ADD
7 STORE_NAME 2 (x)
10 LOAD_CONST 0 (None)
13 RETURN_VALUE
I'd like to be able to transform the disassembly into a dictionary of sets describing which variables are used by each instruction, like this:
>>> my_disassemble('x = a + b')
{'LOAD_NAME': set(['a', 'b']), 'STORE_NAME': set(['x'])}
How can I do this?

Read the source code for the dis module and you'll see that it's easy to do your own disassembly and generate whatever output format you like. Here's some code that generates the sequence of instructions in a code object, together with their arguments:
from opcode import *
def disassemble(co):
"""
Disassemble a code object and generate its instructions.
"""
code = co.co_code
n = len(code)
extended_arg = 0
i = 0
free = None
while i < n:
c = code[i]
op = ord(c)
i = i+1
if op < HAVE_ARGUMENT:
yield opname[op],
else:
oparg = ord(code[i]) + ord(code[i+1])*256 + extended_arg
extended_arg = 0
i = i+2
if op == EXTENDED_ARG:
extended_arg = oparg*65536L
if op in hasconst:
arg = co.co_consts[oparg]
elif op in hasname:
arg = co.co_names[oparg]
elif op in hasjrel:
arg = repr(i + oparg)
elif op in haslocal:
arg = co.co_varnames[oparg]
elif op in hascompare:
arg = cmp_op[oparg]
elif op in hasfree:
if free is None:
free = co.co_cellvars + co.co_freevars
arg = free[oparg]
else:
arg = oparg
yield opname[op], arg
And here's an example disassembly.
>>> def f(x):
... return x + 1
...
>>> list(disassemble(f.func_code))
[('LOAD_FAST', 'x'), ('LOAD_CONST', 1), ('BINARY_ADD',), ('RETURN_VALUE',)]
You can easily transform this into the dictionary-of-sets data structure you want:
>>> from collections import defaultdict
>>> d = defaultdict(set)
>>> for op in disassemble(f.func_code):
... if len(op) == 2:
... d[op[0]].add(op[1])
...
>>> d
defaultdict(<type 'set'>, {'LOAD_FAST': set(['x']), 'LOAD_CONST': set([1])})
(Or you could generate the dictionary-of-sets data structure directly.)
Note that in your application you probably don't actually need look up the name for each opcode. Instead, you could look up the opcodes you need in the opcode.opmap dictionary and create named constants, perhaps like this:
LOAD_FAST = opmap['LOAD_FAST'] # actual value is 124
...
for var in disassembly[LOAD_FAST]:
...
Update: in Python 3.4 you can use the new dis.get_instructions:
>>> def f(x):
... return x + 1
>>> import dis
>>> list(dis.get_instructions(f))
[Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='x',
argrepr='x', offset=0, starts_line=1, is_jump_target=False),
Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=1,
argrepr='1', offset=3, starts_line=None, is_jump_target=False),
Instruction(opname='BINARY_ADD', opcode=23, arg=None, argval=None,
argrepr='', offset=6, starts_line=None, is_jump_target=False),
Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None,
argrepr='', offset=7, starts_line=None, is_jump_target=False)]

I think the challenge here is to capture the output of a dis rather than parsing the output and create a dictionary. The reason I will not cover the second part is, the format and the fields (key, value) of the dictionary is not mentioned and its trivial.
As I mentioned, the reason its a challenge to capture the OP of dis is, its a print rather than a return, but this can be captured through context manager
def foo(co):
import sys
from contextlib import contextmanager
from cStringIO import StringIO
#contextmanager
def captureStdOut(output):
stdout = sys.stdout
sys.stdout = output
yield
sys.stdout = stdout
out = StringIO()
with captureStdOut(out):
dis.disassemble(co.func_code)
return out.getvalue()
import dis
import re
dict(re.findall("^.*?([A-Z_]+)\s+(.*)$", line)[0] for line in foo(foo).splitlines()
if line.strip())
{'LOAD_CONST': '0 (None)', 'WITH_CLEANUP': '', 'SETUP_WITH': '21 (to 107)', 'STORE_DEREF': '0 (sys)', 'POP_TOP': '', 'LOAD_FAST': '4 (out)', 'MAKE_CLOSURE': '0', 'STORE_FAST': '4 (out)', 'IMPORT_FROM': '4 (StringIO)', 'LOAD_GLOBAL': '5 (dis)', 'END_FINALLY': '', 'RETURN_VALUE': '', 'LOAD_CLOSURE': '0 (sys)', 'BUILD_TUPLE': '1', 'CALL_FUNCTION': '0', 'LOAD_ATTR': '8 (getvalue)', 'IMPORT_NAME': '3 (cStringIO)', 'POP_BLOCK': ''}
>>>