We Keep Coding

OSError: The topology is loaded by filename extension, and the detected ".pd" format is not supported

I'm writing a code to generate features of ligands. Code is:
def parsePDB(self):
top = self.pdb.topology
residues = [str(residue)[:3] for residue in top.residues]
residues_cp = residues.copy()
# number of all groups in the complex except the ligand
LIG_number = residues.index('LIG')
self.residues_indices = [i for i in range(top.n_residues) if i != LIG_number]
# Get the types of atoms in the protein and ligand
self.rec_indices = top.select('protein')
self.lig_indices = top.select('resname LIG')
table, bond = top.to_dataframe()
self.all_ele = table['element']
self.lig_ele = table['element'][self.lig_indices]
H_num = []
for num, i in enumerate(self.all_ele):
if i == 'H':
H_num.append(num)
# Get the serial number of the atom in each residue or group
removes = []
for i in self.residues_indices:
atoms = top.residue(i).atoms
each_atoms = [j.index for j in atoms]
heavy_atoms = [x for x in each_atoms if not x in H_num]
if len(heavy_atoms) == 0:
removes.append(i)
else:
self.atoms_indices.append(heavy_atoms)
if len(removes) != 0:
for i in removes:
self.residues_indices.remove(i)
self.residues = [residues_cp[x] for x in self.residues_indices]
# Get the 3D coordinates for all atoms
self.xyz = self.pdb.xyz[0]
return self
I'm getting this error:
OSError: The topology is loaded by filename extension, and the detected ".pd" format is not supported. Supported topology formats include ".pdb", ".pdb.gz", ".h5", ".lh5", ".prmtop", ".parm7", ".prm7", ".psf", ".mol2", ".hoomdxml", ".gro", ".arc", ".hdf5" and ".gsd".
Can anyone please suggest me how to resolve this issue? Thanks

NoneType error while finding minima with Scipy Optimize

I am trying to find the global minima of a function using scipy.optimizer methods and keep running into NoneType issues. I have tried multiple algorithms including differential_evolution, shgo, and brute but keep running into errors.
Here is the setup:
def sizing_trade_study(ranges, payload):
with open("config.yml", "r") as yml:
cfg = yaml.load(yml)
first_int = True
km = []
for range in ranges:
km.append( range * 1000)
print(km)
params = (km, payload)
if first_int:
x0 = [float(cfg['design_variables']['initial_guess']['prop_radius']),
float(cfg['design_variables']['initial_guess']['speed']),
float(cfg['design_variables']['initial_guess']['battery_mass']),
float(cfg['design_variables']['initial_guess']['motor_mass']),
float(cfg['design_variables']['initial_guess']['mtow'])]
lb = [float(cfg['design_variables']['lower_bound']['prop_radius']),
float(cfg['design_variables']['lower_bound']['speed']),
float(cfg['design_variables']['lower_bound']['battery_mass']),
float(cfg['design_variables']['lower_bound']['motor_mass']),
float(cfg['design_variables']['lower_bound']['mtow'])] # Min cruise at 1.3 * VStall
ub = [float(cfg['design_variables']['upper_bound']['prop_radius']),
float(cfg['design_variables']['upper_bound']['speed']),
float(cfg['design_variables']['upper_bound']['battery_mass']),
float(cfg['design_variables']['upper_bound']['motor_mass']),
float(cfg['design_variables']['upper_bound']['mtow'])]
# bounds = (slice(lb[0], ub[0]), slice(lb[1], ub[1]), slice(lb[2], ub[2]), slice(lb[3], ub[3]), slice(lb[4], ub[4]))
# bounds = [(lb[0], ub[0]), (lb[1], ub[1]), (lb[2], ub[2]), (lb[3], ub[3]), (lb[4], ub[4])]
bounds = optimize.Bounds(lb,ub)
result = optimize.differential_evolution(objective_function, bounds, args=(params,))
print(result)
def objective_function(x, *params):
global trials
trials = trials+1
print(trials)
performance.compute_performance(x, params[0][0], params[0][1])
Here is the function I am trying to optimize:
import yaml
import simple_mission
import reserve_mission
import config_weight
def compute_performance(x, range, payload):
rprop = x[0]
speed = x[1]
battery = x[2]
motors = x[3]
mtow = x[4]
w = mtow * 9.8
with open("config.yml", "r") as yml:
cfg = yaml.load(yml)
bat_energy_density = int(cfg['performance']['bat_energy_density'])
motor_power_density = int(cfg['performance']['motor_power_density'])
discharge_depth = float(cfg['performance']['discharge_depth'])
e_nominal, flight_time, hover_output, cruise_output = simple_mission.run_simple_mission(rprop, speed, w, range)
reserve_e = reserve_mission.reserve_mission(rprop,speed, w, range)
mass = config_weight.config_weight(battery,motors, rprop, w, mtow, hover_output, cruise_output, payload)
batt = reserve_e - battery * bat_energy_density * discharge_depth / 1000
motor = hover_output.pow_hover / 1000 - motors * motor_power_density
weight = mass - w
return batt+ motor+ weight
The failure doesn't happen immediately but after a couple of runs of the optimizer function. For example, with differential_evolution, it always happens after the 75th trial.
Here is the stacktrace:
69
70
71
72
73
74
75
76
Traceback (most recent call last):
File "sizing_trade_study.py", line 62, in <module>
sizing_trade_study(args.ranges, args.payload)
File "sizing_trade_study.py", line 42, in sizing_trade_study
result = optimize.differential_evolution(objective_function, bounds, args=(params,))
File "/usr/local/anaconda3/envs/simple_mission/lib/python3.7/site-packages/scipy/optimize/_differentialevolution.py", line 308, in differential_evolution
ret = solver.solve()
File "/usr/local/anaconda3/envs/simple_mission/lib/python3.7/site-packages/scipy/optimize/_differentialevolution.py", line 759, in solve
next(self)
File "/usr/local/anaconda3/envs/simple_mission/lib/python3.7/site-packages/scipy/optimize/_differentialevolution.py", line 1082, in __next__
self.constraint_violation[candidate]):
File "/usr/local/anaconda3/envs/simple_mission/lib/python3.7/site-packages/scipy/optimize/_differentialevolution.py", line 1008, in _accept_trial
return energy_trial <= energy_orig
TypeError: '>=' not supported between instances of 'float' and 'NoneType'
Any help is greatly appreciated!

The issue is with one of the retrieved values from your bounds or objective_function, which in turn is being passed in as a NoneType to energy_orig within differential_evolution()
Source: https://github.com/scipy/scipy/blob/master/scipy/optimize/_differentialevolution.py
if feasible_orig and feasible_trial:
return energy_trial <= energy_orig
You should make sure that each key value is not empty from your config.yml or other function parameters. It's hard to tell which could be the problem. However, you could wrap it around a try/catch to get this to not stop on the 75th try for the meantime.
try:
result = optimize.differential_evolution(
objective_function,
bounds,
args=(params,),
)
except TypeError:
import pdb; pdb.set_trace()
I've set pdb, which will allow to to debug the values of each parameter, feel free to swap it out with a pass if you need to continue swiftly

TypeError: '<' not supported between instances of pyshipping.Package after running 2to3

Python newbie here, very good chance I am doing a silly mistake here..
After losing a good amount of hair and searching for many hours, I am still not able to convert a whole project to python 3. I have a project made in django framework and it uses python 3.7, and I wanted to incorporate this library into my app. But, because pyshipping uses python 2.7, I thought it may cause compatibility issues. Following this answer,I converted the whole project and tried running this file binpack_simple.py. But it gives me an error I am not able to understand at all. When I run this file using my pycharm terminal when the project iterpreter is set to python 2.7 it runs perfectly, but when I set iterpreter to 3.7, it gives me the following error
return _pyprofile._Utils(Profile).run(statement, filename, sort)
File "C:\Users\idadarklord\AppData\Local\Programs\Python\Python37\lib\profile.py", line 53, in run
prof.run(statement)
File "C:\Users\idadarklord\AppData\Local\Programs\Python\Python37\lib\cProfile.py", line 95, in run
return self.runctx(cmd, dict, dict)
File "C:\Users\idadarklord\AppData\Local\Programs\Python\Python37\lib\cProfile.py", line 100, in runctx
exec(cmd, globals, locals)
File "<string>", line 1, in <module>
File "C:/Users/idadarklord/PycharmProjects/untitled/pyshipping/binpack_simple.py", line 230, in test
bins, rest = binpack(packages)
File "C:/Users/idadarklord/PycharmProjects/untitled/pyshipping/binpack_simple.py", line 218, in binpack
return allpermutations(packages, bin, iterlimit)
File "C:/Users/idadarklord/PycharmProjects/untitled/pyshipping/binpack_simple.py", line 203, in allpermutations
trypack(bin, todo, bestpack)
File "C:/Users/idadarklord/PycharmProjects/untitled/pyshipping/binpack_simple.py", line 187, in trypack
bins, rest = packit(bin, packages)
File "C:/Users/idadarklord/PycharmProjects/untitled/pyshipping/binpack_simple.py", line 131, in packit
packages = sorted(originalpackages)
TypeError: '<' not supported between instances of 'Package' and 'Package'
Here is my file. Please let me know if I should upload the whole project for clarifications.
#!/usr/bin/env python
# encoding: utf-8
"""
binpack_simple.py
"""
from builtins import map
from builtins import range
from pyshipping.package import Package
from setuptools import setup, find_packages
from distutils.extension import Extension
import codecs
import time
import random
def packstrip(bin, p):
"""Creates a Strip which fits into bin.
Returns the Packages to be used in the strip, the dimensions of the strip as a 3-tuple
and a list of "left over" packages.
"""
# This code is somewhat optimized and somewhat unreadable
s = [] # strip
r = [] # rest
ss = sw = sl = 0 # stripsize
bs = bin.heigth # binsize
sapp = s.append # speedup
rapp = r.append # speedup
ppop = p.pop # speedup
while p and (ss <= bs):
n = ppop(0)
nh, nw, nl = n.size
if ss + nh <= bs:
ss += nh
sapp(n)
if nw > sw:
sw = nw
if nl > sl:
sl = nl
else:
rapp(n)
return s, (ss, sw, sl), r + p
def packlayer(bin, packages):
strips = []
layersize = 0
layerx = 0
layery = 0
binsize = bin.width
while packages:
strip, (sizex, stripsize, sizez), rest = packstrip(bin, packages)
if layersize + stripsize <= binsize:
packages = rest
if not strip:
# we were not able to pack anything
break
layersize += stripsize
layerx = max([sizex, layerx])
layery = max([sizez, layery])
strips.extend(strip)
else:
# Next Layer please
packages = strip + rest
break
return strips, (layerx, layersize, layery), packages
def packbin(bin, packages):
packages.sort()
layers = []
contentheigth = 0
contentx = 0
contenty = 0
binsize = bin.length
while packages:
layer, (sizex, sizey, layersize), rest = packlayer(bin, packages)
if contentheigth + layersize <= binsize:
packages = rest
if not layer:
# we were not able to pack anything
break
contentheigth += layersize
contentx = max([contentx, sizex])
contenty = max([contenty, sizey])
layers.extend(layer)
else:
# Next Bin please
packages = layer + rest
break
return layers, (contentx, contenty, contentheigth), packages
def packit(bin, originalpackages):
packedbins = []
packages = sorted(originalpackages)
while packages:
packagesinbin, (binx, biny, binz), rest = packbin(bin, packages)
if not packagesinbin:
# we were not able to pack anything
break
packedbins.append(packagesinbin)
packages = rest
# we now have a result, try to get a better result by rotating some bins
return packedbins, rest
# In newer Python versions these van be imported:
# from itertools import permutations
def product(*args, **kwds):
# product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
# product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
pools = list(map(tuple, args)) * kwds.get('repeat', 1)
result = [[]]
for pool in pools:
result = [x + [y] for x in result for y in pool]
for prod in result:
yield tuple(prod)
def permutations(iterable, r=None):
pool = tuple(iterable)
n = len(pool)
r = n if r is None else r
for indices in product(list(range(n)), repeat=r):
if len(set(indices)) == r:
yield tuple(pool[i] for i in indices)
class Timeout(Exception):
pass
def allpermutations_helper(permuted, todo, maxcounter, callback, bin, bestpack, counter):
if not todo:
return counter + callback(bin, permuted, bestpack)
else:
others = todo[1:]
thispackage = todo[0]
for dimensions in set(permutations((thispackage[0], thispackage[1], thispackage[2]))):
thispackage = Package(dimensions, nosort=True)
if thispackage in bin:
counter = allpermutations_helper(permuted + [thispackage], others, maxcounter, callback,
bin, bestpack, counter)
if counter > maxcounter:
raise Timeout('more than %d iterations tries' % counter)
return counter
def trypack(bin, packages, bestpack):
bins, rest = packit(bin, packages)
if len(bins) < bestpack['bincount']:
bestpack['bincount'] = len(bins)
bestpack['bins'] = bins
bestpack['rest'] = rest
if bestpack['bincount'] < 2:
raise Timeout('optimal solution found')
return len(packages)
def allpermutations(todo, bin, iterlimit=5000):
random.seed(1)
random.shuffle(todo)
bestpack = dict(bincount=len(todo) + 1)
try:
# First try unpermuted
trypack(bin, todo, bestpack)
# now try permutations
allpermutations_helper([], todo, iterlimit, trypack, bin, bestpack, 0)
except Timeout:
pass
return bestpack['bins'], bestpack['rest']
def binpack(packages, bin=None, iterlimit=5000):
"""Packs a list of Package() objects into a number of equal-sized bins.
Returns a list of bins listing the packages within the bins and a list of packages which can't be
packed because they are to big."""
if not bin:
bin = Package("600x400x400")
return allpermutations(packages, bin, iterlimit)
def test():
fd = open('small.txt')
vorher = 0
nachher = 0
start = time.time()
for line in fd:
packages = [Package(pack) for pack in line.strip().split()]
if not packages:
continue
bins, rest = binpack(packages)
if rest:
print(("invalid data", rest, line))
else:
vorher += len(packages)
nachher += len(bins)
# print((time.time() - start))
print((vorher, nachher, float(nachher) / vorher * 100))
#
if __name__ == '__main__':
import cProfile
cProfile.run('test()')
# packlayer(bin, packages)
Here is an online link to the file inside the project.

In Python 3 support for the __cmp__ method has been removed. You need to provide a __lt__ method for the class instead if you want to compare two instances. The code for the original Package.__cmp__ is here.
The new method will probably look like:
def __lt__(self, other):
return self.volume < other.volume
but obviously you should test this thoroughly.

You need to set "volume" as your sorting key, meaning:
Change:
packages.sort()
packages = sorted(originalpackages)
To:
packages.sort(key=lambda x: x.volume)
packages = sorted(originalpackages, key=lambda x: x.volume)
Thus, you won't need to change any package inner code manually.

NLTK output changes when using the same input on the same machine

The next piece of code returns different output with the same input(self.SynSets)
Why can it be happening? Am I doing something wrong? or is it caused by python?
def FilterSynSets(self):
self.filteredSysNets = {}
for synset in self.SysNets:
for subsynset in self.SysNets:
length = wn.path_similarity(synset,subsynset)
if not length is None\
and length !=1\
and length >0:
target = synset.__str__().replace("'","")
source =subsynset.__str__().replace("'","")
connection="\"{}\"->\"{}\" [label={}]".format(
target,
source,
str(round(length,3)))
self.filteredSysNets[connection] = length
oldLength = len(self.filteredSysNets)
avarageVal = sum(self.filteredSysNets.values())/len(self.filteredSysNets)
self.filteredSysNets = {k: v for k,v in self.filteredSysNets.items() if v>=avarageVal}
newLength = len(self.filteredSysNets)
prt = newLength/oldLength*100
print ("avr -{}\n old -{}\n new - {}\n prec={}".format(avarageVal,oldLength,newLength,prt))
return self
Outputs:
http://screencast.com/t/eFMOROfkPXR
http://screencast.com/t/Fdd6ufhA

Codec Errors in Python

Does anyone know the name of a codec that can translate any random assortment of bytes into a string? I have been getting the following error after encoding, encrypting, and decoding a string in tkinter.Text.
UnicodeDecodeError: 'utf8' codec can't decode
byte 0x99 in position 151: unexpected code byte
Code used to generate the error follow below. The UTF8 codec listed at the top has problems translating some bytes back into a string. What I am looking for is an answer that solves the problem, not direction.
from tkinter import *
import traceback
from tkinter.scrolledtext import ScrolledText
CODEC = 'utf8'
################################################################################
class MarkovDemo:
def __init__(self, master):
self.prompt_size = Label(master, anchor=W, text='Encode Word Size')
self.prompt_size.pack(side=TOP, fill=X)
self.size_entry = Entry(master)
self.size_entry.insert(0, '8')
self.size_entry.pack(fill=X)
self.prompt_plain = Label(master, anchor=W, text='Plaintext Characters')
self.prompt_plain.pack(side=TOP, fill=X)
self.plain_entry = Entry(master)
self.plain_entry.insert(0, '""')
self.plain_entry.pack(fill=X)
self.showframe = Frame(master)
self.showframe.pack(fill=X, anchor=W)
self.showvar = StringVar(master)
self.showvar.set("encode")
self.showfirstradio = Radiobutton(self.showframe,
text="Encode Plaintext",
variable=self.showvar,
value="encode",
command=self.reevaluate)
self.showfirstradio.pack(side=LEFT)
self.showallradio = Radiobutton(self.showframe,
text="Decode Cyphertext",
variable=self.showvar,
value="decode",
command=self.reevaluate)
self.showallradio.pack(side=LEFT)
self.inputbox = ScrolledText(master, width=60, height=10, wrap=WORD)
self.inputbox.pack(fill=BOTH, expand=1)
self.dynamic_var = IntVar()
self.dynamic_box = Checkbutton(master, variable=self.dynamic_var,
text='Dynamic Evaluation',
offvalue=False, onvalue=True,
command=self.reevaluate)
self.dynamic_box.pack()
self.output = Label(master, anchor=W, text="This is your output:")
self.output.pack(fill=X)
self.outbox = ScrolledText(master, width=60, height=10, wrap=WORD)
self.outbox.pack(fill=BOTH, expand=1)
self.inputbox.bind('<Key>', self.reevaluate)
def select_all(event=None):
event.widget.tag_add(SEL, 1.0, 'end-1c')
event.widget.mark_set(INSERT, 1.0)
event.widget.see(INSERT)
return 'break'
self.inputbox.bind('<Control-Key-a>', select_all)
self.outbox.bind('<Control-Key-a>', select_all)
self.inputbox.bind('<Control-Key-/>', lambda event: 'break')
self.outbox.bind('<Control-Key-/>', lambda event: 'break')
self.outbox.config(state=DISABLED)
def reevaluate(self, event=None):
if event is not None:
if event.char == '':
return
if self.dynamic_var.get():
text = self.inputbox.get(1.0, END)[:-1]
if len(text) < 10:
return
text = text.replace('\n \n', '\n\n')
mode = self.showvar.get()
assert mode in ('decode', 'encode'), 'Bad mode!'
if mode == 'encode':
# Encode Plaintext
try:
# Evaluate the plaintext characters
plain = self.plain_entry.get()
if plain:
PC = eval(self.plain_entry.get())
else:
PC = ''
self.plain_entry.delete(0, END)
self.plain_entry.insert(0, '""')
# Evaluate the word size
size = self.size_entry.get()
if size:
XD = int(size)
while grid_size(text, XD, PC) > 1 << 20:
XD -= 1
else:
XD = 0
grid = 0
while grid <= 1 << 20:
grid = grid_size(text, XD, PC)
XD += 1
XD -= 1
# Correct the size and encode
self.size_entry.delete(0, END)
self.size_entry.insert(0, str(XD))
cyphertext, key, prime = encrypt_str(text, XD, PC)
except:
traceback.print_exc()
else:
buffer = ''
for block in key:
buffer += repr(block)[2:-1] + '\n'
buffer += repr(prime)[2:-1] + '\n\n' + cyphertext
self.outbox.config(state=NORMAL)
self.outbox.delete(1.0, END)
self.outbox.insert(END, buffer)
self.outbox.config(state=DISABLED)
else:
# Decode Cyphertext
try:
header, cypher = text.split('\n\n', 1)
lines = header.split('\n')
for index, item in enumerate(lines):
try:
lines[index] = eval('b"' + item + '"')
except:
lines[index] = eval("b'" + item + "'")
plain = decrypt_str(cypher, tuple(lines[:-1]), lines[-1])
except:
traceback.print_exc()
else:
self.outbox.config(state=NORMAL)
self.outbox.delete(1.0, END)
self.outbox.insert(END, plain)
self.outbox.config(state=DISABLED)
else:
text = self.inputbox.get(1.0, END)[:-1]
text = text.replace('\n \n', '\n\n')
mode = self.showvar.get()
assert mode in ('decode', 'encode'), 'Bad mode!'
if mode == 'encode':
try:
XD = int(self.size_entry.get())
PC = eval(self.plain_entry.get())
size = grid_size(text, XD, PC)
assert size
except:
pass
else:
buffer = 'Grid size will be:\n' + convert(size)
self.outbox.config(state=NORMAL)
self.outbox.delete(1.0, END)
self.outbox.insert(END, buffer)
self.outbox.config(state=DISABLED)
################################################################################
import random
CRYPT = random.SystemRandom()
################################################################################
# This section includes functions that
# can test the required key and bootstrap.
# sudoko_key
# - should be a proper "markov" key
def _check_sudoku_key(sudoku_key):
# Ensure key is a tuple with more than one item.
assert isinstance(sudoku_key, tuple), '"sudoku_key" must be a tuple'
assert len(sudoku_key) > 1, '"sudoku_key" must have more than one item'
# Test first item.
item = sudoku_key[0]
assert isinstance(item, bytes), 'first item must be an instance of bytes'
assert len(item) > 1, 'first item must have more than one byte'
assert len(item) == len(set(item)), 'first item must have unique bytes'
# Test the rest of the key.
for obj in sudoku_key[1:]:
assert isinstance(obj, bytes), 'remaining items must be of bytes'
assert len(obj) == len(item), 'all items must have the same length'
assert len(obj) == len(set(obj)), \
'remaining items must have unique bytes'
assert len(set(item)) == len(set(item).union(set(obj))), \
'all items must have the same bytes'
# boot_strap
# - should be a proper "markov" bootstrap
# - we will call this a "primer"
# sudoko_key
# - should be a proper "markov" key
def _check_boot_strap(boot_strap, sudoku_key):
assert isinstance(boot_strap, bytes), '"boot_strap" must be a bytes object'
assert len(boot_strap) == len(sudoku_key) - 1, \
'"boot_strap" length must be one less than "sudoku_key" length'
item = sudoku_key[0]
assert len(set(item)) == len(set(item).union(set(boot_strap))), \
'"boot_strap" may only have bytes found in "sudoku_key"'
################################################################################
# This section includes functions capable
# of creating the required key and bootstrap.
# bytes_set should be any collection of bytes
# - it should be possible to create a set from them
# - these should be the bytes on which encryption will follow
# word_size
# - this will be the size of the "markov" chains program uses
# - this will be the number of dimensions the "grid" will have
# - one less character will make up bootstrap (or primer)
def make_sudoku_key(bytes_set, word_size):
key_set = set(bytes_set)
blocks = []
for block in range(word_size):
blocks.append(bytes(CRYPT.sample(key_set, len(key_set))))
return tuple(blocks)
# sudoko_key
# - should be a proper "markov" key
def make_boot_strap(sudoku_key):
block = sudoku_key[0]
return bytes(CRYPT.choice(block) for byte in range(len(sudoku_key) - 1))
################################################################################
# This section contains functions needed to
# create the multidimensional encryption grid.
# sudoko_key
# - should be a proper "markov" key
def make_grid(sudoku_key):
grid = expand_array(sudoku_key[0], sudoku_key[1])
for block in sudoku_key[2:]:
grid = expand_array(grid, block)
return grid
# grid
# - should be an X dimensional grid from make_grid
# block_size
# - comes from length of one block in a sudoku_key
def make_decode_grid(grid, block_size):
cache = []
for part in range(0, len(grid), block_size):
old = grid[part:part+block_size]
new = [None] * block_size
key = sorted(old)
for index, byte in enumerate(old):
new[key.index(byte)] = key[index]
cache.append(bytes(new))
return b''.join(cache)
# grid
# - should be an X dimensional grid from make_grid
# block
# - should be a block from a sudoku_key
# - should have same unique bytes as the expanding grid
def expand_array(grid, block):
cache = []
grid_size = len(grid)
block_size = len(block)
for byte in block:
index = grid.index(bytes([byte]))
for part in range(0, grid_size, block_size):
cache.append(grid[part+index:part+block_size])
cache.append(grid[part:part+index])
return b''.join(cache)
################################################################################
# The first three functions can be used to check an encryption
# grid. The eval_index function is used to evaluate a grid cell.
# grid
# - grid object to be checked
# - grid should come from the make_grid function
# - must have unique bytes along each axis
# block_size
# - comes from length of one block in a sudoku_key
# - this is the length of one edge along the grid
# - each axis is this many unit long exactly
# word_size
# - this is the number of blocks in a sudoku_key
# - this is the number of dimensions in a grid
# - this is the length needed to create a needed markon chain
def check_grid(grid, block_size, word_size):
build_index(grid, block_size, word_size, [])
# create an index to access the grid with
def build_index(grid, block_size, word_size, index):
for number in range(block_size):
index.append(number)
if len(index) == word_size:
check_cell(grid, block_size, word_size, index)
else:
build_index(grid, block_size, word_size, index)
index.pop()
# compares the contents of a cell along each grid axis
def check_cell(grid, block_size, word_size, index):
master = eval_index(grid, block_size, index)
for axis in range(word_size):
for value in range(block_size):
if index[axis] != value:
copy = list(index)
copy[axis] = value
slave = eval_index(grid, block_size, copy)
assert slave != master, 'Cell not unique along axis!'
# grid
# - grid object to be accessed and evaluated
# - grid should come from the make_grid function
# - must have unique bytes along each axis
# block_size
# - comes from length of one block in a sudoku_key
# - this is the length of one edge along the grid
# - each axis is this many unit long exactly
# index
# - list of coordinates to access the grid
# - should be of length word_size
# - should be of length equal to number of dimensions in the grid
def eval_index(grid, block_size, index):
offset = 0
for power, value in enumerate(reversed(index)):
offset += value * block_size ** power
return grid[int(offset)]
################################################################################
# The following functions act as a suite that can ultimately
# encrpyt strings, though other functions can be built from them.
# bytes_obj
# - the bytes to encode
# byte_map
# - byte tranform map for inserting into the index
# grid
# - X dimensional grid used to evaluate markov chains
# index
# - list that starts the index for accessing grid (primer)
# - it should be of length word_size - 1
# block_size
# - length of each edge in a grid
def _encode(bytes_obj, byte_map, grid, index, block_size):
cache = bytes()
index = [0] + index
for byte in bytes_obj:
if byte in byte_map:
index.append(byte_map[byte])
index = index[1:]
cache += bytes([eval_index(grid, block_size, index)])
else:
cache += bytes([byte])
return cache, index[1:]
# bytes_obj
# - the bytes to encode
# sudoko_key
# - should be a proper "markov" key
# - this key will be automatically checked for correctness
# boot_strap
# - should be a proper "markov" bootstrap
def encrypt(bytes_obj, sudoku_key, boot_strap):
_check_sudoku_key(sudoku_key)
_check_boot_strap(boot_strap, sudoku_key)
# make byte_map
array = sorted(sudoku_key[0])
byte_map = dict((byte, value) for value, byte in enumerate(array))
# create two more arguments for encode
grid = make_grid(sudoku_key)
index = list(map(byte_map.__getitem__, boot_strap))
# run the actual encoding algorithm and create reversed map
code, index = _encode(bytes_obj, byte_map, grid, index, len(sudoku_key[0]))
rev_map = dict(reversed(item) for item in byte_map.items())
# fix the boot_strap and return the results
boot_strap = bytes(rev_map[number] for number in index)
return code, boot_strap
# string
# - should be the string that you want encoded
# word_size
# - length you want the markov chains to be of
# plain_chars
# - characters that you do not want to encrypt
def encrypt_str(string, word_size, plain_chars=''):
byte_obj = string.encode(CODEC)
encode_on = set(byte_obj).difference(set(plain_chars.encode()))
sudoku_key = make_sudoku_key(encode_on, word_size)
boot_strap = make_boot_strap(sudoku_key)
cyphertext = encrypt(byte_obj, sudoku_key, boot_strap)[0]
# return encrypted string, key, and original bootstrap
return cyphertext.decode(CODEC), sudoku_key, boot_strap
def grid_size(string, word_size, plain_chars):
encode_on = set(string.encode()).difference(set(plain_chars.encode()))
return len(encode_on) ** word_size
################################################################################
# The following functions act as a suite that can ultimately
# decrpyt strings, though other functions can be built from them.
# bytes_obj
# - the bytes to encode
# byte_map
# - byte tranform map for inserting into the index
# grid
# - X dimensional grid used to evaluate markov chains
# index
# - list that starts the index for accessing grid (primer)
# - it should be of length word_size - 1
# block_size
# - length of each edge in a grid
def _decode(bytes_obj, byte_map, grid, index, block_size):
cache = bytes()
index = [0] + index
for byte in bytes_obj:
if byte in byte_map:
index.append(byte_map[byte])
index = index[1:]
decoded = eval_index(grid, block_size, index)
index[-1] = byte_map[decoded]
cache += bytes([decoded])
else:
cache += bytes([byte])
return cache, index[1:]
# bytes_obj
# - the bytes to decode
# sudoko_key
# - should be a proper "markov" key
# - this key will be automatically checked for correctness
# boot_strap
# - should be a proper "markov" bootstrap
def decrypt(bytes_obj, sudoku_key, boot_strap):
_check_sudoku_key(sudoku_key)
_check_boot_strap(boot_strap, sudoku_key)
# make byte_map
array = sorted(sudoku_key[0])
byte_map = dict((byte, value) for value, byte in enumerate(array))
# create two more arguments for decode
grid = make_grid(sudoku_key)
grid = make_decode_grid(grid, len(sudoku_key[0]))
index = list(map(byte_map.__getitem__, boot_strap))
# run the actual decoding algorithm and create reversed map
code, index = _decode(bytes_obj, byte_map, grid, index, len(sudoku_key[0]))
rev_map = dict(reversed(item) for item in byte_map.items())
# fix the boot_strap and return the results
boot_strap = bytes(rev_map[number] for number in index)
return code, boot_strap
# string
# - should be the string that you want decoded
# word_size
# - length you want the markov chains to be of
# plain_chars
# - characters that you do not want to encrypt
def decrypt_str(string, sudoku_key, boot_strap):
byte_obj = string.encode(CODEC)
plaintext = decrypt(byte_obj, sudoku_key, boot_strap)[0]
# return encrypted string, key, and original bootstrap
return plaintext.decode(CODEC)
################################################################################
def convert(number):
"Convert bytes into human-readable representation."
assert 0 < number < 1 << 110, 'Number Out Of Range'
ordered = reversed(tuple(format_bytes(partition_number(number, 1 << 10))))
cleaned = ', '.join(item for item in ordered if item[0] != '0')
return cleaned
################################################################################
def partition_number(number, base):
"Continually divide number by base until zero."
div, mod = divmod(number, base)
yield mod
while div:
div, mod = divmod(div, base)
yield mod
def format_bytes(parts):
"Format partitioned bytes into human-readable strings."
for power, number in enumerate(parts):
yield '{} {}'.format(number, format_suffix(power, number))
def format_suffix(power, number):
"Compute the suffix for a certain power of bytes."
return (PREFIX[power] + 'byte').capitalize() + ('s' if number != 1 else '')
################################################################################
PREFIX = ' kilo mega giga tera peta exa zetta yotta bronto geop'.split(' ')
################################################################################
if __name__ == '__main__':
root = Tk()
root.title('Markov Demo')
demo = MarkovDemo(root)
root.mainloop()

Strings are by definition a sequence of bytes that only have meaning when interpreted with the knowledge of the encoding. That's one reason why the equivalent of Python 2's string type in Python 3 is the bytes type. As long as you know the encoding of the strings you're working with, I'm not sure you specifically need to recode it just to compress/encrypt it. Details of what you're actually doing might make a difference, though.

Python's decode has error settings. The default is strict which throws an exception.
Wherever you are doing the decoding, you can specify 'ignore' or 'replace' as a setting, and this will take care of your problems.
Please see the codecs documentation.

In Python HOWTOs from the Python v3.1.1 documentation, there is a helpful section regarding Unicode HOWTO. The table of content contains an entry to Python’s Unicode Support that explains string & byte.
The String Type
>>> b'\x80abc'.decode("utf-8", "strict")
Traceback (most recent call last):
File "<stdin>", line 1, in ?
UnicodeDecodeError: 'utf8' codec can't decode byte 0x80 in position 0:
unexpected code byte
>>> b'\x80abc'.decode("utf-8", "replace")
'\ufffdabc'
>>> b'\x80abc'.decode("utf-8", "ignore")
'abc'
Converting to Bytes
>>> u = chr(40960) + 'abcd' + chr(1972)
>>> u.encode('utf-8')
b'\xea\x80\x80abcd\xde\xb4'
>>> u.encode('ascii')
Traceback (most recent call last):
File "<stdin>", line 1, in ?
UnicodeEncodeError: 'ascii' codec can't encode character '\ua000' in
position 0: ordinal not in range(128)
>>> u.encode('ascii', 'ignore')
b'abcd'
>>> u.encode('ascii', 'replace')
b'?abcd?'
>>> u.encode('ascii', 'xmlcharrefreplace')
b'ꀀabcd޴'
One possible solution to the problem listed above involves covert all occurrences of
.encode(CODEC) with .encode(CODEC, 'ignore'). Likewise, all .decode(CODEC) become .decode(CODEC, 'ignore').