I try to run this Neural Network script (for a regression model)
There are two classes defined above. One is Standardizer class and other is Neural Net class. The Standardizer class normalizes all the values and the NeuralNet class builds the neural network that learns the data through feed forward and back propagation.
This function takes the the number of inputs, hidden units, and outputs as the three parameters.
The set_hunit function is used to either update or initiate the weights.It takes the weight as the parameter.
The Pack function packs the multiple weights of each layer into one vector. The unpack function does vice versa.
Forward pass in neural network propagates as shown below:
ππ=β(ππβ
π)=ππβ
π
Activation function is used to make the network non linear. We may use tanh or RBG or etc.
In the backward pass the function takes the the z values, Target values and the error as input. Based on the delta value, the weights and the bias are updated accoringly. This method returns the weight vector packed together of that particualr layer. Below are the functions that are excecuted during backward pass.
ππβπ+πΌβ1π1πΎππβ€((πβπ)πβ€β(1βπ2))βπ+πΌπ1π1πΎππβ€(πβπ)
The train function takes the feautures and the target as the input. The gradientf unpacks the weights,proceeds with the forward pass by calling forward function. Now error is calculated using results of forward pass. Now back propagation is proceeded by calling backward function with parameters as error, Z, T(Target), _lambda.
The optimtarget function tries to reduce the error by using the object function and updates the weights accordingly.
The use method is applied to the test data after training the model. Testing data is passed as parameter and it stadardizes the data. Then forward is applied on the data which returns the predictions
This shows module not found error, but I have installed grad module with pip installation
#Importing required libraries
import pandas as pd
import numpy as np
import seaborn as sns
import grad
import matplotlib.pyplot as plt
# Reading data using pandas library
vehicle_data=pd.read_csv('processed_Data.csv')
# Overall idea about distribution of data
vehicle_data.hist(bins=40, figsize=(20,15))
plt.show()
# Count plot of Ellectric Range
sns.countplot(x='Electric Range',data=vehicle_data)
# Joint plot between Latitude on x axis and Longitude on y axis
sns.jointplot(x=vehicle_data.BaseMSRP.values,y=vehicle_data.LegislativeDistrict.values,height=10)
plt.xlabel("Base MSRP",fontsize=10)
plt.ylabel("Lengislative District",fontsize=10)
# function to drop the rows that has null or missing values
vehicle_data=vehicle_data.dropna()
# Data is already clean and has no missing values
vehicle_data.shape
#Dropping unwanted columns
vehicle_data=vehicle_data.drop(['VIN (1-10)','County', 'City', 'State', 'ZIP Code', 'DOL Vehicle ID'],axis=1)
vehicle_data.shape
# Seperating target variable
t=pd.DataFrame(vehicle_data.iloc[:,8])
vehicle_data=vehicle_data.drop(['Electric Range'],axis=1)
t
vehicle_data.head()
#NeuralNet class for regression
# standardization class
class Standardizer:
""" class version of standardization """
def __init__(self, X, explore=False):
self._mu = np.mean(X,8)
self._sigma = np.std(X,8)
if explore:
print ("mean: ", self._mu)
print ("sigma: ", self._sigma)
print ("min: ", np.min(X,8))
print ("max: ", np.max(X,8))
def set_sigma(self, s):
self._sigma[:] = s
def standardize(self,X):
return (X - self._mu) / self._sigma
def unstandardize(self,X):
return (X * self._sigma) + self._mu
def add_ones(w):
return np.hstack((np.ones((w.shape[8], 1)), w))
from grad import scg, steepest
from copy import copy
class NeuralNet:
def __init__(self, nunits):
self._nLayers=len(nunits)-1
self.rho = [1] * self._nLayers
self._W = []
wdims = []
lenweights = 0
for i in range(self._nLayers):
nwr = nunits[i] + 1
nwc = nunits[i+1]
wdims.append((nwr, nwc))
lenweights = lenweights + nwr * nwc
self._weights = np.random.uniform(-0.1,0.1, lenweights)
start = 0 # fixed index error 20110107
for i in range(self._nLayers):
end = start + wdims[i][0] * wdims[i][1]
self._W.append(self._weights[start:end])
self._W[i].resize(wdims[i])
start = end
self.stdX = None
self.stdT = None
self.stdTarget = True
def add_ones(self, w):
return np.hstack((np.ones((w.shape[8], 1)), w))
def get_nlayers(self):
return self._nLayers
def set_hunit(self, w):
for i in range(self._nLayers-1):
if w[i].shape != self._W[i].shape:
print("set_hunit: shapes do not match!")
break
else:
self._W[i][:] = w[i][:]
def pack(self, w):
return np.hstack(map(np.ravel, w))
def unpack(self, weights):
self._weights[:] = weights[:] # unpack
def cp_weight(self):
return copy(self._weights)
def RBF(self, X, m=None,s=None):
if m is None: m = np.mean(X)
if s is None: s = 2 #np.std(X)
r = 1. / (np.sqrt(2*np.pi)* s)
return r * np.exp(-(X - m) ** 2 / (2 * s ** 2))
def forward(self,X):
t = X
Z = []
for i in range(self._nLayers):
Z.append(t)
if i == self._nLayers - 1:
t = np.dot(self.add_ones(t), self._W[i])
else:
t = np.tanh(np.dot(self.add_ones(t), self._W[i]))
#t = self.RBF(np.dot(np.hstack((np.ones((t.shape[0],1)),t)),self._W[i]))
return (t, Z)
def backward(self, error, Z, T, lmb=0):
delta = error
N = T.size
dws = []
for i in range(self._nLayers - 1, -1, -1):
rh = float(self.rho[i]) / N
if i==0:
lmbterm = 0
else:
lmbterm = lmb * np.vstack((np.zeros((1, self._W[i].shape[1])),
self._W[i][1:,]))
dws.insert(0,(-rh * np.dot(self.add_ones(Z[i]).T, delta) + lmbterm))
if i != 0:
delta = np.dot(delta, self._W[i][1:, :].T) * (1 - Z[i]**2)
return self.pack(dws)
def _errorf(self, T, Y):
return T - Y
def _objectf(self, T, Y, wpenalty):
return 0.5 * np.mean(np.square(T - Y)) + wpenalty
def train(self, X, T, **params):
verbose = params.pop('verbose', False)
# training parameters
_lambda = params.pop('Lambda', 0.)
#parameters for scg
niter = params.pop('niter', 1000)
wprecision = params.pop('wprecision', 1e-10)
fprecision = params.pop('fprecision', 1e-10)
wtracep = params.pop('wtracep', False)
ftracep = params.pop('ftracep', False)
# optimization
optim = params.pop('optim', 'scg')
if self.stdX == None:
explore = params.pop('explore', False)
self.stdX = Standardizer(X, explore)
Xs = self.stdX.standardize(X)
if self.stdT == None and self.stdTarget:
self.stdT = Standardizer(T)
T = self.stdT.standardize(T)
def gradientf(weights):
self.unpack(weights)
Y,Z = self.forward(Xs)
error = self._errorf(T, Y)
return self.backward(error, Z, T, _lambda)
def optimtargetf(weights):
""" optimization target function : MSE
"""
self.unpack(weights)
#self._weights[:] = weights[:] # unpack
Y,_ = self.forward(Xs)
Wnb=np.array([])
for i in range(self._nLayers):
if len(Wnb)==0: Wnb=self._W[i][1:,].reshape(self._W[i].size-self._W[i][0,].size,1)
else: Wnb = np.vstack((Wnb,self._W[i][1:,].reshape(self._W[i].size-self._W[i][0,].size,1)))
wpenalty = _lambda * np.dot(Wnb.flat ,Wnb.flat)
return self._objectf(T, Y, wpenalty)
if optim == 'scg':
result = scg(self.cp_weight(), gradientf, optimtargetf,
wPrecision=wprecision, fPrecision=fprecision,
nIterations=niter,
wtracep=wtracep, ftracep=ftracep,
verbose=False)
self.unpack(result['w'][:])
self.f = result['f']
elif optim == 'steepest':
result = steepest(self.cp_weight(), gradientf, optimtargetf,
nIterations=niter,
xPrecision=wprecision, fPrecision=fprecision,
xtracep=wtracep, ftracep=ftracep )
self.unpack(result['w'][:])
if ftracep:
self.ftrace = result['ftrace']
if 'reason' in result.keys() and verbose:
print(result['reason'])
return result
def use(self, X, retZ=False):
if self.stdX:
Xs = self.stdX.standardize(X)
else:
Xs = X
Y, Z = self.forward(Xs)
if self.stdT is not None:
Y = self.stdT.unstandardize(Y)
if retZ:
return Y, Z
return Y
Try to open command prompt and type pip install grad or if you using jupyter notebook, make a new code shell and type !pip install grad before you importing it
Hope that solves your problem
In Mathematica I can convert multivariable moments in cumulants and back using MomentConvert:
MomentConvert[Cumulant[{2, 2,1}], "Moment"] // TraditionalForm
as one can try in wolframcloud.
I would like to do exactly the same in python. Is there any library in python capable of this?
At least the one direction I now programmed by myself:
# from http://code.activestate.com/recipes/577211-generate-the-partitions-of-a-set-by-index/
from collections import defaultdict
class Partition:
def __init__(self, S):
self.data = list(S)
self.m = len(S)
self.table = self.rgf_table()
def __getitem__(self, i):
#generates set partitions by index
if i > len(self) - 1:
raise IndexError
L = self.unrank_rgf(i)
result = self.as_set_partition(L)
return result
def __len__(self):
return self.table[self.m,0]
def as_set_partition(self, L):
# Transform a restricted growth function into a partition
n = max(L[1:]+[1])
m = self.m
data = self.data
P = [[] for _ in range(n)]
for i in range(m):
P[L[i+1]-1].append(data[i])
return P
def rgf_table(self):
# Compute the table values
m = self.m
D = defaultdict(lambda:1)
for i in range(1,m+1):
for j in range(0,m-i+1):
D[i,j] = j * D[i-1,j] + D[i-1,j+1]
return D
def unrank_rgf(self, r):
# Unrank a restricted growth function
m = self.m
L = [1 for _ in range(m+1)]
j = 1
D = self.table
for i in range(2,m+1):
v = D[m-i,j]
cr = j*v
if cr <= r:
L[i] = j + 1
r -= cr
j += 1
else:
L[i] = r // v + 1
r %= v
return L
# S = set(range(4))
# P = Partition(S)
# for x in P:
# print (x)
# using https://en.wikipedia.org/wiki/Cumulant#Joint_cumulants
import math
def Cum2Mom(arr, state):
def E(op):
return qu.expect(op, state)
def Arr2str(arr,sep):
r = ''
for i,x in enumerate(arr):
r += str(x)
if i<len(arr)-1:
r += sep
return r
if isinstance( arr[0],str):
myprod = lambda x: Arr2str(x,'*')
mysum = lambda x: Arr2str(x,'+')
E=lambda x: 'E('+str(x)+')'
myfloat = str
else:
myfloat = lambda x: x
myprod = np.prod
mysum = sum
S = set(range(len(arr)))
P = Partition(S)
return mysum([
myprod([myfloat(math.factorial(len(pi)-1) * (-1)**(len(pi)-1))
,myprod([
E(myprod([
arr[i]
for i in B
]))
for B in pi])])
for pi in P])
print(Cum2Mom(['a','b','c','d'],1) )
import qutip as qu
print(Cum2Mom([qu.qeye(3) for i in range(3)],qu.qeye(3)) )
It's designed to work with qutip opjects and it also works with strings to verify the correct separation and prefactors.
Exponents of the variables can be represented by repeating the variable.
First of all I must say my knowledge with using Sage math is really very limited, but I really want to improve and to be able to solve these problems I'm having. I have been asked to implement the following:
Use an Sage implementation of the cryptosystem NTRU and the library βcryptographyβ to build a KEM/DEM system with security of 128 bits, generated key of 128 bits and, on the DEM phase, use the cipher AES of 128 bits.
While trying to solve I came across an implementation of NTRU-Prime in sage and wanted to use it to solve this problem:
My Attemp:
p = 739; q = 9829; t = 204
Zx.<x> = ZZ[]; R.<xp> = Zx.quotient(x^p-x-1)
Fq = GF(q); Fqx.<xq> = Fq[]; Rq.<xqp> = Fqx.quotient(x^p-x-1)
F3 = GF(3); F3x.<x3> = F3[]; R3.<x3p> = F3x.quotient(x^p-x-1)
import itertools
def concat(lists):
return list(itertools.chain.from_iterable(lists))
def nicelift(u):
return lift(u + q//2) - q//2
def nicemod3(u): # r in {0,1,-1} with u-r in {...,-3,0,3,...}
return u - 3*round(u/3)
def int2str(u,bytes):
return ''.join([chr((u//256^i)%256) for i in range(bytes)])
def str2int(s):
return sum([ord(s[i])*256^i for i in range(len(s))])
def encodeZx(m): # assumes coefficients in range {-1,0,1,2}
m = [m[i]+1 for i in range(p)] + [0]*(-p % 4)
return ''.join([int2str(m[i]+m[i+1]*4+m[i+2]*16+m[i+3]*64,1) for i in range(0,len(m),4)])
def decodeZx(mstr):
m = [str2int(mstr[i:i+1]) for i in range(len(mstr))]
m = concat([[m[i]%4,(m[i]//4)%4,(m[i]//16)%4,m[i]//64] for i in range(len(m))])
return Zx([m[i]-1 for i in range(p)])
def encodeRq(h, nBits = 9856):
h = [lift(h[i]) for i in range(p)] + [0]*(-p % 3)
h = ''.join([int2str(h[i]+h[i+1]*10240+h[i+2]*10240^2,5) for i in range(0,len(h),3)])
return h[0:(nBits/8)]
def decodeRq(hstr):
h = [str2int(hstr[i:i+5]) for i in range(0,len(hstr),5)]
h = concat([[h[i]%10240,(h[i]//10240)%10240,h[i]//10240^2] for i in range(len(h))])
if max(h) >= q: raise Exception("pk out of range")
return Rq(h)
def encoderoundedRq(c,nBits):
c = [1638 + nicelift(c[i]/3) for i in range(p)] + [0]*(-p % 2)
c = ''.join([int2str(c[i]+c[i+1]*4096,3) for i in range(0,len(c),2)])
return c[0:1109]
def decoderoundedRq(cstr):
c = [str2int(cstr[i:i+3]) for i in range(0,len(cstr),3)]
c = concat([[c[i]%4096,c[i]//4096] for i in range(len(c))])
if max(c) > 3276: raise Exception("c out of range")
return 3*Rq([c[i]-1638 for i in range(p)])
def randomR(): # R element with 2t coeffs +-1
L = [2*randrange(2^31) for i in range(2*t)]
L += [4*randrange(2^30)+1 for i in range(p-2*t)]
L.sort()
L = [(L[i]%4)-1 for i in range(p)]
return Zx(L)
def keygen():
while True:
g = Zx([randrange(3)-1 for i in range(p)])
if R3(g).is_unit(): break
f = randomR()
h = Rq(g)/(3*Rq(f))
pk = encodeRq(h)
return pk,encodeZx(f) + encodeZx(R(lift(1/R3(g)))) + pk
import hashlib
def hash(s): h = hashlib.sha512(); h.update(s); return h.digest()
def encapsulate(pk):
h = decodeRq(pk)
r = randomR()
hr = h * Rq(r)
m = Zx([-nicemod3(nicelift(hr[i])) for i in range(p)])
c = Rq(m) + hr
fullkey = hash(encodeZx(r))
return fullkey[:32] + encoderoundedRq(c,128),fullkey[32:]
def decapsulate(cstr,sk):
f,ginv,h = decodeZx(sk[:185]),decodeZx(sk[185:370]),decodeRq(sk[370:])
confirm,c = cstr[:32],decoderoundedRq(cstr[32:])
f3mgr = Rq(3*f) * c
f3mgr = [nicelift(f3mgr[i]) for i in range(p)]
r = R3(ginv) * R3(f3mgr)
r = Zx([nicemod3(lift(r[i])) for i in range(p)])
hr = h * Rq(r)
m = Zx([-nicemod3(nicelift(hr[i])) for i in range(p)])
checkc = Rq(m) + hr
fullkey = hash(encodeZx(r))
if sum([r[i]==0 for i in range(p)]) != p-2*t: return False
if checkc != c: return False
if fullkey[:32] != confirm: return False
return fullkey[32:]
print("Exe 2")
print("")
pk,sk = keygen()
c,k = encapsulate(pk)
k == decapsulate(c,sk)
print("")
print("{:d} bytes in public key that is {:d} bits".format(len(pk),len(pk)*8))
print("{:d} bytes in secret key that is {:d} bits".format(len(sk),len(sk)*8))
print("{:d} bytes in ciphertext that is {:d} bits".format(len(c),len(c)*8))
print("{:d} bytes in shared secret that is {:d} bits".format(len(k),len(k)*8))
Now I know I can use this to get to the solution of the question mentioned above. I assume the key mentioned in the question is the private key because that is the one we generate (am I right?) so I know I have to edit this function:
def keygen():
while True:
g = Zx([randrange(3)-1 for i in range(p)])
if R3(g).is_unit(): break
f = randomR()
h = Rq(g)/(3*Rq(f))
pk = encodeRq(h) #Encode private key with 128 bits
return pk,encodeZx(f) + encodeZx(R(lift(1/R3(g)))) + pk
In order to do this I tried editing the function encodeRq used on this one to encode 128 bits only but that brought up lots of compiling errors that I just couldn't understand. But at least am I right to assume it's here where I have to set my key to be generated with 128 bits?
I believe the KEM mentioned in the question is handled with the function encapsulate and believe that I don't have to change anything there (am I right?)
The biggest problem is really the DEM phase wich I believe is being implemented on the function decapsulate (am I right?) but how should I change this to use AES? How do I do it on sage? any library I should know about?
I am a bit lost here and just want to know if my assumptions are correct and to be indicated on the right path. Thanks for any answer in advance.
When I solve the problem of Linear Programming, like in the following formula, I want the result of x all to be int type
Consider the following problem:
Minimize: f = -1*x[0] + 4*x[1]
Subject to:
-3*x[0] + 1*x[1] <= 6
1*x[0] + 2*x[1] <= 4
x[1] >= -3
where: -inf <= x[0] <= inf
next is the python coder
>>> c = [-1, 4]
>>> A = [[-3, 1], [1, 2]]
>>> b = [6, 4]
>>> x0_bounds = (None, None)
>>> x1_bounds = (-3, None)
>>> res = linprog(c, A_ub=A, b_ub=b, bounds=(x0_bounds, x1_bounds),
... options={"disp": True})
>>> print(res)
Optimization terminated successfully.
Current function value: -11.428571
Iterations: 2
status: 0
success: True
fun: -11.428571428571429
x: array([-1.14285714, 2.57142857])
message: 'Optimization terminated successfully.'
nit: 2
From the docs:
method : str, optional Type of solver. At this time only βsimplexβ is
supported.
Simplex cannot handle integrality constraints so you cannot solve integer programming problems with scipy.optimize.linprog yet. You can try other libraries like PuLP, Pyomo or CVXOPT.
Scipy added scipy.optimize.milp in version 1.9.0 just recently.
It also added the integrality= parameter to the existing linprog. So your code can be updated as follows
import scipy.optimize
import numpy as np
c = [-1, 4]
A = [[-3, 1.], [1, 2]]
b = [6, 4]
x0_bounds = (None, None)
x1_bounds = (-3.5, None)
res = scipy.optimize.linprog(
c, A_ub=A, b_ub=b, bounds=(x0_bounds, x1_bounds),
integrality=[1, 1],
options={"disp": True})
res.x
array([10., -3.])
Where integrality=[1, 1] specifies that both variables x0 and x1 are to be integers.
(I changed a bound from -3 to -3.5 so that it actually has an interesting difference in solution between integers and reals.)
Following is a python module that includes a function LPmi(.) to solve mixed integer linear programs. It employs the Branch and Bound algorithm on top of scipy.optimize.linprog(.). It is this responder's creation; anyone is free to use or modify it. It also includes an example in the form of a test(.) function.
import numpy as np
from scipy.optimize import linprog
import copy
class LP:
minimise = True
c = None
A_ub = None
b_ub = None
A_eq = None
b_eq = None
bounds = None
method = ""
fun = 0.
x = None
success = False
def __init__(self,c,minimise=True,Aub=None, bub=None, Aeq=None, beq=None,
bounds=None,method="revised simplex"):
self.minimise = minimise
if self.minimise:
self.c = c
else:
self.c = -1 * c
self.A_ub = Aub
self.b_ub = bub
self.A_eq = Aeq
self.b_eq = beq
self.bounds = bounds
self.method = method
def cal(self):
res = linprog(self.c,A_ub=self.A_ub, b_ub=self.b_ub,
A_eq=self.A_eq, b_eq=self.b_eq,bounds=self.bounds,
method=self.method)
if res["success"] == False:
return res["message"]
else:
self.success = True
if self.minimise:
self.fun = res["fun"]
else:
self.fun = -1 * res["fun"]
self.x = res["x"]
def get_res(self):
return (self.x, self.fun, self.success)
class LP_Data:
minimise = True
c = None
A_ub = None
b_ub = None
A_eq = None
b_eq = None
bounds = None
method = ""
def __init__(self,c,minimise=True,Aub=None, bub=None, Aeq=None, beq=None,
bounds=None,method="revised simplex"):
self.minimise = minimise
if self.minimise:
self.c = c
else:
self.c = -1 * c
self.A_ub = Aub
self.b_ub = bub
self.A_eq = Aeq
self.b_eq = beq
self.bounds = bounds
self.method = method
def set_bounds(self,bounds_list):
self.bounds = bounds_list
class LPd:
data = []
fun = 0.
x = []
success = False
result = None
def __init__(self,lp_data):
self.data = lp_data
self.clip_bound = np.array(list(zip(*self.data.bounds)))
def cal(self):
result = None
res = linprog(self.data.c,A_ub=self.data.A_ub, b_ub=self.data.b_ub,
A_eq=self.data.A_eq, b_eq=self.data.b_eq,
bounds=self.data.bounds,
method=self.data.method)
if res["success"] == False:
self.result = ([], np.NaN, False, None)
else:
self.success = True
if self.data.minimise:
self.fun = res["fun"]
else:
self.fun = -1 * res["fun"]
self.x = res["x"].clip(self.clip_bound[0], self.clip_bound[1])
self.result = (self.x, self.fun, self.success, self.data)
def get_res(self):
return self.result
def level_iterator(level0, int_index):
level1 = []
for k in range(len(level0)):
for i in int_index:
if level0[k][0][i-1] != np.floor(level0[k][0][i-1]):
cur_bounds = level0[k][3].bounds[i-1]
lp = LPd(copy.deepcopy(level0[k][3]))
lp.data.bounds[i-1] = (cur_bounds[0],
max(cur_bounds[0], floor(level0[k][0][i-1])))
lp.cal()
output = lp.get_res()
level1.append(output)
lp = LPd(copy.deepcopy(level0[k][3]))
lp.data.bounds[i-1] = (min(np.ceil(level0[k][0][i-1]),
cur_bounds[1]), cur_bounds[1])
lp.cal()
output = lp.get_res()
level1.append(output)
break
return level1
def intsol(solution,int_index):
is_int = True
for i in int_index:
if solution[0][i-1] != np.floor(solution[0][i-1]):
is_int = False
break
return is_int
def feasiblelevl(level, solution, int_index):
newlevel = []
solution = solution
for k in range(len(level)):
lp = level[k]
if len(lp[0]) > 0:
if lp[2] == False:
pass
elif intsol(lp,int_index) and lp[1] >= solution[1]:
solution = lp
elif lp[1] > solution[1]:
newlevel.append(lp)
return (newlevel, solution)
def LPmi(data, int_index):
level0 = []
lp = LPd(data)
lp.cal()
level0.append(lp.get_res())
solution = [None,-np.inf,None,None]
level0 = (level0, solution)
level0 = feasiblelevl(level0[0], solution, int_index)
if len(level0[0]) == 0:
return level0[1][0:3]
else:
for k in range(10):
level1 = level_iterator(level0[0],int_index)
level1 = feasiblelevl(level1, solution, int_index)
if len(level1[0]) == 0:
break
level0 = level1
return level1[1][0:3]
def test():
c = np.array([3.,7])
minimise = False
A_ub = np.array([[7.,8],[1,3]])
b_ub = np.array([56,12])
data = LP_Data(c,minimise,A_ub,b_ub,bounds=[(0,None),(0,None)])
int_index = [2] #or [1,2] or [] or [1]#
out = LPmi(data,int_index)
print(out)
if __name__ == "__main__":
np.set_printoptions(precision=3,suppress=True)
test()
Inspired by w-k-yeung, and having always wanted to do a little implementation of branch and bound, I cooked up my own version.
Do not use this. It's not well-tested and won't perform well. It won't find all possible solutions. There are too many good, free, OSS solutions to recommend this one. However....
If you just want something simple and easy to setup w/ your existing code, and it's a toy problem, and you don't care about the quality of results (or plan to manually check results for final usage), AND you're just doing this for fun, you may want to have a look at this.
I've implemented the branching as depth-first, so memory usage tends to stay roughly constant (but is a function of the search depth you choose and the size of your model.) I've tried to be good with memory where I could and implemented a few optimizations. It's still painfully slow. I'm playing with a 51-variable problem where each has a range of 0-4 or so, and it'll run for days. Plenty of FOSS packages should solve this in seconds or minutes.
Enjoy! This code is this responder's creation. Anyone is free to use or modify it. It has no warranty. I will not support it.
"""A Mixed-Integer solver based on scipy.optimize.linprog.
This code implements branch-and-bound on the linear relaxation of a given
mixed-integer program. It requires numpy and scipy.optimize.
Usage examples are given in the test() and test2() functions. Parameters of
MipModel are mostly as documented in scipy.optimize.linprog. The additional
parameter "int_vars" gives a sequence of indexes of the variables that should be
constrained to integer solutions.
Typical usage will follow the pattern of:
import mip
mip_model = mip.MipModel(C, minimize, Aub, bub, Aeq, beq, bounds, int_vars)
mip.set_debug_prints(True)
best_solution = mip.find_solutions(mip_model, depth_limit=C.shape[0] - 1)
print(f' soln: {best_solution.x}\n objective value: {best_solution.fun}\n'
f'success: '{best_solution.success}')
"""
import collections
from dataclasses import dataclass, field, replace
import datetime
import itertools
import numpy as np
from scipy.optimize import linprog
import sys
from typing import Generator, List, MutableSequence, Optional, Sequence, Tuple
_DEBUG = True
def set_debug_prints(is_on):
global _DEBUG
_DEBUG = is_on
#dataclass(frozen=True)
class MipResult:
model: 'MipModel'
fun: float = np.inf
x: np.ndarray = np.array([])
success: bool = False
def is_int_soln(self) -> bool:
"""Returns True if the result has integer values for integer variables."""
return np.all(self.x[self.model.int_vars] ==
np.floor(self.x[self.model.int_vars]))
def vars_to_split(self) -> List[int]:
"""Returns the list of integer var indexes with non-integer solutions."""
if self.success:
return list(np.where(self.x[self.model.int_vars] !=
np.floor(self.x[self.model.int_vars]))[0])
else:
return []
#dataclass(frozen=True)
class MipModel:
c: np.ndarray
minimize: bool = True
Aub: Optional[np.ndarray] = None
bub: Optional[np.ndarray] = None
Aeq: Optional[np.ndarray] = None
beq: Optional[np.ndarray] = None
bounds : Tuple = ()
int_vars: Sequence[int] = field(default_factory=list)
method: str = 'revised simplex'
clip_bound: np.ndarray = field(init=False)
def __post_init__(self):
if not self.minimize:
object.__setattr__(self, 'c', -1 * self.c)
if not self.bounds:
object.__setattr__(self, 'clip_bound',
np.tile(np.array([[0], [np.inf]]), self.c.shape[0]))
else:
lower, upper = zip(*self.bounds)
numeric_upper = [np.inf if u is None else u for u in upper]
object.__setattr__(self, 'clip_bound',
np.array((lower, numeric_upper), dtype=np.float64))
def solve(self) -> MipResult:
res = linprog(self.c,A_ub=self.Aub, b_ub=self.bub, A_eq=self.Aeq,
b_eq=self.beq, bounds=self.bounds, method=self.method)
if res["success"]:
result = MipResult(
self,
(int(self.minimize) * 2 - 1) * res['fun'],
res['x'].clip(self.clip_bound[0], self.clip_bound[1]),
res['success'])
else:
result = MipResult(self)
return result
def split_on_var(self, var_i: int, value: Optional[float] = None
) -> Generator['MipModel', None, None]:
"""Yields two new models with bound `var_i` split at `value` or middle"""
assert var_i in range(len(self.bounds)), 'Bad variable index for split'
bound_i = self.bounds[var_i]
if value is None:
if bound_i[1] is not None:
value = self.clip_bound[:, var_i].sum() / 2.0
else:
yield self
return
# We know where to split, have to treat None carefully.
elif bound_i[1] is None:
bound_i = (bound_i[0], np.inf)
# else bound and value are set numerically.
assert value >= bound_i[0] and value <= bound_i[1], 'Bad value in split.'
new_bounds = (*self.bounds[:var_i],
(bound_i[0], max(bound_i[0], np.floor(value))),
*self.bounds[var_i + 1:])
yield replace(self, bounds=new_bounds)
if np.isinf(bound_i[1]):
new_upper = (np.ceil(value), None)
else:
new_upper = (min(np.ceil(value), bound_i[1]), bound_i[1])
new_bounds = (*self.bounds[:var_i],
new_upper,
*self.bounds[var_i + 1:])
yield replace(self, bounds=new_bounds)
def filter_result(result: MipResult, best_soln: MipResult,
results: MutableSequence[MipResult]=[]
) -> Tuple[MutableSequence[MipResult], MipResult]:
if result.success:
if result.is_int_soln() and result.fun <= best_soln.fun:
if _DEBUG:
print('\n', f' {result.x}: {result.fun}')
sys.stdout.flush()
best_soln = result
else:
results.append(result)
return results, best_soln
def walk_branches(solutions: Sequence[MipResult], best_soln: MipResult,
depth_limit: int) -> Tuple[MipResult, int]:
"""Does depth-first search w/ branch & bound on the models in `solutions`.
This function keeps track of a best solution and branches on variables
for each solution in `solutions` up to a depth of `depth_limit`. Intermediate
best solutions are printed with objective function and timestamp.
Args:
solutions: Iterable of MipResult, assumes "not result.is_int_soln()"
best_soln: MipResult of the best integer solution (by fun) so far.
depth_limit: Integer depth limit of the search. This function will use up
to (2**depth_limit + 1) * 8 bytes of memory to store hashes that
identify previoulsy solved instances. Shallow copies are used
aggressively, but memory may be an issue as a function of this
parameter. CPU time _certainly_ is. Remove use of "repeats" for constant
memory utilization.
Returns: The best MipResult obtained in the search (by result.fun) and a
count of stopped branches.
"""
def chirp(top_count):
if _DEBUG:
now = datetime.datetime.now()
print(f'{len(stack):3} {now.strftime("%m-%d %H:%M:%S")}')
sys.stdout.flush()
num_cut = 0
repeats = set()
# Put solutions in a stack paired with variable indexs that need splitting.
# Pop an element, split it, and push any new branches back on, reducing vars
# to split by one element. Repeat until stack_limit reached or stack is
# empty.
stack = [(s, s.vars_to_split()) for s in solutions]
stack_limit = len(stack) + depth_limit + 1
start_depth = len(stack)
chirp(start_depth)
while stack and (len(stack) <= stack_limit):
if len(stack) < start_depth:
chirp(len(stack))
start_depth = len(stack)
cur_depth = len(stack) - start_depth
soln, split_vars = stack.pop()
var_i = split_vars.pop()
if split_vars:
stack.append((soln, split_vars))
for model in soln.model.split_on_var(var_i, soln.x[var_i]):
# Skip any set of bounds we've (probably) already seen.
bounds_hash = hash(model.bounds)
if bounds_hash in repeats:
continue
# Filter out any infeasible or integer solutions to avoid further
# processing.
result = model.solve()
if result.success:
if result.is_int_soln() and result.fun <= best_soln.fun:
if _DEBUG:
now = datetime.datetime.now()
time = now.strftime("%H:%M:%S")
print(f'\n {result.x}: {result.fun} (d={cur_depth}, {time})')
sys.stdout.flush()
best_soln = result
elif len(stack) < stack_limit:
new_vars = result.vars_to_split()
if new_vars:
stack.append((result, new_vars))
else:
num_cut += 1
# Save bounds for all but the last layer (which uses too much storage
# for the compute it saves.)
if cur_depth < depth_limit - 1:
repeats.add(bounds_hash)
return best_soln, num_cut
def find_solutions(mip_model: MipModel, depth_limit: int = 0) -> MipResult:
"""Searches for mixed integer solutions to mip_model with branch & bound.
This function searches for mixed-integer solutions to the given model by
branching on the linear relaxtion of the model. Results will be returned
early (if set_debug_prints(True) is called) by passing over results in
depth_limit // npasses (currently 3), so depth_limit can be passed in large
and the program terminated early if adequate results are found.
The search does not (at the moment) branch on all variables, only those for
which the linear relaxation gives non-integer solutions. This may limit the
search space and miss some solutions. To fully exhaust the search space, set
depth_limit to the number of variables you have.
If set_debug_prints(True) is called before, the search will also print a
countdown and timestamps to give a since of how far the search is and over
what runtime.
Args:
mip_model: The MipModel to search.
depth_limit: The maximum depth of search tree branching. Default 0 means
branch for each variable in the model (i.e. full search).
Returns:
MipResult of the best solution found.
"""
best_soln = MipResult(mip_model)
linear_soln = mip_model.solve()
if not linear_soln.success:
print('Model is infeasible even for linear relaxation.')
return best_soln
if not len(mip_model.int_vars):
return linear_soln
# Start with some reasonable areas to search.
solutions = []
models = [mip_model] + list(itertools.chain.from_iterable(
mip_model.split_on_var(v) for v in mip_model.int_vars))
for result in map(MipModel.solve, models):
solutions, best_soln = filter_result(result, best_soln, solutions)
# Default depth_limit means branch on all the vars.
if depth_limit == 0:
depth_limit = mip_model.c.shape[0]
# Since we did one split above, we've already fixed one var and done one
# level, so depth_limit can be reduced.
best_soln, num_cut = walk_branches(solutions, best_soln, depth_limit - 1)
if _DEBUG:
now = datetime.datetime.now()
print('{}: {} branches un-explored.\n'.format(
now.strftime("%m-%d %H:%M:%S"),
num_cut))
return best_soln
def test():
"""Implements the example from scipy.optimizie.linprog doc w/ 1 int var."""
set_debug_prints(False)
C = np.array([-1, 4])
minimize = True
Aub = np.array([[-3, 1],[1, 2]])
bub = np.array([6, 4])
int_vars = [1] # or [0, 1] or [] or [0]
mip_model = MipModel(C, minimize, Aub, bub, bounds=((0, None), (-3, None)),
int_vars=int_vars)
out = find_solutions(mip_model)
print(f'test:\n soln: {out.x}\n objective value: {out.fun}\n success: '
f'{out.success}')
def test2():
"""Implements the reference problem from https://xkcd.com/287/."""
import math
set_debug_prints(True)
print('test2: (takes ~15 seconds)')
target = 15.05
C = np.array([2.15, 2.75, 3.35, 3.55, 4.2, 5.8])
bounds = tuple((0, math.floor(target / c)) for c in C)
Aeq = C[np.newaxis, :]
beq = np.array([target])
# Aub = None
# bub = None
minimize = False
int_vars = np.arange(C.shape[0])
mip_model = MipModel(C, minimize, None, None, Aeq, beq, bounds=bounds,
int_vars=int_vars)
out = find_solutions(mip_model, depth_limit=C.shape[0] - 1)
print(f' soln: {out.x}\n objective value: {out.fun}\n success: '
f'{out.success}')
if __name__ == "__main__":
np.set_printoptions(precision=3,suppress=True)
test()
test2()
scipy has now added the milp function in their upcoming 1.9.0 release (see the official documentation) so you can now also solve mixed-integer problems. It uses the performant open source HiGHS solver under the hood.
For your example:
import numpy as np
from scipy.optimize import linprog, milp, Bounds, LinearConstraint
# linprog
c = [-1, 4]
A_ub = [[-3, 1], [1, 2]]
b_ub = [6, 4]
x0_bounds = (None, None)
x1_bounds = (-3, None)
bounds = (x0_bounds, x1_bounds)
options = {"disp": False}
res_linprog = linprog(
c,
A_ub=A_ub,
b_ub=b_ub,
bounds=bounds,
options=options)
print(res_linprog)
# MILP
integrality = [1, 1]
lb = [-np.inf, -3]
ub = [np.inf, np.inf]
variable_bounds = Bounds(lb, ub)
constraints = LinearConstraint(A_ub, -np.inf, b_ub)
res_milp = milp(
c,
integrality=integrality,
bounds=variable_bounds,
constraints=constraints,
options=options)
# milp() returns float values even for integer variables
# so we can cast to int
res_milp.x = [int(x_i) for x_i in res_milp.x]
print(res_milp)