What is the progress object to upload functions on Minio? - python

I know that this can be a dumb question but, what's the Progress object that I must pass to MinIO to see the upload status of an object?
This documentation:
https://docs.min.io/docs/python-client-api-reference.html
The method:
fput_object(bucket_name, object_name, file_path, content_type="application/octet-stream", metadata=None, sse=None, progress=None, part_size=0, num_parallel_uploads=3, tags=None, retention=None, legal_hold=False)
An example provided by the documentation:
# Upload data with progress bar.
result = client.fput_object(
"my-bucket", "my-object", "my-filename",
progress=Progress(),
)
print(
"created {0} object; etag: {1}, version-id: {2}".format(
result.object_name, result.etag, result.version_id,
),
)
The problem is the object Progress, I cannot find an applicable instance to run it. Can anyone help me on this or provide an example?


Just add the source code in the link.
https://github.com/minio/minio-py/blob/master/examples/progress.py
from minio import Minio
from minio.error import S3Error
import sys
import time
import urllib3
from queue import Empty, Queue
from threading import Thread
_BAR_SIZE = 20
_KILOBYTE = 1024
_FINISHED_BAR = '#'
_REMAINING_BAR = '-'
_UNKNOWN_SIZE = '?'
_STR_MEGABYTE = ' MB'
_HOURS_OF_ELAPSED = '%d:%02d:%02d'
_MINUTES_OF_ELAPSED = '%02d:%02d'
_RATE_FORMAT = '%5.2f'
_PERCENTAGE_FORMAT = '%3d%%'
_HUMANINZED_FORMAT = '%0.2f'
_DISPLAY_FORMAT = '|%s| %s/%s %s [elapsed: %s left: %s, %s MB/sec]'
_REFRESH_CHAR = '\r'
def main():
# Create client with anonymous access.
client = Minio(
"192.168.2.193:9000",
access_key="access",
secret_key="password",
secure=False,
http_client=urllib3.ProxyManager(
"http://192.168.2.193:9000",
timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
# cert_reqs="CERT_REQUIRED",
retries=urllib3.Retry(
total=5,
backoff_factor=0.2,
status_forcelist=[500, 502, 503, 504],
),
),
)
found = client.bucket_exists('test1')
if not found:
client.make_bucket('test1')
else:
print('Bucket test1 already exists')
client.fput_object('test1', 'dgk_csv_no_space.zip', 'D:/dgk_csv_no_space.zip', progress=Progress())
print("success")
class Progress(Thread):
"""
Constructs a :class:`Progress` object.
:param interval: Sets the time interval to be displayed on the screen.
:param stdout: Sets the standard output
:return: :class:`Progress` object
"""
def __init__(self, interval=1, stdout=sys.stdout):
Thread.__init__(self)
self.daemon = True
self.total_length = 0
self.interval = interval
self.object_name = None
self.last_printed_len = 0
self.current_size = 0
self.display_queue = Queue()
self.initial_time = time.time()
self.stdout = stdout
self.start()
def set_meta(self, total_length, object_name):
"""
Metadata settings for the object. This method called before uploading
object
:param total_length: Total length of object.
:param object_name: Object name to be showed.
"""
self.total_length = total_length
self.object_name = object_name
self.prefix = self.object_name + ': ' if self.object_name else ''
def run(self):
displayed_time = 0
while True:
try:
# display every interval secs
task = self.display_queue.get(timeout=self.interval)
except Empty:
elapsed_time = time.time() - self.initial_time
if elapsed_time > displayed_time:
displayed_time = elapsed_time
self.print_status(current_size=self.current_size,
total_length=self.total_length,
displayed_time=displayed_time,
prefix=self.prefix)
continue
current_size, total_length = task
displayed_time = time.time() - self.initial_time
self.print_status(current_size=current_size,
total_length=total_length,
displayed_time=displayed_time,
prefix=self.prefix)
self.display_queue.task_done()
if current_size == total_length:
self.done_progress()
def update(self, size):
"""
Update object size to be showed. This method called while uploading
:param size: Object size to be showed. The object size should be in
bytes.
"""
if not isinstance(size, int):
raise ValueError('{} type can not be displayed. '
'Please change it to Int.'.format(type(size)))
self.current_size += size
self.display_queue.put((self.current_size, self.total_length))
def done_progress(self):
self.total_length = 0
self.object_name = None
self.last_printed_len = 0
self.current_size = 0
def print_status(self, current_size, total_length, displayed_time, prefix):
formatted_str = prefix + format_string(
current_size, total_length, displayed_time)
self.stdout.write(_REFRESH_CHAR + formatted_str + ' ' *
max(self.last_printed_len - len(formatted_str), 0))
self.stdout.flush()
self.last_printed_len = len(formatted_str)
def seconds_to_time(seconds):
"""
Consistent time format to be displayed on the elapsed time in screen.
:param seconds: seconds
"""
minutes, seconds = divmod(int(seconds), 60)
hours, m = divmod(minutes, 60)
if hours:
return _HOURS_OF_ELAPSED % (hours, m, seconds)
else:
return _MINUTES_OF_ELAPSED % (m, seconds)
def format_string(current_size, total_length, elapsed_time):
"""
Consistent format to be displayed on the screen.
:param current_size: Number of finished object size
:param total_length: Total object size
:param elapsed_time: number of seconds passed since start
"""
n_to_mb = current_size / _KILOBYTE / _KILOBYTE
elapsed_str = seconds_to_time(elapsed_time)
rate = _RATE_FORMAT % (
n_to_mb / elapsed_time) if elapsed_time else _UNKNOWN_SIZE
frac = float(current_size) / total_length
bar_length = int(frac * _BAR_SIZE)
bar = (_FINISHED_BAR * bar_length +
_REMAINING_BAR * (_BAR_SIZE - bar_length))
percentage = _PERCENTAGE_FORMAT % (frac * 100)
left_str = (
seconds_to_time(
elapsed_time / current_size * (total_length - current_size))
if current_size else _UNKNOWN_SIZE)
humanized_total = _HUMANINZED_FORMAT % (
total_length / _KILOBYTE / _KILOBYTE) + _STR_MEGABYTE
humanized_n = _HUMANINZED_FORMAT % n_to_mb + _STR_MEGABYTE
return _DISPLAY_FORMAT % (bar, humanized_n, humanized_total, percentage,
elapsed_str, left_str, rate)
if __name__ == '__main__':
try:
main()
except S3Error as exc:
print("error ouccured.", exc)

Related

QUERY_EXCEEDED_MAX_MATCHES_ALLOWED error on Kaltura API (Python)

I'm unable to generate all entries in Kaltura. An ApiException with the message "Unable to generate list. max matches value was reached" (Error: QUERY_EXCEEDED_MAX_MATCHES_ALLOWED) gets triggered.
I tried to work around such issue by setting my sessionPrivileges to disableentitlement
class class_chk_integrity():
client = None
pagesize = 0
def __init__(self,worker_num, progress):
self.pagesize = 30
self.worker_num = worker_num
self.progress = progress
config = KalturaConfiguration(2723521)
config.serviceUrl = "https://www.kaltura.com/"
self.client = KalturaClient(config)
ks = self.client.session.start("KALTURA_ADMIN_SECRET",
"email#email.com",
KalturaPluginsCore.KalturaSessionType.ADMIN,
"KALTURA_PARTNER_ID",
432000,
"disableentitlement")
self.client.setKs(ks)
I also tried to filter based on the id's. However, I can't manage to put the filter.idNotIn to work properly.
def get_total_reg(self, cont, lastEntryIds, lastEntryCreatedAt):
filter = KalturaPluginsCore.KalturaBaseEntryFilter()
if lastEntryIds != "":
filter.idNotIn = lastEntryIds
filter.orderBy = KalturaBaseEntryOrderBy.CREATED_AT_DESC
pager = KalturaPluginsCore.KalturaFilterPager()
pageIndex = 1
entriesGot = 0
pager.pageSize = self.pagesize
pager.setPageIndex = pageIndex
result = self.client.baseEntry.list(filter, pager)
totalCount = result.totalCount
if totalCount > 10000:
totalCount = 9970
if totalCount <= 0:
cont = False
while entriesGot < totalCount:
pager.pageSize = self.pagesize
pageIndex += 1
pager.pageIndex = pageIndex
result = self.client.baseEntry.list(filter, pager)
entriesGot += len(result.objects)
for e in result.objects:
if lastEntryIds == "":
lastEntryIds.append(e.id)
else:
lastEntryIds.append(e.id)
lastEntryCreatedAt = e.createdAt
return result.totalCount, self.pagesize, cont, lastEntryIds, lastEntryCreatedAt
This is my how I'm calling the functions
if __name__ == '__main__':
try:
log = _ServiceUtils.log()
log.setup('all', 'integrity')
cont = True
lastEntryIds = []
lastEntryCreatedAt = 0
while cont is True:
kmc = class_chk_integrity(0,0)
kmc_total_reg, kmc_page_size, cont, lastEntryIds, lastEntryCreatedAt = kmc.get_total_reg(cont, lastEntryIds, lastEntryCreatedAt)
interval = 10
max_threads = math.ceil(kmc_total_reg / (interval * kmc_page_size))
# max_threads = 1
threads_list = []
print('TOTAL REG : %s | PAGE_SIZE : %s | INTERVAL : %s | THREADS : %s' % (kmc_total_reg,kmc_page_size,interval,max_threads))
progress = class_progress_thread(max_threads)
for index in range(0,max_threads):
page_ini = index * interval
page_end = index * interval + interval
progress.add_worker_progress(index,datetime.now())
threads_list.append(threading.Thread(target=thread_chk_integrity, args=(index, log, index * interval + 1,index * interval + interval,progress)))
threads_list.append(threading.Thread(target=thread_output_progress, args=(progress,max_threads)))
for thread in threads_list:
thread.start()
for thread in threads_list:
thread.join()
while not progress.stop(): time.sleep(30)
except KeyboardInterrupt:
try:
sys.exit(0)
except SystemExit:
os._exit(0)
I'd appreciate any help with this.
Thank you for your attention.

if totalCount > 10000:
totalCount = 9970
I'm curious to know why you are changing the totalCount this way.
Short answer - paging works as long as the result set is up to 10K.
To work around that, sort the result by creation date (as you did), and when you get to 10K, start with a new search where the created_at date in the filter is the last value you got in the previous search. Reset your paging of course.

Python socket recv buffer handling

Im attempting to speed up socket handling in python using a buffer/queue
below an implementation for cyclic buffer I use to read max size out of localhost socket used for IPC
class BufferHandling:
def __init__(self, buffer_size, max_chunk_size):
self._read_index = 0
self._write_index = 0
self._buffer = bytearray(buffer_size)
self._mem_view = memoryview(self._buffer)
self._max_size = max_chunk_size
def fill_from_socket(self, _socket):
logging.debug("2 fill: write index %s , read_index %s", self._write_index , self._read_index)
if self._read_index > self._write_index:
if self._read_index - self._write_index < self._max_size:
raise RuntimeError("no place to write in buffer")
ret = _socket.recv_into(self._mem_view[self._write_index:], self._max_size)
self._write_index += ret
#logging.debug("1 recv_from return with %s, write index %s", ret, self._write_index)
return
# write >= read
_1st_write_size = len(self._mem_view) - self._write_index
_2nd_write_size = self._read_index
if _1st_write_size + _2nd_write_size < self._max_size:
raise RuntimeError("no place to write in buffer")
if _1st_write_size >= self._max_size:
ret = _socket.recv_into(self._mem_view[self._write_index:], self._max_size)
self._write_index += ret
#logging.debug("2 recv_from return with %s , write index %s", ret, self._write_index)
return
# clear some room ...
self._mem_view[:self._write_index - self._read_index] = self._mem_view[self._read_index: self._write_index]
self._write_index = self._write_index - self._read_index
self._read_index = 0
ret = _socket.recv_into(self._mem_view[self._write_index:], self._max_size)
self._write_index += ret
logging.debug("3 recv_from return with %s, write index %s", ret, self._write_index)
"""
# read data from cyclic buffer
# if data found , return data according to _to_read size
# else return empty string
"""
def read_from_buffer(self, _to_read):
logging.debug("1 write_index %s , read_index %s , _to_read %s", self._write_index, self._read_index, _to_read)
if self._read_index > self._write_index:
_1st_read_size = len(self._mem_view) - self._read_index
_2nd_read_size = self._write_index
if _1st_read_size >= _to_read:
data = self._buffer[self._read_index: self._read_index + _to_read]
self._read_index += _to_read
else:
data = self._buffer[self._read_index:] + self._buffer[:_to_read - _1st_read_size]
self._read_index = _to_read - _1st_read_size
else: # write >= read
_read_less = min(_to_read, self._write_index - self._read_index)
# if self._write_index - self._read_index < _to_read:
# logging.debug("2 write_index %s , read_index %s , _to_read %s", self._write_index,self._read_index,_to_read)
# #return None
logging.debug("2 write_index %s , read_index %s , _to_read %s, _read %s", self._write_index, self._read_index, _to_read, _read_less)
data = self._buffer[self._read_index: self._read_index + _read_less]
self._read_index += _read_less
return data
I using it like:
self.__buffer = BufferHandling(self.__LOCALHOST_SOCKET_BUFFER_LEN * 4,
self.__LOCALHOST_SOCKET_BUFFER_LEN)
data = self.__buffer.read_from_buffer(_to_read)
if len(data) < _to_read:
self._selector.select(rlist = [self.__connection], elist = [self.__connection])
self.__buffer.fill_from_socket(self.__connection)
data += self.__buffer.read_from_buffer(_to_read)
After facing many issues, my question: is there a more Pythonic way to get the same affect?
or maybe suggest a more "clean" way to get some form of io buffering

I have created a short method to "clear" the buffer of a UDP socket. Here is the code:
def empty(sock):
"""Empty the UDP buffer."""
sock.setblocking(0)
while True:
try:
sock.recv(1024)
except BlockingIOError:
sock.setblocking(1)
return

API Exporting Issue

I need someone's expertise on this exporting problem of mine.
How it works: Select a camera (animated or not is optional) >> File >> Export Selection >> File Type : .chan (need to load this script as a plugin)
Here's where the problem starts. It is able to create a .text file, however, it is not 'exporting' or writing out the contents into the text file and the file size is of zero bytes.
I am making use of the current API that it has been coded, modifying the code to add in some maya cmds
Can someone kindly help me out?
import math, sys, string, os
import maya.OpenMaya as OpenMaya
import maya.OpenMayaMPx as OpenMayaMPx
import maya.OpenMayaAnim as OpenMayaAnim
import maya.cmds as cmds
import maya.mel as mel
kPluginTranslatorTypeName = "chan Export/Import"
kVersionNumber = "0.5a"
camSel = []
win_name = "chan_window"
class CustomNodeTranslator(OpenMayaMPx.MPxFileTranslator):
def __init__(self):
OpenMayaMPx.MPxFileTranslator.__init__(self)
def haveWriteMethod(self):
return True
def haveReadMethod(self):
return True
def filter(self):
return " .chan"
def defaultExtension(self):
return "chan"
def writer( self, fileObject, optionString, accessMode ):
try:
fullName = fileObject.fullName()
fileHandle = open(fullName,"w")
selectList = OpenMaya.MSelectionList()
OpenMaya.MGlobal.getActiveSelectionList(selectList)
node = OpenMaya.MObject()
depFn = OpenMaya.MFnDependencyNode()
path = OpenMaya.MDagPath()
iterator = OpenMaya.MItSelectionList(selectList)
animationTime = OpenMayaAnim.MAnimControl()
maxTime = int(animationTime.maxTime().value())
minTime = int(animationTime.minTime().value())
while (iterator.isDone() == 0):
iterator.getDependNode(node)
depFn.setObject(node)
iterator.getDagPath(path, node)
cameraObject = OpenMaya.MFnCamera(path)
transform = OpenMaya.MFnTransform(path)
chanMe = fileExporter(transform, minTime, maxTime, cameraObject)
for all in chanMe():
fileHandle.write(all)
iterator.next()
fileHandle.close()
except:
sys.stderr.write( "Failed to write file information\n")
raise
def processLine( self, lineStr ):
self.importTheChan.writeFrameData(lineStr)
class fileExporter():
""" module for exporting chan files from application. arguments: object, startFrame, endFrame """
def __init__(self, transform, startAnimation, endAnimation, cameraObj):
self.fileExport = []
self.transform = transform
self.cameraObj = cameraObj
self.start = startAnimation
self.end = endAnimation
self.exportWin()
def exportWin(self):
self.expWindow = cmds.window(w=150, h=100, title = "Export Selection" )
cmds.columnLayout( adjustableColumn=True )
form = cmds.formLayout(numberOfDivisions=100)
cmds.radioCollection()
self.chk1 = cmds.radioButton( label='option1', onc = self.opt1On, ofc = self.opt1Off )
self.chk2 = cmds.radioButton( label='option2', onc = self.opt2On, ofc = self.opt2Off )
self.okayBtn = cmds.button(label='okay!', command=self.runSel, width=150, height=35)
cmds.formLayout(form, edit=True, attachForm=[\
(self.chk1, 'top', 15),\
(self.chk1, 'left', 15),\
(self.chk2, 'top', 30),\
(self.chk2, 'left', 15),\
(self.okayBtn, 'top', 50),\
(self.okayBtn, 'left', 15)])
cmds.showWindow( self.expWindow )
def opt1On(self, args):
print "User checked option1"
startAnimation = cmds.playbackOptions(query=True, minTime=True)
endAnimation = cmds.playbackOptions(query=True, maxTime=True)
self.start = startAnimation
self.end = endAnimation
def opt1Off(self, args):
print "User un-checked option1"
cmds.radioButton(self.chk2, edit = True, enable = True)
self.start = ""
self.end = ""
def opt2On(self, args):
print "User checked option2"
startAnimation = cmds.findKeyframe(which='first')
endAnimation = cmds.findKeyframe(which='last')
self.start = startAnimation
self.end = endAnimation
#self.start.append(int(startAnimation))
#self.end.append(int(endAnimation))
def opt2Off(self, args):
print "User un-checked option2"
self.start = ""
self.end = ""
def runSel(self, args):
chkVal1 = cmds.radioButton(self.chk1, query=True, sl=1)
chkVal2 = cmds.radioButton(self.chk2, query=True, sl=1)
if chkVal1 == 1:
print "opt1 Pressed!"
print self.start
print self.end
self.test()
self.closeWindow()
elif chkVal2 == 1:
print "opt2 Pressed!"
print self.start
print self.end
self.test()
self.closeWindow()
else:
cmds.warning("Check an option")
def closeWindow(self):
cmds.deleteUI(self.expWindow, window=True)
def test(self):
self.actualExp(self.transform, self.start, self.end, self.cameraObj)
def actualExp(self, transform, startAnimation, endAnimation, cameraObj):
mayaGlobal = OpenMaya.MGlobal()
mayaGlobal.viewFrame(OpenMaya.MTime(1))
# Converts the float arguement into integer
for i in range(int(startAnimation), int(endAnimation + 1)):
focalLength = cameraObj.focalLength()
vFilmApp = cameraObj.verticalFilmAperture()
focalOut = 2 math.degrees(math.atan(vFilmApp 25.4/ (2 focalLength)))
myEuler = OpenMaya.MEulerRotation()
spc = OpenMaya.MSpace.kWorld
trans = transform.getTranslation(spc)
rotation = transform.getRotation(myEuler)
rotVector = OpenMaya.MVector(myEuler.asVector())
self.fileExport.append((str(i) + '\t' + str(trans[0]) + "\t" + str(trans[1]) + "\t" + str(trans[2]) + "\t" + str(math.degrees(rotVector[0])) + "\t" + str(math.degrees(rotVector[1])) + "\t" + str(math.degrees(rotVector[2])) + "\t" + str(focalOut) + "\n"))
mayaGlobal.viewFrame(OpenMaya.MTime(i+1))
def __call__(self, args):
return self.fileExport
def radianToDegree(self, radians):
outDegrees = 0.0
outDegrees = (float(radians) / (math.pi)) 180
return outDegrees
# creator
def translatorCreator():
return OpenMayaMPx.asMPxPtr( CustomNodeTranslator() )
# initialize the script plug-in
def initializePlugin(mobject):
mplugin = OpenMayaMPx.MFnPlugin(mobject)
try:
mplugin.registerFileTranslator(kPluginTranslatorTypeName, None, translatorCreator)
except:
sys.stderr.write( "Failed to register translator: %s" % kPluginTranslatorTypeName )
raise
# uninitialize the script plug-in
def uninitializePlugin(mobject):
mplugin = OpenMayaMPx.MFnPlugin(mobject)
try:
mplugin.deregisterFileTranslator( kPluginTranslatorTypeName )
except:
sys.stderr.write( "Failed to deregister translator: %s" % kPluginTranslatorTypeName )
raise

the __call__ method is what's supposed to provide the contents of the file. It returns self.fileExport, which is an empty list that is not getting populated.

The problem here is the writer method of the plugin will not wait for your exportWin UI to return the user inputs when you call
chanMe = fileExporter(transform, minTime, maxTime, cameraObject)
By the time the user has entered the inputs, the statements that follow have already been executed:
for all in chanMe():
fileHandle.write(all)
iterator.next()
fileHandle.close()
That is why plugin-based file exporters like these have their options UI tucked away in the option box. These options will be passed prior to call to the plugin's writer().
You will need to export your options UI code (in a certain specific format) using MEL in another script file and specify the name of that file in the optionsScriptName param of the registerFileTranslator call. There is a communication protocol that needs to be followed for communication between this options UI and the writer plugin itself. RobTheBloke's awesome post illustrates this process.
Ideally, the writer() method should have all the details it needs for computing and exporting without having to wait for user input.
Alternatively, if you prefer to have your own UI window and more control over the flow of things, you could write the exporter not as a plugin, but as a simple MEL/Python module. You could still use the power of the API.
Hope this helped!

ip address networks in Python 3.3

I am trying to convert a group of IP ranges that can start and end on any given address, but not necessarily on a .0, .127, or .255, etc. I have code that is mostly working; however, it can be slow for large ranges.
For example find_range("1.40.0.0","1.44.255.255") will take over a minute to return the correct result of 1.40.0.0/14 and 1.44.0.0/16.
Also, I am have trouble when the starting range does not end with a .0. How can I fix these 2 issues: slowness on large IP ranges and when the starting range does not end in .0?
For the slowness problem, I tried skipping more than 1 address at a time, but then this would miss smaller ranges.
import ipaddress, socket, struct
def ip2int(addr):
return struct.unpack("!I", socket.inet_aton(addr))[0]
def int2ip(addr):
return socket.inet_ntoa(struct.pack("!I", addr))
def ipminus(ip, amount=1):
tmp = ip2int(ip)
return int2ip(tmp - amount)
def ipplus(ip):
tmp = ip2int(ip)
return int2ip(tmp + 1)
def cidr_notation(a,b):
for mask in range(32, 6, -1):
test = "%s/%s" % (a,mask)
try:
n = ipaddress.IPv4Network(test,False)
if b == "%s" % (n.broadcast_address):
return test
except:
pass
return None
def split_range(a,b):
a1 = ip2int(a)
b1 = ip2int(b)
needed = 1
while needed:
result = cidr_notation(a,b)
if result:
print( "* %16s\t%16s\t%16s" % (result, a, b))
if ip2int(b) > b1:
needed = 0
else:
a = ipplus(b)
b = int2ip(b1)
else:
b = ipminus(b)
return result
def find_range(x,y):
result = cidr_notation(x,y)
if result:
print( "# %16s\t%16s\t%16s" % (result, x, y))
else:
split_range(x,y)
# main...
print("%16s\t%16s\t%16s" % ("mask","start","end"))
print("%16s\t%16s\t%16s" % ("----","-----","---"))
find_range("128.191.0.0","128.191.255.255") #fast
find_range("10.55.96.106","10.55.96.106") #fast
find_range("5.135.14.0","5.135.61.11") #slow
find_range("4.31.64.72","4.59.175.255") #does not work, how to fix?
find_range("1.40.0.0","1.44.255.255") #very slow
# 5000 more find_range() calls...

Based on bsdlp's comment, this code got a lot simpler and faster!
def find_range(x,y,c_code="",c_name=""):
print()
print("%29s\t%23s" % (x,y))
print("%16s\t%16s\t%16s\t%4s\t%s" % ("mask","start","end","code","name"))
print("%16s\t%16s\t%16s\t%4s\t%s" % ("----","-----","---","----","----"))
result = ipaddress.summarize_address_range( ipaddress.IPv4Address(x), ipaddress.IPv4Address(y) )
for entry in result:
net = str( entry.network_address )
bcast = str( entry.broadcast_address )
print( "%16s\t%16s\t%16s\t%4s\t%s" % (entry, net, bcast,c_code,c_name))

Importing code from file into Python and adding file dialog box

I have a python script signalgen.py that plays audio using equations but I would like to be able to hard code the file where the equation is stored in eq1.txt or choose a file and import the equation.
The problems I'm having are:
1) How can I hard code a file and it's path correctly so it will play the equation as audio
I get an error
Traceback (most recent call last):
File "signalgen.py", line 484, in need_data
v += (datum * self.sig_level)
TypeError: can't multiply sequence by non-int of type 'float'
The specific block of code which I believe is causing the issue
def equation_import_function(self,t,f):
fileobj=open("/home/rat/eq1.txt","r")
eqdata =fileobj.read() #read whole file
fileobj.close()
#return math.tan(2.0*math.pi*f*t)
return eqdata
I have this line of code in the eq1.txt file-> math.tan(2.0*math.pi*f*t)
2) How can I add a file open dialog box to be able to choose a file and import the equation.
PS I'm using Ubuntu 10.04 (Linux) and the equations will be several pages long this is the reason I would like to import them into python from text files
Here's the entire code if you want to look at what I'm using below or seen on pastebin which includes line numbers http://pastebin.com/HZg0Jhaw
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# ***************************************************************************
# * Copyright (C) 2011, Paul Lutus *
# * *
# * This program is free software; you can redistribute it and/or modify *
# * it under the terms of the GNU General Public License as published by *
# * the Free Software Foundation; either version 2 of the License, or *
# * (at your option) any later version. *
# * *
# * This program is distributed in the hope that it will be useful, *
# * but WITHOUT ANY WARRANTY; without even the implied warranty of *
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
# * GNU General Public License for more details. *
# * *
# * You should have received a copy of the GNU General Public License *
# * along with this program; if not, write to the *
# * Free Software Foundation, Inc., *
# * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
# ***************************************************************************
# version date 01-12-2011
VERSION = '1.1'
import re, sys, os
import gobject
gobject.threads_init()
import gst
import gtk
gtk.gdk.threads_init()
import time
import struct
import math
import random
import signal
import webbrowser
class Icon:
icon = [
"32 32 17 1",
" c None",
". c #2A2E30",
"+ c #333739",
"# c #464A4C",
"# c #855023",
"$ c #575A59",
"% c #676A69",
"& c #CC5B00",
"* c #777A78",
"= c #DB731A",
"- c #8A8C8A",
"; c #969895",
"> c #F68C22",
", c #A5A7A4",
"' c #F49D4A",
") c #B3B5B2",
"! c #DEE0DD",
" &&&&&&& ",
" &&&===='''''& ",
" &'''''====&'& ",
" +++++&'&&&&& &'& ",
" +#$%****&'&+ &'& ",
" +#**%$#++#&'&*#+ &'& ",
" +#**#+++++++&'&#**#+ &'& ",
" +$*$+++++++++&'&++$*$+ &'& ",
" #*#++++++++++&'&+++##&&&'& ",
" +*#++++++++#&&&'&+++#=''''& ",
" +*$++++++++#=''''&+++&'>>>'& ",
" #*+++++++++&'>>>'&+++#='''= ",
" +%$++++++++##='''=###++#&&&# ",
" +*#+++++++####&&&######++#*+ ",
" +*+++++++####++#$%$$####++*+ ",
" +*++++++##+#;,,*##*$$$###+*+ ",
" +*#++++###%!!!!,;#$*$$$###*+ ",
" +%$++++##+)!!!),-*+-%$$$#$%+ ",
" +#*+++###+-!!!,;-%#;%%$$+*#+ ",
" +*#++####+$*-*%#+*-%%$##*+ ",
" ++*#+###$$%#++#%;;*%%$#-$+ ",
" +#%+###$$%*;;;;-*%%%#**+ ",
" .+$%###$$$*******%$$*-+. ",
" .+#%%##$$*#*#%%%$%-%+. ",
" .++#%$$$$$$%%%%--#+. ",
" +++##$%*****%+++ ",
" +++++++++++++#. ",
" #--%#++#$*-%+ ",
" +%,))),;%+. ",
" ++++++. ",
" ",
" "
]
# this should be a temporary hack
class WidgetFinder:
def localize_widgets(self,parent,xmlfile):
# an unbelievable hack made necessary by
# someone unwilling to fix a year-old bug
with open(xmlfile) as f:
for name in re.findall('(?s) id="(.*?)"',f.read()):
if re.search('^k_',name):
obj = parent.builder.get_object(name)
setattr(parent,name,obj)
class ConfigManager:
def __init__(self,path,dic):
self.path = path
self.dic = dic
def read_config(self):
if os.path.exists(self.path):
with open(self.path) as f:
for record in f.readlines():
se = re.search('(.*?)\s*=\s*(.*)',record.strip())
if(se):
key,value = se.groups()
if (key in self.dic):
widget = self.dic[key]
typ = type(widget)
if(typ == list):
widget[0] = value
elif(typ == gtk.Entry):
widget.set_text(value)
elif(typ == gtk.HScale):
widget.set_value(float(value))
elif(typ == gtk.Window):
w,h = value.split(',')
widget.resize(int(w),int(h))
elif(typ == gtk.CheckButton or typ == gtk.RadioButton or typ == gtk.ToggleButton):
widget.set_active(value == 'True')
elif(typ == gtk.ComboBox):
if(value in widget.datalist):
i = widget.datalist.index(value)
widget.set_active(i)
else:
print "ERROR: reading, cannot identify key %s with type %s" % (key,type(widget))
def write_config(self):
with open(self.path,'w') as f:
for key,widget in sorted(self.dic.iteritems()):
typ = type(widget)
if(typ == list):
value = widget[0]
elif(typ == gtk.Entry):
value = widget.get_text()
elif(typ == gtk.HScale):
value = str(widget.get_value())
elif(typ == gtk.Window):
_,_,w,h = widget.get_allocation()
value = "%d,%d" % (w,h)
elif(typ == gtk.CheckButton or typ == gtk.RadioButton or typ == gtk.ToggleButton):
value = ('False','True')[widget.get_active()]
elif(typ == gtk.ComboBox):
value = widget.get_active_text()
else:
print "ERROR: writing, cannot identify key %s with type %s" % (key,type(widget))
value = "Error"
f.write("%s = %s\n" % (key,value))
def preset_combobox(self,box,v):
if(v in box.datalist):
i = box.datalist.index(v)
box.set_active(i)
else:
box.set_active(0)
def load_combobox(self,obj,data):
if(len(obj.get_cells()) == 0):
# Create a text cell renderer
cell = gtk.CellRendererText ()
obj.pack_start(cell)
obj.add_attribute (cell, "text", 0)
obj.get_model().clear()
for s in data:
obj.append_text(s.strip())
setattr(obj,'datalist',data)
class TextEntryController:
def __init__(self,parent,widget):
self.par = parent
self.widget = widget
widget.connect('scroll-event',self.scroll_event)
widget.set_tooltip_text('Enter number or:\n\
Mouse wheel: increase,decrease\n\
Shift/Ctrl/Alt: faster change')
def scroll_event(self,w,evt):
q = (-1,1)[evt.direction == gtk.gdk.SCROLL_UP]
# magnify change if shift,ctrl,alt pressed
for m in (1,2,4):
if(self.par.mod_key_val & m): q *= 10
s = self.widget.get_text()
v = float(s)
v += q
v = max(0,v)
s = self.par.format_num(v)
self.widget.set_text(s)
class SignalGen:
M_AM,M_FM = range(2)
W_SINE,W_TRIANGLE,W_SQUARE,W_SAWTOOTH,W_EQUATION_IMPORT = range(5)
waveform_strings = ('Sine','Triangle','Square','Sawtooth', 'Equation_Import')
R_48000,R_44100,R_22050,R_16000,R_11025,R_8000,R_4000 = range(7)
sample_rates = ('48000','44100','22050','16000', '11025', '8000', '4000')
def __init__(self):
self.restart = False
# exit correctly on system signals
signal.signal(signal.SIGTERM, self.close)
signal.signal(signal.SIGINT, self.close)
# precompile struct operator
self.struct_int = struct.Struct('i')
self.max_level = (2.0**31)-1
self.gen_functions = (
self.sine_function,
self.triangle_function,
self.square_function,
self.sawtooth_function,
self.equation_import_function
)
self.main_color = gtk.gdk.color_parse('#c04040')
self.sig_color = gtk.gdk.color_parse('#40c040')
self.mod_color = gtk.gdk.color_parse('#4040c0')
self.noise_color = gtk.gdk.color_parse('#c040c0')
self.pipeline = False
self.count = 0
self.imod = 0
self.rate = 1
self.mod_key_val = 0
self.sig_freq = 440
self.mod_freq = 3
self.sig_level = 100
self.mod_level = 100
self.noise_level = 100
self.enable = True
self.sig_waveform = SignalGen.W_SINE
self.sig_enable = True
self.sig_function = False
self.mod_waveform = SignalGen.W_SINE
self.mod_function = False
self.mod_mode = SignalGen.M_AM
self.mod_enable = False
self.noise_enable = False
self.sample_rate = SignalGen.R_22050
self.left_audio = True
self.right_audio = True
self.program_name = self.__class__.__name__
self.config_file = os.path.expanduser("~/." + self.program_name)
self.builder = gtk.Builder()
self.xmlfile = 'signalgen_gui.glade'
self.builder.add_from_file(self.xmlfile)
WidgetFinder().localize_widgets(self,self.xmlfile)
self.k_quit_button.connect('clicked',self.close)
self.k_help_button.connect('clicked',self.launch_help)
self.k_mainwindow.connect('destroy',self.close)
self.k_mainwindow.set_icon(gtk.gdk.pixbuf_new_from_xpm_data(Icon.icon))
self.title = self.program_name + ' ' + VERSION
self.k_mainwindow.set_title(self.title)
self.tooltips = {
self.k_sample_rate_combobox : 'Change data sampling rate',
self.k_left_checkbutton : 'Enable left channel audio',
self.k_right_checkbutton : 'Enable right channel audio',
self.k_sig_waveform_combobox : 'Select signal waveform',
self.k_mod_waveform_combobox : 'Select modulation waveform',
self.k_mod_enable_checkbutton : 'Enable modulation',
self.k_sig_enable_checkbutton : 'Enable signal',
self.k_noise_enable_checkbutton : 'Enable white noise',
self.k_mod_am_radiobutton : 'Enable amplitude modulation',
self.k_mod_fm_radiobutton : 'Enable frequency modulation',
self.k_quit_button : 'Quit %s' % self.title,
self.k_enable_checkbutton : 'Enable output',
self.k_help_button : 'Visit the %s Web page' % self.title,
}
for k,v in self.tooltips.iteritems():
k.set_tooltip_text(v)
self.config_data = {
'SampleRate' : self.k_sample_rate_combobox,
'LeftChannelEnabled' : self.k_left_checkbutton,
'RightChannelEnabled' : self.k_right_checkbutton,
'SignalWaveform' : self.k_sig_waveform_combobox,
'SignalFrequency' : self.k_sig_freq_entry,
'SignalLevel' : self.k_sig_level_entry,
'SignalEnabled' : self.k_sig_enable_checkbutton,
'ModulationWaveform' : self.k_mod_waveform_combobox,
'ModulationFrequency' : self.k_mod_freq_entry,
'ModulationLevel' : self.k_mod_level_entry,
'ModulationEnabled' : self.k_mod_enable_checkbutton,
'AmplitudeModulation' : self.k_mod_am_radiobutton,
'FrequencyModulation' : self.k_mod_fm_radiobutton,
'NoiseEnabled' : self.k_noise_enable_checkbutton,
'NoiseLevel' : self.k_noise_level_entry,
'OutputEnabled' : self.k_enable_checkbutton,
}
self.cm = ConfigManager(self.config_file,self.config_data)
self.cm.load_combobox(self.k_sig_waveform_combobox,self.waveform_strings)
self.k_sig_waveform_combobox.set_active(self.sig_waveform)
self.cm.load_combobox(self.k_mod_waveform_combobox,self.waveform_strings)
self.k_mod_waveform_combobox.set_active(self.mod_waveform)
self.cm.load_combobox(self.k_sample_rate_combobox,self.sample_rates)
self.k_sample_rate_combobox.set_active(self.sample_rate)
self.k_sig_freq_entry.set_text(self.format_num(self.sig_freq))
self.k_sig_level_entry.set_text(self.format_num(self.sig_level))
self.k_mod_freq_entry.set_text(self.format_num(self.mod_freq))
self.k_mod_level_entry.set_text(self.format_num(self.mod_level))
self.k_noise_level_entry.set_text(self.format_num(self.noise_level))
self.k_main_viewport_border.modify_bg(gtk.STATE_NORMAL,self.main_color)
self.k_sig_viewport_border.modify_bg(gtk.STATE_NORMAL,self.sig_color)
self.k_mod_viewport_border.modify_bg(gtk.STATE_NORMAL,self.mod_color)
self.k_noise_viewport_border.modify_bg(gtk.STATE_NORMAL,self.noise_color)
self.sig_freq_cont = TextEntryController(self,self.k_sig_freq_entry)
self.sig_level_cont = TextEntryController(self,self.k_sig_level_entry)
self.mod_freq_cont = TextEntryController(self,self.k_mod_freq_entry)
self.mod_level_cont = TextEntryController(self,self.k_mod_level_entry)
self.noise_level_cont = TextEntryController(self,self.k_noise_level_entry)
self.k_mainwindow.connect('key-press-event',self.key_event)
self.k_mainwindow.connect('key-release-event',self.key_event)
self.k_enable_checkbutton.connect('toggled',self.update_values)
self.k_sig_freq_entry.connect('changed',self.update_entry_values)
self.k_sig_level_entry.connect('changed',self.update_entry_values)
self.k_sig_enable_checkbutton.connect('toggled',self.update_checkbutton_values)
self.k_mod_freq_entry.connect('changed',self.update_entry_values)
self.k_mod_level_entry.connect('changed',self.update_entry_values)
self.k_noise_level_entry.connect('changed',self.update_entry_values)
self.k_sample_rate_combobox.connect('changed',self.update_values)
self.k_sig_waveform_combobox.connect('changed',self.update_values)
self.k_mod_waveform_combobox.connect('changed',self.update_values)
self.k_left_checkbutton.connect('toggled',self.update_checkbutton_values)
self.k_right_checkbutton.connect('toggled',self.update_checkbutton_values)
self.k_mod_enable_checkbutton.connect('toggled',self.update_checkbutton_values)
self.k_noise_enable_checkbutton.connect('toggled',self.update_checkbutton_values)
self.k_mod_am_radiobutton.connect('toggled',self.update_checkbutton_values)
self.cm.read_config()
self.update_entry_values()
self.update_checkbutton_values()
self.update_values()
def format_num(self,v):
return "%.2f" % v
def get_widget_text(self,w):
typ = type(w)
if(typ == gtk.ComboBox):
return w.get_active_text()
elif(typ == gtk.Entry):
return w.get_text()
def get_widget_num(self,w):
try:
return float(self.get_widget_text(w))
except:
return 0.0
def restart_test(self,w,pv):
nv = w.get_active()
self.restart |= (nv != pv)
return nv
def update_entry_values(self,*args):
self.sig_freq = self.get_widget_num(self.k_sig_freq_entry)
self.sig_level = self.get_widget_num(self.k_sig_level_entry) / 100.0
self.mod_freq = self.get_widget_num(self.k_mod_freq_entry)
self.mod_level = self.get_widget_num(self.k_mod_level_entry) / 100.0
self.noise_level = self.get_widget_num(self.k_noise_level_entry) / 100.0
def update_checkbutton_values(self,*args):
self.left_audio = self.k_left_checkbutton.get_active()
self.right_audio = self.k_right_checkbutton.get_active()
self.mod_enable = self.k_mod_enable_checkbutton.get_active()
self.sig_enable = self.k_sig_enable_checkbutton.get_active()
self.mod_mode = (SignalGen.M_FM,SignalGen.M_AM)[self.k_mod_am_radiobutton.get_active()]
self.noise_enable = self.k_noise_enable_checkbutton.get_active()
def update_values(self,*args):
self.restart = (not self.sig_function)
self.sample_rate = self.restart_test(self.k_sample_rate_combobox, self.sample_rate)
self.enable = self.restart_test(self.k_enable_checkbutton,self.enable)
self.mod_waveform = self.k_mod_waveform_combobox.get_active()
self.mod_function = self.gen_functions[self.mod_waveform]
self.sig_waveform = self.k_sig_waveform_combobox.get_active()
self.sig_function = self.gen_functions[self.sig_waveform]
self.k_sample_rate_combobox.set_sensitive(not self.enable)
if(self.restart):
self.init_audio()
def make_and_chain(self,name):
target = gst.element_factory_make(name)
self.chain.append(target)
return target
def unlink_gst(self):
if(self.pipeline):
self.pipeline.set_state(gst.STATE_NULL)
self.pipeline.remove_many(*self.chain)
gst.element_unlink_many(*self.chain)
for item in self.chain:
item = False
self.pipeline = False
time.sleep(0.01)
def init_audio(self):
self.unlink_gst()
if(self.enable):
self.chain = []
self.pipeline = gst.Pipeline("mypipeline")
self.source = self.make_and_chain("appsrc")
rs = SignalGen.sample_rates[self.sample_rate]
self.rate = float(rs)
self.interval = 1.0 / self.rate
caps = gst.Caps(
'audio/x-raw-int,'
'endianness=(int)1234,'
'channels=(int)2,'
'width=(int)32,'
'depth=(int)32,'
'signed=(boolean)true,'
'rate=(int)%s' % rs)
self.source.set_property('caps', caps)
self.sink = self.make_and_chain("autoaudiosink")
self.pipeline.add(*self.chain)
gst.element_link_many(*self.chain)
self.source.connect('need-data', self.need_data)
self.pipeline.set_state(gst.STATE_PLAYING)
def key_event(self,w,evt):
cn = gtk.gdk.keyval_name(evt.keyval)
if(re.search('Shift',cn) != None):
mod = 1
elif(re.search('Control',cn) != None):
mod = 2
elif(re.search('Alt|Meta',cn) != None):
mod = 4
else:
return
if(evt.type == gtk.gdk.KEY_PRESS):
self.mod_key_val |= mod
else:
self.mod_key_val &= ~mod
def sine_function(self,t,f):
return math.sin(2.0*math.pi*f*t)
def triangle_function(self,t,f):
q = 4*math.fmod(t*f,1)
q = (q,2-q)[q > 1]
return (q,-2-q)[q < -1]
def square_function(self,t,f):
if(f == 0): return 0
q = 0.5 - math.fmod(t*f,1)
return (-1,1)[q > 0]
def sawtooth_function(self,t,f):
return 2.0*math.fmod((t*f)+0.5,1.0)-1.0
def equation_import_function(self,t,f):
fileobj=open("/home/rat/eq1.txt","r")
eqdata =fileobj.read() #read whole file
fileobj.close()
#return math.tan(2.0*math.pi*f*t)
return eqdata
def need_data(self,src,length):
bytes = ""
# sending two channels, so divide requested length by 2
ld2 = length / 2
for tt in range(ld2):
t = (self.count + tt) * self.interval
if(not self.mod_enable):
datum = self.sig_function(t,self.sig_freq)
else:
mod = self.mod_function(t,self.mod_freq)
# AM mode
if(self.mod_mode == SignalGen.M_AM):
datum = 0.5 * self.sig_function(t,self.sig_freq) * (1.0 + (mod * self.mod_level))
# FM mode
else:
self.imod += (mod * self.mod_level * self.interval)
datum = self.sig_function(t+self.imod,self.sig_freq)
v = 0
if(self.sig_enable):
v += (datum * self.sig_level)
if(self.noise_enable):
noise = ((2.0 * random.random()) - 1.0)
v += noise * self.noise_level
v *= self.max_level
v = max(-self.max_level,v)
v = min(self.max_level,v)
left = (0,v)[self.left_audio]
right = (0,v)[self.right_audio]
bytes += self.struct_int.pack(left)
bytes += self.struct_int.pack(right)
self.count += ld2
src.emit('push-buffer', gst.Buffer(bytes))
def launch_help(self,*args):
webbrowser.open("http://arachnoid.com/python/signalgen_program.html")
def close(self,*args):
self.unlink_gst()
self.cm.write_config()
gtk.main_quit()
app=SignalGen()
gtk.main()

The imp module will help you to cleanly load Python code chunks from arbitrary files.
#!/usr/bin/env python
# equation in equation-one.py
def eqn(arg):
return arg * 3 + 2
#!/usr/bin/env python
# your code
import imp
path = "equation-one.py"
eq_mod = imp.load_source("equation", path, open(path))
print("Oh the nice stuff in eq_mod: %s" % dir(eq_mod))
In your custom function definition, you can create a file selector dialog, get the selected file path, load the code using imp, and return the result of the function inside the imported module.

I was commenting before, but I stared at your code long enough and kinda realized what you were trying to do, so it was easier for me to post an answer. Please refer to cJ Zougloubs answer as I expand on his suggestion to use the imp module.
Your equation files should implement a common interface:
# equation1.py
def eqn(*args):
return sum(*args)
Then you would load them in using cj Zougloubs suggestion, but with a common interface:
# python_rt.py
def equation_import_function(self, *args):
filepath = ''
# filepath = ... do file chooser dialog here ...
eq_mod = imp.load_source("equation", filepath)
eqdata = eq_mod.eqn(*args)
return eqdata
Now you have a function in your main code that takes any number of arguments, asks the user to pick the equation file, and gets the result for you.
Edit To address your comment more specifically
# equation1.py
import math
def eqn(*args):
f = args[0]
t = args[1]
return math.tan(2.0*math.pi*f*t)
And in your main tool, you would use imp.load_source to bring it in. Wherever you needed that equation for your audio, you could then do:
eq_mod.eqn(f, t)

Categories