We Keep Coding

Data in pickle format not loading into DataFrame

I got this code that has data stored in pickle format, I am trying to unpickle it. After it successfully ran, it was returning this output: <__main__.Container at 0x232ac6bde80>, but I want the data loaded into the DataFrame for further manipulations. I have searched StackOverflow for possible solutions to no avail. Kindly help. Here is the code:
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import rcParams
impor numpy as np
import tpickle
import skill_metrics as sm
from sys import version_info
def load_obj(name):
# Load object from file in pickle format
if version_info[0] == 2:
suffix = 'pkl'
else:
suffix = 'pkl3'
with open(name + '.' + suffix, 'rb') as f:
return pickle.load(f) # Python2 succeeds
class Container(object):
def __init__(self, pred1, pred2, pred3, ref):
self.pred1 = pred1
self.pred2 = pred2
self.pred3 = pred3
self.ref = ref
if __name__ == '__main__':
# Set the figure properties (optional)
rcParams["figure.figsize"] = [8.0, 6.4] #works
# rcParams['lines.linewidth'] = 2 # line width for plots
rcParams.update({'font.size': 12}) # font size of axes text
# Close any previously open graphics windows
plt.close("all")
# Read data from pickle file
load_obj('target_data')
df = pd.read_pickle('target_data.pkl')
df
Thanks.

Convert blob from csv, imported as string

I am trying to read Blob-Data which is stored in a csv-file. I can import it in string format but if I want to use numpy.frombuffer with dtype='<f4' (fixed to get correct output), I get the error:
line 52, in <module>
data = np.frombuffer(blob_data, dtype='<f4') #<f4
ValueError: buffer size must be a multiple of element size
The code is the following:
import numpy as np
import datetime
import math
import csv
import pandas
from binascii import unhexlify
#import mysql.connector
# from pylab import figure, plot, show, legend, xlabel, ylabel
from matplotlib import pyplot as plt
def read_CSV(dataid):
with open('spectrometer2.csv') as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
line_count = 0
result = bytes('1',encoding='UTF-8')
for row in csv_reader:
if line_count > 0:
#print(row[0])
if str(dataid) == str(row[0]):
result = row[3][2:-1]
print('FOUND####################################')
print(str(row[3])[2:-1])
break
line_count += 1
print(str(type(result)))
return result
####### MAIN PROGRAM #######
#spectrumRange = np.arange(10, 11011, 200)
spectrumRange = np.arange(8000, 9000, 200) # Auszug zur Leistungsoptimierung im Test
query_init = "SELECT * FROM `spectrometer2` WHERE data_id="
plt.figure()
for spectrum_id in spectrumRange:
spectrometerquery = query_init + str(spectrum_id) + ";"
print("Current data ID: " + str(spectrum_id))
#y_stream = interact_with_MySQL("database_name", spectrometerquery)
blob_data = read_CSV(spectrum_id)
#print(binary2int(blob_data))
if(blob_data != 0):
blob_data = bytes(blob_data,encoding = 'UTF-8')
print(blob_data)
data = np.frombuffer(blob_data, dtype='<f4') #<f4
print(data)
plt.plot(data)
plt.title('Spectrometer 2 data')
legend = []
for x in spectrumRange:
legend.append(('data id:', x))
plt.legend(legend)
plt.show()
I don't know much about the csv-file but it should contain the output of a optical sensor. The sql-statements are commented out because I have to replace them with the csv-file.
Extract of the file (shown in excel):

"Global variable is undefined at the module level" is the error I get but it only happend when I run the program from pycharm and not from Jupyter

import wordcloud
import numpy as np
from matplotlib import pyplot as plt
from IPython.display import display
import fileupload
import io
import sys
def _upload():
_upload_widget = fileupload.FileUploadWidget()
def _cb(change):
global file_contents
decoded = io.StringIO(change['owner'].data.decode('utf-8'))
filename = change['owner'].filename
print('Uploaded `{}` ({:.2f} kB)'.format(
filename, len(decoded.read()) / 2 **10))
file_contents = decoded.getvalue()
_upload_widget.observe(_cb, names='data')
display(_upload_widget)
_upload()
...
myimage = calculate_frequencies(file_contents)
plt.imshow(myimage, interpolation = 'nearest')
plt.axis('off')
plt.show()
This is the error I get on pycharm:
myimage = calculate_frequencies(file_contents)
NameError: name 'file_contents' is not defined
FileUploadWidget(label='Browse', _dom_classes=('widget_item', 'btn-group'))

Displaying matplotlib plot in CGI with cStringIO

Trying to return a simple plot (saved in StringIO) to web browser. After hours of reading, finally getting close, maybe.
import cgi
import cStringIO
import matplotlib.pyplot as plt
import numpy as np
def doit():
x = np.linspace(-2,2,100)
y = np.sin(x)
format = "png"
ggg = cStringIO.StringIO()
plt.plot(x, y)
plt.savefig(ggg, format=format)
data_uri = ggg.read().encode('base64').replace('\n', '')
img_tag = '<img src="data:image/png;base64,{0}" alt="thisistheplot"/>'.format(data_uri)
print("Content-type: text/html\n")
print("<title>Try Ageen</title>")
print("<h1>Hi</h1>")
print(img_tag)
doit()
It's returning a broken image icon. I've looked already read this: Dynamically serving a matplotlib image to the web using python, among others...

Actually, just figured it out. Will leave posted, as I haven't seen this approach being taken and hope it can help:
#!/usr/bin/env python
import cgi
import cStringIO
import matplotlib.pyplot as plt
import numpy as np
def doit():
x = np.linspace(-2,2,100)
y = np.sin(x)
format = "png"
sio = cStringIO.StringIO()
plt.plot(x, y)
plt.savefig(sio, format=format)
data_uri = sio.getvalue().encode('base64').replace('\n', '')
img_tag = '<img src="data:image/png;base64,{0}" alt="sucka" />'.format(data_uri)
print("Content-type: text/html\n")
print("<title>Try Ageen</title>")
print("<h1>Hi</h1>")
print(img_tag)
doit()
The goal is to have a client input a trigonometric function and have it returned on a subsequent page. Still any comments to help this code perform/look better are welcomed.

How to store an image in a variable

I would like to store the image generated by matplotlib in a variable raw_data to use it as inline image.
import os
import sys
os.environ['MPLCONFIGDIR'] = '/tmp/'
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
print "Content-type: image/png\n"
plt.plot(range(10, 20))
raw_data = plt.show()
if raw_data:
uri = 'data:image/png;base64,' + urllib.quote(base64.b64encode(raw_data))
print '<img src = "%s"/>' % uri
else:
print "No data"
#plt.savefig(sys.stdout, format='png')
None of the functions suit my use case:
plt.savefig(sys.stdout, format='png') - Writes it to stdout. This does help.. as I have to embed the image in a html file.
plt.show() / plt.draw() does nothing when executed from command line

Have you tried cStringIO or an equivalent?
import os
import sys
import matplotlib
import matplotlib.pyplot as plt
import StringIO
import urllib, base64
plt.plot(range(10, 20))
fig = plt.gcf()
imgdata = StringIO.StringIO()
fig.savefig(imgdata, format='png')
imgdata.seek(0) # rewind the data
print "Content-type: image/png\n"
uri = 'data:image/png;base64,' + urllib.quote(base64.b64encode(imgdata.buf))
print '<img src = "%s"/>' % uri

Complete python 3 version, putting together Paul's answer and metaperture's comment.
import matplotlib.pyplot as plt
import io
import urllib, base64
plt.plot(range(10))
fig = plt.gcf()
buf = io.BytesIO()
fig.savefig(buf, format='png')
buf.seek(0)
string = base64.b64encode(buf.read())
uri = 'data:image/png;base64,' + urllib.parse.quote(string)
html = '<img src = "%s"/>' % uri