I have this "flask app" with two links, each mapping to different matplotlib visualizations, for example: localhost:5000/line_chart and localhost:5000/bar_chart.
When I start the server, and click the a route (any of them), I see what I expect.
localhost:5000/bar_chart
When I go back and view the other link, both graphs break.
localhost:5000/line_chart
localhost:5000/bar_chart
I can reproduce this every time by closing the server then running the "run.py" script again. Seems to be an overwriting conflict with the in-memory buffer. Has anyone had this issue before?
app/views.py
import matplotlib
matplotlib.use('Agg') # this allows PNG plotting
import matplotlib.pyplot as plt
import base64
from flask import render_template
from app import app
from io import BytesIO
#app.route('/')
#app.route('/index')
def index():
res = ''
navigation = [['Line Chart','line_chart'],['Bar Chart','bar_chart']]
res = res + '<h1>Matplotlib Chart Examples</h1>'
res = res + '<ul>'
for item in navigation:
name = item[0]
link = item[1]
res = res + '<li>'+ name +'</li>'
res = res +'</ul>'
return res
#app.route('/bar_chart')
def bar_chart():
movies = ["Annie Hall", "Ben-Hur", "Casablanca", "Gandhi", "West Side Story"]
num_oscars = [5, 11, 3, 8, 10]
# bars are by default width 0.8, so we'll add 0.1 to the left coordinates
# so that each bar is centered
xs = [i + 0.1 for i, _ in enumerate(movies)]
# plot bars with left x-coordinates [xs], heights [num_oscars]
plt.bar(xs, num_oscars)
plt.ylabel("# of Academy Awards")
plt.title("My Favorite Movies")
# label x-axis with movie names at bar centers
plt.xticks([i + 0.5 for i, _ in enumerate(movies)], movies)
return compute(plt)
#app.route('/line_chart')
def line_chart():
years = [1950, 1960, 1970, 1980, 1990, 2000, 2010]
gdp = [300.2, 543.3, 1075.9, 2862.5, 5979.6, 10289.7, 14958.3]
# create a line chart, years on x-axis, gdp on y-axis
plt.plot(years, gdp, color='green', marker='o', linestyle='solid')
# add a title
plt.title("Nominal GDP")
# add a label to the y-axis
plt.ylabel("Billions of $")
return compute(plt)
def compute(plt):
# run plt.plot, plt.title, etc.
figfile = BytesIO()
plt.savefig(figfile, format='png')
figfile.seek(0) # rewind to beginning of file
#figfile.getvalue() extracts string (stream of bytes)
figdata_png = base64.b64encode(figfile.getvalue())
return render_template('index.html',
title='matplotlib chart',
results=figdata_png)
Thank you for your time.
I guess you need two figures, test this code and tell what happened:
#app.route('/bar_chart')
def bar_chart():
movies = ["Annie Hall", "Ben-Hur", "Casablanca", "Gandhi", "West Side Story"]
num_oscars = [5, 11, 3, 8, 10]
# bars are by default width 0.8, so we'll add 0.1 to the left coordinates
# so that each bar is centered
xs = [i + 0.1 for i, _ in enumerate(movies)]
# plot bars with left x-coordinates [xs], heights [num_oscars]
plt.figure(1)
plt.bar(xs, num_oscars)
plt.ylabel("# of Academy Awards")
plt.title("My Favorite Movies")
# label x-axis with movie names at bar centers
plt.xticks([i + 0.5 for i, _ in enumerate(movies)], movies)
return compute(plt, 1)
#app.route('/line_chart')
def line_chart():
years = [1950, 1960, 1970, 1980, 1990, 2000, 2010]
gdp = [300.2, 543.3, 1075.9, 2862.5, 5979.6, 10289.7, 14958.3]
# create a line chart, years on x-axis, gdp on y-axis
plt.figure(2)
plt.plot(years, gdp, color='green', marker='o', linestyle='solid')
# add a title
plt.title("Nominal GDP")
# add a label to the y-axis
plt.ylabel("Billions of $")
return compute(plt,2)
def compute(plt, fignum):
# run plt.plot, plt.title, etc.
plt.figure(fignum)
figfile = BytesIO()
plt.savefig(figfile, format='png')
figfile.seek(0) # rewind to beginning of file
#figfile.getvalue() extracts string (stream of bytes)
figdata_png = base64.b64encode(figfile.getvalue())
return render_template('index.html',
title='matplotlib chart',
results=figdata_png)
In my case, that solution didn't work. It seems that there is a race condition when trying to access plot. I first tried to use a lock from a library, but that didn't work, so instead I sort of engineered out a lock. In my case, I wanted to create n images using the same function on the same view, so I started by creating a list in the following way:
queue = [False for i in range(n)]
Then, my flask app look something like this:
#app.route('/vis/<j>')
def vis(j):
global queue
# We check that it's image's #j turn, as if it was single threaded
j = int(j)
if j == 0:
for i in range(len(queue)):
queue[i] = False
else:
while not queue[j-1]:
# If it's not, we sleep for a short time (from time import sleep)
sleep(0.5)
# This is not important, it's how I was plotting some random figures
# (from random import seed) (from datetime import datetime)
seed(datetime.now())
n = 10
p1 = [randint(0, 10) for _ in range(n)]
p2 = [randint(0, 10) for _ in range(n)]
t = [i for i in range(n)]
fig = plt.figure(j)
plt.clf()
plt.plot(t, p1, color='blue')
plt.plot(t, p2, color='orange')
plt.xlabel('Time')
plt.ylabel('Value')
# Save the plot
img = BytesIO()
fig.savefig(img, dpi=128)
img.seek(0)
# We finished using everything related to plot, so we free the "lock"
queue[j] = True
# Return the object as a file that can be accessed
return send_file(img, mimetype='image/png')
Finally, when wanting to display this in my flask app, all I had to do was using this <img src="/vis/1"> in my html file.
Edit: I forgot one of the most important part! For some reason, this would still create some unrelated thread issue. I looked it up and that's when I came with the full solution. The threading issue was solved by adding at the beginning of the file:
import matplotlib
import matplotlib.pyplot as plt
matplotlib.use('Agg')
For some reason, using that Agg backend solved the second threading I was having. I don't really have a good explanation for that, but it does work, so it's enough for me.
Alternatively, what also worked was running the app disabling threads by adding:
if __name__ == '__main__':
app.run(threading=False, debug=True)
I don't know however, at the moment, whether this works in production, so I preferred the other solution. :)
I hope this helps if you had the same issue!
Related
In Short:
I want to change the color of blue marker in the graph. So that I can do comparison with other plots easily.
You can download the data files and script from this link
Problem Explanation
I have two data files, full.dat and part.dat(Note: part.dat is also there in full.dat).
I got the plotting scripts from the internet, and it is working very well. But as a noob in Python and Matplotlib, I am facing difficulties in changing the color of part.dat.
Please see the graph first, then the following scripts.
Script-1: Function and definitions: let's say: "func.py"
# This was written by Levi Lentz for the Kolpak Group at MIT
import numpy as np
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
import matplotlib.gridspec as gs
import sys
#This function extracts the high symmetry points from the output of bandx.out
def Symmetries(fstring):
f = open(fstring,'r')
x = np.zeros(0)
for i in f:
if "high-symmetry" in i:
x = np.append(x,float(i.split()[-1]))
f.close()
return x
# This function takes in the datafile, the fermi energy, the symmetry file, a subplot, and the label
# It then extracts the band data, and plots the bands, the fermi energy in red, and the high symmetry points
def bndplot(datafile_full,datafile,fermi,symmetryfile,subplot,**kwargs):
if 'shift_fermi' in kwargs:
bool_shift_efermi = kwargs['shift_fermi']
else:
bool_shift_efermi = 0
if 'color' in kwargs:
color_bnd=kwargs['color']
else:
color_bnd='black'
if 'linestyle' in kwargs:
line_bnd=kwargs['linestyle']
else:
line_bnd='solid'
z = np.loadtxt(datafile_full) #This loads the full.dat file
x = np.unique(z[:,0]) #This is all the unique x-points
[a,b,w]=np.loadtxt(datafile,unpack=True) #Weight
bands = []
bndl = len(z[z[:,0]==x[1]]) #This gives the number of bands in the calculation
Fermi = float(fermi)
if bool_shift_efermi:
fermi_shift=Fermi
else:
fermi_shift=0
axis = [min(x),max(x)]
for i in range(0,bndl):
bands.append(np.zeros([len(x),2])) #This is where we storre the bands
for i in range(0,len(x)):
sel = z[z[:,0] == x[i]] #Here is the energies for a given x
test = []
for j in range(0,bndl): #This separates it out into a single band
bands[j][i][0] = x[i]
#bands[j][i][1] = np.multiply(sel[j][1],13.605698066)
bands[j][i][1] = sel[j][1]
#Here we plots the bands
for i in bands:
subplot.plot(i[:,0],i[:,1]-fermi_shift,color=color_bnd,linestyle=line_bnd, linewidth=0.7,alpha=0.5)
# plt.scatter(a,b-fermi_shift,c=w,cmap='viridis',alpha=0.5)
# plt.colorbar()
if 'legend' in kwargs:
#empty plot to generate legend
subplot.plot([None],[None],color=color_bnd,linestyle=line_bnd,label=kwargs['legend'])
temp = Symmetries(symmetryfile)
for j in temp: #This is the high symmetry lines
x1 = [j,j]
subplot.axvline(x=j,linestyle='dashed',color='black',alpha=0.75)
subplot.plot([min(x),max(x)],[Fermi-fermi_shift,Fermi-fermi_shift],color='red',linestyle='dotted')
subplot.set_xticks(temp)
subplot.set_xticklabels([])
if 'name_k_points' in kwargs:
if len(kwargs['name_k_points'])==len(temp):
subplot.set_xticklabels(kwargs['name_k_points'])
if 'range' in kwargs:
range_plot=kwargs['range']
subplot.set_ylim([range_plot[0],range_plot[1]])
subplot.set_xlim([axis[0],axis[1]])
subplot.set_xlabel('k')
subplot.set_ylabel('E-E$_f$')
plt.scatter(a,b-fermi_shift,s=70*np.array(w))
if 'legend' in kwargs:
plt.legend()
script-2 Plotting script: let's say: "plot.py"
#!/usr/bin/python3
from func import *
El='el'
orb='orb'
plt.rcParams["figure.figsize"]=(4,15)
datafile_full='bands.dat.gnu'
#datafile=El+'_'+orb+'.dat.all'
datafile=El+'_'+orb+'.dat.all'
fermi = 10.2382
symmetryfile='band.out'
bool_shift_efermi= True
fig, ax = plt.subplots()
#bndplot(datafile,fermi,symmetryfile,ax)
bndplot(datafile_full,datafile,fermi,symmetryfile,ax,shift_fermi=1,color='black',linestyle='solid',name_k_points=['K','G','M','K','H','A','L','H'], legend=El+', '+orb+'-orbital')
#ax.set_ylim(-5,5)
ax.set_ylim(-10,12)
fig.set_figheight(6)
fig.set_figwidth(4)
plt.rcParams.update({'font.size': 22})
fig.savefig("el-orb.eps")
plt.show()
In script-2, there is an option to change the color, however I want to change the color of blue marker/solid-circles(please see the graph) so that I can compare with other graphs.
Whenever I change the color, it changes the line color only.
Please help me out I am trying to understand Matplotlib uses and examples from past few hrs However as a noob I was not able to figure out how to do.
I am working on a Python application where I am collecting data from a device, and attempting to plot it in an excel file by using the Openpyxl library. I am successfully able to do everything including plotting the data, and formatting the scatter plot that I made, but I am having some trouble in adding minor gridlines to the plot.
I feel like this is definitely possible because in the API, I can see under the openpyxl.chart.axis module, there is a “minorGridlines” attribute, but it is not a boolean input (ON/OFF), rather it takes a Chartlines class. I tried going a bit down the rabbit-hole of seeing how I would do this, but I am wondering what the most straightforward way of adding the minor-gridlines would be? Do you have to construct chart lines manually, or is there a simple way of doing this?
I would really appreciate any help!
I think I answered my own question, but I will post it here if anybody else needs this (as I don’t see any other answers to this question on the forum).
Example Code (see lines 4, 38):
# Imports for script
from openpyxl import Workbook # For plotting things in excel
from openpyxl.chart import ScatterChart, Reference, Series
from openpyxl.chart.axis import ChartLines
from math import log10
# Variables for script
fileName = 'testFile.xlsx'
dataPoints = 100
# Generating a workbook to test with
wb = Workbook()
ws = wb.active # Fill data into the first sheet
ws_name = ws.title
# We will just generate a logarithmic plot, and scale the axis logarithmically (will look linear)
x_data = []
y_data = []
for i in range(dataPoints):
x_data.append(i + 1)
y_data.append(log10(i + 1))
# Go back through the data, and place the data into the sheet
ws['A1'] = 'x_data'
ws['B1'] = 'y_data'
for i in range(dataPoints):
ws['A%d' % (i + 2)] = x_data[i]
ws['B%d' % (i + 2)] = y_data[i]
# Generate a reference to the cells that we can plot
x_axis = Reference(ws, range_string='%s!A2:A%d' % (ws_name, dataPoints + 1))
y_axis = Reference(ws, range_string='%s!B2:B%d' % (ws_name, dataPoints + 1))
function = Series(xvalues=x_axis, values=y_axis)
# Actually create the scatter plot, and append all of the plots to it
ScatterPlot = ScatterChart()
ScatterPlot.x_axis.minorGridlines = ChartLines()
ScatterPlot.x_axis.scaling.logBase = 10
ScatterPlot.series.append(function)
ScatterPlot.x_axis.title = 'X_Data'
ScatterPlot.y_axis.title = 'Y_Data'
ScatterPlot.title = 'Openpyxl Plotting Test'
ws.add_chart(ScatterPlot, 'D2')
# Save the file at the end to output it
wb.save(fileName)
Background on solution:
I looked at how the code for Openpyxl generates the Major axis gridlines, which seems to follow a similar convention as the Minor axis gridlines, and I found that in the ‘NumericAxis’ class, they generated the major gridlines with the following line (labeled ‘##### This Line #####’ which is originally copied from the ‘openpyxl->chart->axis’ file):
class NumericAxis(_BaseAxis):
tagname = "valAx"
axId = _BaseAxis.axId
scaling = _BaseAxis.scaling
delete = _BaseAxis.delete
axPos = _BaseAxis.axPos
majorGridlines = _BaseAxis.majorGridlines
minorGridlines = _BaseAxis.minorGridlines
title = _BaseAxis.title
numFmt = _BaseAxis.numFmt
majorTickMark = _BaseAxis.majorTickMark
minorTickMark = _BaseAxis.minorTickMark
tickLblPos = _BaseAxis.tickLblPos
spPr = _BaseAxis.spPr
txPr = _BaseAxis.txPr
crossAx = _BaseAxis.crossAx
crosses = _BaseAxis.crosses
crossesAt = _BaseAxis.crossesAt
crossBetween = NestedNoneSet(values=(['between', 'midCat']))
majorUnit = NestedFloat(allow_none=True)
minorUnit = NestedFloat(allow_none=True)
dispUnits = Typed(expected_type=DisplayUnitsLabelList, allow_none=True)
extLst = Typed(expected_type=ExtensionList, allow_none=True)
__elements__ = _BaseAxis.__elements__ + ('crossBetween', 'majorUnit',
'minorUnit', 'dispUnits',)
def __init__(self,
crossBetween=None,
majorUnit=None,
minorUnit=None,
dispUnits=None,
extLst=None,
**kw
):
self.crossBetween = crossBetween
self.majorUnit = majorUnit
self.minorUnit = minorUnit
self.dispUnits = dispUnits
kw.setdefault('majorGridlines', ChartLines()) ######## THIS Line #######
kw.setdefault('axId', 100)
kw.setdefault('crossAx', 10)
super(NumericAxis, self).__init__(**kw)
#classmethod
def from_tree(cls, node):
"""
Special case value axes with no gridlines
"""
self = super(NumericAxis, cls).from_tree(node)
gridlines = node.find("{%s}majorGridlines" % CHART_NS)
if gridlines is None:
self.majorGridlines = None
return self
I took a stab, and after importing the ‘Chartlines’ class like so:
from openpyxl.chart.axis import ChartLines
I was able to add minor gridlines to the x-axis like so:
ScatterPlot.x_axis.minorGridlines = ChartLines()
As far as formatting the minor gridlines, I’m at a bit of a loss, and personally have no need, but this at least is a good start.
I'm having an issue exactly like this post, but slightly more frustrating.
I'm using matplotlib to read from a file that is being fed data from another application. For some reason, the ends of the data only connect after the animation (animation.FuncAnimation) has completed its first refresh. Here are some images:
This is before the refresh:
And this is after the refresh:
Any ideas as to why this could be happening? Here is my code:
import json
import itertools
import dateutil.parser
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib import style
import scipy.signal as sig
import numpy as np
import pylab as plt
sensors = {}
data = []
lastLineReadNum = 0
class Sensor:
def __init__(self, name, points = 0, lastReading = 0):
self.points = points
self.lastReading = lastReading
self.name = name
self.x = []
self.y = []
class ScanResult:
def __init__(self, name, id, rssi, macs, time):
self.name = name
self.id = id
self.rssi = rssi
self.macs = macs
# Is not an integer, but a datetime.datetime
self.time = time
def readJSONFile(filepath):
with open(filepath, "r") as read_file:
global lastLineReadNum
# Load json results into an object that holds important info
for line in itertools.islice(read_file, lastLineReadNum, None):
temp = json.loads(line)
# Only reads the most recent points...
data.append(ScanResult(name = temp["dev_id"],
id = temp["hardware_serial"],
rssi = temp["payload_fields"]["rssis"],
macs = temp["payload_fields"]["macs"],
time = dateutil.parser.parse(temp["metadata"]["time"])))
lastLineReadNum += 1
return data
style.use('fivethirtyeight')
fig = plt.figure()
ax1 = fig.add_subplot(1, 1, 1)
def smooth(y, box_pts):
box = np.ones(box_pts)/box_pts
y_smooth = np.convolve(y, box, mode='same')
return y_smooth
def determineClosestSensor():
global sensors
#sensors.append(Sensor(time = xs3, rssi = ys3))
def determineXAxisTime(scanresult):
return ((scanresult.time.hour * 3600) + (scanresult.time.minute * 60) + (scanresult.time.second)) / 1000.0
def animate(i):
data = readJSONFile(filepath = "C:/python_testing/rssi_logging_json.json")
for scan in data:
sensor = sensors.get(scan.name)
# First time seeing the sensor
if(sensor == None):
sensors[scan.name] = Sensor(scan.name)
sensor = sensors.get(scan.name)
sensor.name = scan.name
sensor.x.append(determineXAxisTime(scan))
sensor.y.append(scan.rssi)
else:
sensor.x.append(determineXAxisTime(scan))
sensor.y.append(scan.rssi)
ax1.clear()
#basic smoothing using nearby averages
#y_smooth3 = smooth(np.ndarray.flatten(np.asarray(sensors.get("sentrius_sensor_3").y)), 1)
for graphItem in sensors.itervalues():
smoothed = smooth(np.ndarray.flatten(np.asarray(graphItem.y)), 1)
ax1.plot(graphItem.x, smoothed, label = graphItem.name, linewidth = 2.0)
ax1.legend()
determineClosestSensor()
fig.suptitle("Live RSSI Graph from Sentrius Sensors", fontsize = 14)
def main():
ani = animation.FuncAnimation(fig, animate, interval = 15000)
plt.show()
if __name__ == "__main__":
main()
As far as I can tell you are regenerating your data in each animation step by appending to the existing datasets, but then this means that your last x point from the first step is followed by the first x point in the second step, leading to a rewind in the plot. This appears as the line connecting the last datapoint with the first one; the rest of the data is unchanged.
The relevant part of animate:
def animate(i):
data = readJSONFile(filepath = "C:/python_testing/rssi_logging_json.json")
for scan in data:
sensor = sensors.get(scan.name)
# First time seeing the sensor
if(sensor is None): # always check for None with `is`!
... # stuff here
else:
sensor.x.append(determineXAxisTime(scan)) # always append!
sensor.y.append(scan.rssi) # always append!
... # rest of the stuff here
So, in each animation step you
1. load the same JSON file
2. append the same data to an existing sensor identified by sensors.get(scan.name)
3. plot stuff without ever using i.
Firstly, your animate should naturally make use of the index i: you're trying to do something concerning step i. I can't see i being used anywhere.
Secondly, your animate should be as lightweigh as possible in order to get a smooth animation. Load your data once before plotting, and only handle the drawing differences in animate. This will involve slicing or manipulating your data as a function of i.
Of course if the file really does change from step to step, and this is the actual dynamics in the animation (i.e. i is a dummy variable that is never used), all you need to do is zero-initialize all the plotting data in each step. Start with a clean slate. Then you'll stop seeing the lines corresponding to these artificial jumps in the data. But again, if you want a lightweigh animate, you should look into manipulating the underlying data of existing plots rather than replotting everything all the time (especially since calls to ax1.plot will keep earlier points on the canvas, which is not what you usually want in an animation).
try changing :
ani = animation.FuncAnimation(fig, animate, interval = 15000)
to :
ani = animation.FuncAnimation(fig, animate, interval = 15000, repeat = False)
I write a small program using web.py, and in one of classes I use numpy/plot.
I found that every first I visit the page , it works fine. but after several minutes, the function of plt.figure() frozen! this function will never return! That's so weird.
please have a look of my codes:
def DrawMapMain(MapParameter,inputfile='out.txt',imgfile='out.png'):
print "DrawMapMain..."
plt.ioff() # turn off interactive mode
plt.close('all')
xmin,xmax,ymin,ymax = MapParameter['xmin'],MapParameter['xmax'],MapParameter['ymin'],MapParameter['ymax']
print('LevelFile:',MapParameter['LevelFile'])
LonCenter = (xmin+xmax)/2.0
LatCenter = (ymin+ymax)/2.0
nx, ny = 200,200
if(not os.path.isfile(inputfile)):
print(u'输入文件%s不存在，请检查！'%(inputfile))
sys.exit(0)
Region = np.loadtxt(inputfile)
#print(Region)
x,y,z = Region[:,1],Region[:,2],Region[:,3]
lon_array = np.linspace(xmin, xmax, nx)
lat_array = np.linspace(ymin, ymax, ny)
print('Data lon/lat box :',x.min(),x.max(),y.min(),y.max())
print(u'离散点插值到网格')
zi,xi,yi = Interpolater.griddata_all(x,y,z,lon_array,lat_array,func='line_rbf')#scipy_idw')# #line_rbf
print(u'扩展矩阵插值: ')
zi,xi,yi,lon_array,lat_array,nx,ny=Interpolater.extened_grid(zi,lon_array,lat_array,zoom=int(2)) #
print(u'mask非绘图区域')
grid1 = Interpolater.build_inside_mask_array(MapParameter['ShapeFile'],lon_array,lat_array)
zi[np.logical_not(grid1)]=np.NaN
#-----------------------------------------------------------------------------------
print(u'Create figure...')
#fig = plt.figure(num=1,figsize=(12, 9), dpi=100)
fig = plt.figure(figsize=(12, 9), dpi=100)
#fig = plt.figure()
print(u'Create figure...Done')
.........skipped
first time I visit the page, I got:
mask非绘图区域
Create figure...
Create figure...Done
(104, 35, 108, 39.5)
this is ok, but after a while, visit again, I got:
mask非绘图区域
Create figure...
and I can see the process 'python' take 25% of my cpu(which have 4 core), that means it falls into a deadloop!
this is my web.py class, , for reference:
class Month:
def POST(self):
form = ParameterForm()
if not form.validates():
return render.Month(form)
else:
StationInfoFile='./StationsId.txt' # make sure this file is exist.
if(not os.path.isfile(StationInfoFile)):
print(u'StationInfoFile 文件%s不存在！'%(inputfile))
sys.exit(0)
StationsInfo = np.loadtxt(StationInfoFile) # load all data as integer and float, not string
StationsId,StationsLon,StationsLat = StationsInfo[:,0].astype(np.int64),StationsInfo[:,1],StationsInfo[:,2]
basedir, DataCats, DataCatsDict=u'D:/测试数据',[ u'逐日平均', u'逐日降水'],{ u'逐日平均':'td', u'逐日降水':'rd'}
iFrom,iEnd= \
int(form['Start Year'].value)*10000+ int(form['Start Month'].value)*100+ int(form['Start Day'].value), \
int(form['End Year'].value)*10000+ int(form['End Month'].value)*100+ int(form['End Day'].value) # value from form is string!
MapParameter=GetMapParameter()
if (u'温度' == form['Data Source'].value):
d=u'逐日平均'
tmpDataTxt='Test_temp.txt'
tmpOutPNG='./static/'+'Test_temp.png'
MapParameter['LevelFile']='.\maplev_temp.LEV'
MapParameter['Title']=u'逐日平均'
elif (u'降水(mm)' == form['Data Source'].value):
d=u'逐日降水'
tmpDataTxt='Test_pred.txt'
tmpOutPNG='./static/'+'Test_temp.png'
MapParameter['LevelFile']='.\maplev_rain.LEV'
MapParameter['Title']=u'逐日降水'
else:
print "form['Data Source'].value=",form['Data Source'].value
print "----------- PROCESSING FOR CATEGORY:",d
tmpMeanVal=[]
for i in range(len(StationsId)):
s,lo,la=StationsId[i],StationsLon[i],StationsLat[i]
#print basedir,d, str(s),DataCatsDict[d]+'.txt'
datafile=os.path.join(basedir,d, str(s))+DataCatsDict[d]+'.txt'
print datafile,iFrom,iEnd
data=getdata.GetData(datafile,iFrom,iEnd)
a=np.mean(np.array(data)[:,1])*0.1
tmpMeanVal.append([s,lo,la,a])
rec=np.array(tmpMeanVal,dtype=[('int','int'),('float','float')])
print 'Writing data ...'
np.savetxt(tmpDataTxt,tmpMeanVal,fmt="%6i %-7.2f %-7.2f %8.2f")
print 'Writing data ... Done.'
sssss=open(tmpDataTxt,'r')
print sssss.read()
sssss.close()
DrawMapMain(MapParameter,inputfile=tmpDataTxt,imgfile=tmpOutPNG)
return render.Reports(tmpOutPNG)
First I suspect that the plt.figure may have some memory leak problems, so I us clf,plt.close('all') at the beginning and end of the function both! I even wrote a segment of test code :
if __name__ == "__main__":
MapParameter=GetMapParameter()
MapParameter['LevelFile']='.\maplev_rain.LEV'
MapParameter['Title']=u'逐日降水'
for iloop in range(0,10):
DrawMapMain(MapParameter,inputfile='Test_pred.txt',imgfile='c:/Test_pred'+str(iloop)+'.png')
MapParameter['LevelFile']='.\maplev_temp.LEV'
MapParameter['Title']=u'逐日temp'
for iloop in range(0,10):
DrawMapMain(MapParameter,inputfile='Test_temp.txt',imgfile='c:/Test_temp'+str(iloop)+'.png')
this code works fine. It's so wired, does anybody know some clue? very thanks!
This question has an answer in the comments:
What matplotlib backend are you using? You should be using one of the non-interactive ones if you're running things from a webserver. E.g. do import matplotlib; matplotlib.use('Agg') before import matplotlib.pyplot as plt. – Joe Kington Jan 16 at 17:56
For more information on matplotlib backends, see: http://matplotlib.org/faq/usage_faq.html#what-is-a-backend – Joe Kington Jan 17 at 15:21
I am trying to plot a very big file (~5 GB) using python and matplotlib. I am able to load the whole file in memory (the total available in the machine is 16 GB) but when I plot it using simple imshow I get a segmentation fault. This is most probable to the ulimit which I have set to 15000 but I cannot set higher. I have come to the conclusion that I need to plot my array in batches and therefore made a simple code to do that. My main isue is that when I plot a batch of the big array the x coordinates start always from 0 and there is no way I can overlay the images to create a final big one. If you have any suggestion please let me know. Also I am not able to install new packages like "Image" on this machine due to administrative rights. Here is a sample of the code that reads the first 12 lines of my array and make 3 plots.
import os
import sys
import scipy
import numpy as np
import pylab as pl
import matplotlib as mpl
import matplotlib.cm as cm
from optparse import OptionParser
from scipy import fftpack
from scipy.fftpack import *
from cmath import *
from pylab import *
import pp
import fileinput
import matplotlib.pylab as plt
import pickle
def readalllines(file1,rows,freqs):
file = open(file1,'r')
sizer = int(rows*freqs)
i = 0
q = np.zeros(sizer,'float')
for i in range(rows*freqs):
s =file.readline()
s = s.split()
#print s[4],q[i]
q[i] = float(s[4])
if i%262144 == 0:
print '\r ',int(i*100.0/(337*262144)),' percent complete',
i += 1
file.close()
return q
parser = OptionParser()
parser.add_option('-f',dest="filename",help="Read dynamic spectrum from FILE",metavar="FILE")
parser.add_option('-t',dest="dtime",help="The time integration used in seconds, default 10",default=10)
parser.add_option('-n',dest="dfreq",help="The bandwidth of each frequency channel in Hz",default=11.92092896)
parser.add_option('-w',dest="reduce",help="The chuncker divider in frequency channels, integer default 16",default=16)
(opts,args) = parser.parse_args()
rows=12
freqs = 262144
file1 = opts.filename
s = readalllines(file1,rows,freqs)
s = np.reshape(s,(rows,freqs))
s = s.T
print s.shape
#raw_input()
#s_shift = scipy.fftpack.fftshift(s)
#fig = plt.figure()
#fig.patch.set_alpha(0.0)
#axes = plt.axes()
#axes.patch.set_alpha(0.0)
###plt.ylim(0,8)
plt.ion()
i = 0
for o in range(0,rows,4):
fig = plt.figure()
#plt.clf()
plt.imshow(s[:,o:o+4],interpolation='nearest',aspect='auto', cmap=cm.gray_r, origin='lower')
if o == 0:
axis([0,rows,0,freqs])
fdf, fdff = xticks()
print fdf
xticks(fdf+o)
print xticks()
#axis([o,o+4,0,freqs])
plt.draw()
#w, h = fig.canvas.get_width_height()
#buf = np.fromstring(fig.canvas.tostring_argb(), dtype=np.uint8)
#buf.shape = (w,h,4)
#buf = np.rol(buf, 3, axis=2)
#w,h,_ = buf.shape
#img = Image.fromstring("RGBA", (w,h),buf.tostring())
#if prev:
# prev.paste(img)
# del prev
#prev = img
i += 1
pl.colorbar()
pl.show()
If you plot any array with more than ~2k pixels across something in your graphics chain will down sample the image in some way to display it on your monitor. I would recommend down sampling in a controlled way, something like
data = convert_raw_data_to_fft(args) # make sure data is row major
def ds_decimate(row,step = 100):
return row[::step]
def ds_sum(row,step):
return np.sum(row[:step*(len(row)//step)].reshape(-1,step),1)
# as per suggestion from tom10 in comments
def ds_max(row,step):
return np.max(row[:step*(len(row)//step)].reshape(-1,step),1)
data_plotable = [ds_sum(d) for d in data] # plug in which ever function you want
or interpolation.
Matplotlib is pretty memory-inefficient when plotting images. It creates several full-resolution intermediate arrays, which is probably why your program is crashing.
One solution is to downsample the image before feeding it into matplotlib, as #tcaswell suggests.
I also wrote some wrapper code to do this downsampling automatically, based on your screen resolution. It's at https://github.com/ChrisBeaumont/mpl-modest-image, if it's useful. It also has the advantage that the image is resampled on the fly, so you can still pan and zoom without sacrificing resolution where you need it.
I think you're just missing the extent=(left, right, bottom, top) keyword argument in plt.imshow.
x = np.random.randn(2, 10)
y = np.ones((4, 10))
x[0] = 0 # To make it clear which side is up, etc
y[0] = -1
plt.imshow(x, extent=(0, 10, 0, 2))
plt.imshow(y, extent=(0, 10, 2, 6))
# This is necessary, else the plot gets scaled and only shows the last array
plt.ylim(0, 6)
plt.colorbar()
plt.show()