We Keep Coding

how to change the color of subplot datafile in matplotlib

In Short:
I want to change the color of blue marker in the graph. So that I can do comparison with other plots easily.
You can download the data files and script from this link
Problem Explanation
I have two data files, full.dat and part.dat(Note: part.dat is also there in full.dat).
I got the plotting scripts from the internet, and it is working very well. But as a noob in Python and Matplotlib, I am facing difficulties in changing the color of part.dat.
Please see the graph first, then the following scripts.
Script-1: Function and definitions: let's say: "func.py"
# This was written by Levi Lentz for the Kolpak Group at MIT
import numpy as np
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
import matplotlib.gridspec as gs
import sys
#This function extracts the high symmetry points from the output of bandx.out
def Symmetries(fstring):
f = open(fstring,'r')
x = np.zeros(0)
for i in f:
if "high-symmetry" in i:
x = np.append(x,float(i.split()[-1]))
f.close()
return x
# This function takes in the datafile, the fermi energy, the symmetry file, a subplot, and the label
# It then extracts the band data, and plots the bands, the fermi energy in red, and the high symmetry points
def bndplot(datafile_full,datafile,fermi,symmetryfile,subplot,**kwargs):
if 'shift_fermi' in kwargs:
bool_shift_efermi = kwargs['shift_fermi']
else:
bool_shift_efermi = 0
if 'color' in kwargs:
color_bnd=kwargs['color']
else:
color_bnd='black'
if 'linestyle' in kwargs:
line_bnd=kwargs['linestyle']
else:
line_bnd='solid'
z = np.loadtxt(datafile_full) #This loads the full.dat file
x = np.unique(z[:,0]) #This is all the unique x-points
[a,b,w]=np.loadtxt(datafile,unpack=True) #Weight
bands = []
bndl = len(z[z[:,0]==x[1]]) #This gives the number of bands in the calculation
Fermi = float(fermi)
if bool_shift_efermi:
fermi_shift=Fermi
else:
fermi_shift=0
axis = [min(x),max(x)]
for i in range(0,bndl):
bands.append(np.zeros([len(x),2])) #This is where we storre the bands
for i in range(0,len(x)):
sel = z[z[:,0] == x[i]] #Here is the energies for a given x
test = []
for j in range(0,bndl): #This separates it out into a single band
bands[j][i][0] = x[i]
#bands[j][i][1] = np.multiply(sel[j][1],13.605698066)
bands[j][i][1] = sel[j][1]
#Here we plots the bands
for i in bands:
subplot.plot(i[:,0],i[:,1]-fermi_shift,color=color_bnd,linestyle=line_bnd, linewidth=0.7,alpha=0.5)
# plt.scatter(a,b-fermi_shift,c=w,cmap='viridis',alpha=0.5)
# plt.colorbar()
if 'legend' in kwargs:
#empty plot to generate legend
subplot.plot([None],[None],color=color_bnd,linestyle=line_bnd,label=kwargs['legend'])
temp = Symmetries(symmetryfile)
for j in temp: #This is the high symmetry lines
x1 = [j,j]
subplot.axvline(x=j,linestyle='dashed',color='black',alpha=0.75)
subplot.plot([min(x),max(x)],[Fermi-fermi_shift,Fermi-fermi_shift],color='red',linestyle='dotted')
subplot.set_xticks(temp)
subplot.set_xticklabels([])
if 'name_k_points' in kwargs:
if len(kwargs['name_k_points'])==len(temp):
subplot.set_xticklabels(kwargs['name_k_points'])
if 'range' in kwargs:
range_plot=kwargs['range']
subplot.set_ylim([range_plot[0],range_plot[1]])
subplot.set_xlim([axis[0],axis[1]])
subplot.set_xlabel('k')
subplot.set_ylabel('E-E$_f$')
plt.scatter(a,b-fermi_shift,s=70*np.array(w))
if 'legend' in kwargs:
plt.legend()
script-2 Plotting script: let's say: "plot.py"
#!/usr/bin/python3
from func import *
El='el'
orb='orb'
plt.rcParams["figure.figsize"]=(4,15)
datafile_full='bands.dat.gnu'
#datafile=El+'_'+orb+'.dat.all'
datafile=El+'_'+orb+'.dat.all'
fermi = 10.2382
symmetryfile='band.out'
bool_shift_efermi= True
fig, ax = plt.subplots()
#bndplot(datafile,fermi,symmetryfile,ax)
bndplot(datafile_full,datafile,fermi,symmetryfile,ax,shift_fermi=1,color='black',linestyle='solid',name_k_points=['K','G','M','K','H','A','L','H'], legend=El+', '+orb+'-orbital')
#ax.set_ylim(-5,5)
ax.set_ylim(-10,12)
fig.set_figheight(6)
fig.set_figwidth(4)
plt.rcParams.update({'font.size': 22})
fig.savefig("el-orb.eps")
plt.show()
In script-2, there is an option to change the color, however I want to change the color of blue marker/solid-circles(please see the graph) so that I can compare with other graphs.
Whenever I change the color, it changes the line color only.
Please help me out I am trying to understand Matplotlib uses and examples from past few hrs However as a noob I was not able to figure out how to do.

Python - ProcessPoolExecutor hangs when called from a mpl_connect handler

I'm using parallel processing to generate a plot of functions using complex numbers. My script allows you to zoom in on an area of the plot using the standard matplotlib controls and then regenerate the plot within the new limits to improve resolution.
This is my first foray into parallel processing and I've got as far as understanding that I need to preface with if __name__ == __main__: to allow the module to be imported properly. When running my script, the first plot is successfully generated and appears as expected. However, when the plotting function is called again from my event handler it instead hangs indefinitely. I assume that the hang is caused by some similar issue to that of requiring if __name__ == __main__:, as the parallel processes are being spawned from outside the main body of the script, but I haven't figured out anything further than this.
import numpy as np
import matplotlib.pyplot as plt
from concurrent.futures import ProcessPoolExecutor
import multiprocessing
res = [1000, 1000]
base_factor = 2.
cpuNum = multiprocessing.cpu_count()
def brot(c, depth=200):
z = complex(0)
for i in range(depth):
z = pow(z, 2) + c
if abs(z) > 2:
return i
return -1
def brot_gen(span):
re_span = span[0]
im_span = span[1]
mset = np.zeros([len(im_span), len(re_span)])
for re in range(len(re_span)):
for im in range(len(im_span)):
mset[im][re] = brot(complex(re_span[re], im_span[im]))
return mset
def brot_gen_parallel(re_lim, im_lim):
re_span = np.linspace(re_lim[0], re_lim[1], res[0])
im_span = np.linspace(im_lim[0], im_lim[1], res[1])
split_re_span = np.array_split(re_span, cpuNum)
packages = [(sec, im_span) for sec in split_re_span]
print("Generating set between", re_lim, "and", im_lim, "...")
with ProcessPoolExecutor(max_workers = cpuNum) as executor:
result = executor.map(brot_gen, packages)
mset = np.concatenate(list(result), axis=1)
print("Set generated")
return mset
def handler(ax):
def action(event):
if event.button == 2:
cur_re_lim = ax.get_xlim()
cur_im_lim = ax.get_ylim()
mset = brot_gen_parallel(cur_re_lim, cur_im_lim)
ax.cla()
ax.imshow(mset, extent=[cur_re_lim[0], cur_re_lim[1], cur_im_lim[0], cur_im_lim[1]], origin="lower", vmin=0, vmax=200, interpolation="bilinear")
plt.draw()
fig = ax.get_figure()
fig.canvas.mpl_connect('button_release_event', action)
return action
if __name__ == "__main__":
re_lim = np.array([-2.5, 2.5])
im_lim = res[1]/res[0] * re_lim
mset = brot_gen_parallel(re_lim, im_lim)
plt.imshow(mset, extent=[re_lim[0], re_lim[1], im_lim[0], im_lim[1]], origin="lower", vmin=0, vmax=200, interpolation="bilinear")
ax = plt.gca()
f = handler(ax)
plt.show()
EDIT: I wondered if there was a bug in the code causing an exception, but that this might not be being successfully passed back to the console, however I tested this by running the same task without splitting it into parallel tasks and it completed successfully.

I have discovered the answer to my own question. The answer lies in the IDE I was using. In my experience, in most IDEs plt.show() blocks execution by default, however in Spyder the default seems to be the equivalent of plt.show(block=False), meaning that the script completed and so whatever was required to successfully start the parallel processes was no longer available, causing the hang. This was solved by simply changing the statement to plt.show(block=True), meaning that the script was still live.
I'm still very new to parallel processing so I'd be very interested in any more information anyone can give on what was lacking to stop the parallel processing from working.

Matplotlib connects end of data after animation refreshes

I'm having an issue exactly like this post, but slightly more frustrating.
I'm using matplotlib to read from a file that is being fed data from another application. For some reason, the ends of the data only connect after the animation (animation.FuncAnimation) has completed its first refresh. Here are some images:
This is before the refresh:
And this is after the refresh:
Any ideas as to why this could be happening? Here is my code:
import json
import itertools
import dateutil.parser
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib import style
import scipy.signal as sig
import numpy as np
import pylab as plt
sensors = {}
data = []
lastLineReadNum = 0
class Sensor:
def __init__(self, name, points = 0, lastReading = 0):
self.points = points
self.lastReading = lastReading
self.name = name
self.x = []
self.y = []
class ScanResult:
def __init__(self, name, id, rssi, macs, time):
self.name = name
self.id = id
self.rssi = rssi
self.macs = macs
# Is not an integer, but a datetime.datetime
self.time = time
def readJSONFile(filepath):
with open(filepath, "r") as read_file:
global lastLineReadNum
# Load json results into an object that holds important info
for line in itertools.islice(read_file, lastLineReadNum, None):
temp = json.loads(line)
# Only reads the most recent points...
data.append(ScanResult(name = temp["dev_id"],
id = temp["hardware_serial"],
rssi = temp["payload_fields"]["rssis"],
macs = temp["payload_fields"]["macs"],
time = dateutil.parser.parse(temp["metadata"]["time"])))
lastLineReadNum += 1
return data
style.use('fivethirtyeight')
fig = plt.figure()
ax1 = fig.add_subplot(1, 1, 1)
def smooth(y, box_pts):
box = np.ones(box_pts)/box_pts
y_smooth = np.convolve(y, box, mode='same')
return y_smooth
def determineClosestSensor():
global sensors
#sensors.append(Sensor(time = xs3, rssi = ys3))
def determineXAxisTime(scanresult):
return ((scanresult.time.hour * 3600) + (scanresult.time.minute * 60) + (scanresult.time.second)) / 1000.0
def animate(i):
data = readJSONFile(filepath = "C:/python_testing/rssi_logging_json.json")
for scan in data:
sensor = sensors.get(scan.name)
# First time seeing the sensor
if(sensor == None):
sensors[scan.name] = Sensor(scan.name)
sensor = sensors.get(scan.name)
sensor.name = scan.name
sensor.x.append(determineXAxisTime(scan))
sensor.y.append(scan.rssi)
else:
sensor.x.append(determineXAxisTime(scan))
sensor.y.append(scan.rssi)
ax1.clear()
#basic smoothing using nearby averages
#y_smooth3 = smooth(np.ndarray.flatten(np.asarray(sensors.get("sentrius_sensor_3").y)), 1)
for graphItem in sensors.itervalues():
smoothed = smooth(np.ndarray.flatten(np.asarray(graphItem.y)), 1)
ax1.plot(graphItem.x, smoothed, label = graphItem.name, linewidth = 2.0)
ax1.legend()
determineClosestSensor()
fig.suptitle("Live RSSI Graph from Sentrius Sensors", fontsize = 14)
def main():
ani = animation.FuncAnimation(fig, animate, interval = 15000)
plt.show()
if __name__ == "__main__":
main()

As far as I can tell you are regenerating your data in each animation step by appending to the existing datasets, but then this means that your last x point from the first step is followed by the first x point in the second step, leading to a rewind in the plot. This appears as the line connecting the last datapoint with the first one; the rest of the data is unchanged.
The relevant part of animate:
def animate(i):
data = readJSONFile(filepath = "C:/python_testing/rssi_logging_json.json")
for scan in data:
sensor = sensors.get(scan.name)
# First time seeing the sensor
if(sensor is None): # always check for None with `is`!
... # stuff here
else:
sensor.x.append(determineXAxisTime(scan)) # always append!
sensor.y.append(scan.rssi) # always append!
... # rest of the stuff here
So, in each animation step you
1. load the same JSON file
2. append the same data to an existing sensor identified by sensors.get(scan.name)
3. plot stuff without ever using i.
Firstly, your animate should naturally make use of the index i: you're trying to do something concerning step i. I can't see i being used anywhere.
Secondly, your animate should be as lightweigh as possible in order to get a smooth animation. Load your data once before plotting, and only handle the drawing differences in animate. This will involve slicing or manipulating your data as a function of i.
Of course if the file really does change from step to step, and this is the actual dynamics in the animation (i.e. i is a dummy variable that is never used), all you need to do is zero-initialize all the plotting data in each step. Start with a clean slate. Then you'll stop seeing the lines corresponding to these artificial jumps in the data. But again, if you want a lightweigh animate, you should look into manipulating the underlying data of existing plots rather than replotting everything all the time (especially since calls to ax1.plot will keep earlier points on the canvas, which is not what you usually want in an animation).

try changing :
ani = animation.FuncAnimation(fig, animate, interval = 15000)
to :
ani = animation.FuncAnimation(fig, animate, interval = 15000, repeat = False)

Multiple route mapping to different matplotlib graphs in flask app

I have this "flask app" with two links, each mapping to different matplotlib visualizations, for example: localhost:5000/line_chart and localhost:5000/bar_chart.
When I start the server, and click the a route (any of them), I see what I expect.
localhost:5000/bar_chart
When I go back and view the other link, both graphs break.
localhost:5000/line_chart
localhost:5000/bar_chart
I can reproduce this every time by closing the server then running the "run.py" script again. Seems to be an overwriting conflict with the in-memory buffer. Has anyone had this issue before?
app/views.py
import matplotlib
matplotlib.use('Agg') # this allows PNG plotting
import matplotlib.pyplot as plt
import base64
from flask import render_template
from app import app
from io import BytesIO
#app.route('/')
#app.route('/index')
def index():
res = ''
navigation = [['Line Chart','line_chart'],['Bar Chart','bar_chart']]
res = res + '<h1>Matplotlib Chart Examples</h1>'
res = res + '<ul>'
for item in navigation:
name = item[0]
link = item[1]
res = res + '<li>'+ name +'</li>'
res = res +'</ul>'
return res
#app.route('/bar_chart')
def bar_chart():
movies = ["Annie Hall", "Ben-Hur", "Casablanca", "Gandhi", "West Side Story"]
num_oscars = [5, 11, 3, 8, 10]
# bars are by default width 0.8, so we'll add 0.1 to the left coordinates
# so that each bar is centered
xs = [i + 0.1 for i, _ in enumerate(movies)]
# plot bars with left x-coordinates [xs], heights [num_oscars]
plt.bar(xs, num_oscars)
plt.ylabel("# of Academy Awards")
plt.title("My Favorite Movies")
# label x-axis with movie names at bar centers
plt.xticks([i + 0.5 for i, _ in enumerate(movies)], movies)
return compute(plt)
#app.route('/line_chart')
def line_chart():
years = [1950, 1960, 1970, 1980, 1990, 2000, 2010]
gdp = [300.2, 543.3, 1075.9, 2862.5, 5979.6, 10289.7, 14958.3]
# create a line chart, years on x-axis, gdp on y-axis
plt.plot(years, gdp, color='green', marker='o', linestyle='solid')
# add a title
plt.title("Nominal GDP")
# add a label to the y-axis
plt.ylabel("Billions of $")
return compute(plt)
def compute(plt):
# run plt.plot, plt.title, etc.
figfile = BytesIO()
plt.savefig(figfile, format='png')
figfile.seek(0) # rewind to beginning of file
#figfile.getvalue() extracts string (stream of bytes)
figdata_png = base64.b64encode(figfile.getvalue())
return render_template('index.html',
title='matplotlib chart',
results=figdata_png)
Thank you for your time.

I guess you need two figures, test this code and tell what happened:
#app.route('/bar_chart')
def bar_chart():
movies = ["Annie Hall", "Ben-Hur", "Casablanca", "Gandhi", "West Side Story"]
num_oscars = [5, 11, 3, 8, 10]
# bars are by default width 0.8, so we'll add 0.1 to the left coordinates
# so that each bar is centered
xs = [i + 0.1 for i, _ in enumerate(movies)]
# plot bars with left x-coordinates [xs], heights [num_oscars]
plt.figure(1)
plt.bar(xs, num_oscars)
plt.ylabel("# of Academy Awards")
plt.title("My Favorite Movies")
# label x-axis with movie names at bar centers
plt.xticks([i + 0.5 for i, _ in enumerate(movies)], movies)
return compute(plt, 1)
#app.route('/line_chart')
def line_chart():
years = [1950, 1960, 1970, 1980, 1990, 2000, 2010]
gdp = [300.2, 543.3, 1075.9, 2862.5, 5979.6, 10289.7, 14958.3]
# create a line chart, years on x-axis, gdp on y-axis
plt.figure(2)
plt.plot(years, gdp, color='green', marker='o', linestyle='solid')
# add a title
plt.title("Nominal GDP")
# add a label to the y-axis
plt.ylabel("Billions of $")
return compute(plt,2)
def compute(plt, fignum):
# run plt.plot, plt.title, etc.
plt.figure(fignum)
figfile = BytesIO()
plt.savefig(figfile, format='png')
figfile.seek(0) # rewind to beginning of file
#figfile.getvalue() extracts string (stream of bytes)
figdata_png = base64.b64encode(figfile.getvalue())
return render_template('index.html',
title='matplotlib chart',
results=figdata_png)

In my case, that solution didn't work. It seems that there is a race condition when trying to access plot. I first tried to use a lock from a library, but that didn't work, so instead I sort of engineered out a lock. In my case, I wanted to create n images using the same function on the same view, so I started by creating a list in the following way:
queue = [False for i in range(n)]
Then, my flask app look something like this:
#app.route('/vis/<j>')
def vis(j):
global queue
# We check that it's image's #j turn, as if it was single threaded
j = int(j)
if j == 0:
for i in range(len(queue)):
queue[i] = False
else:
while not queue[j-1]:
# If it's not, we sleep for a short time (from time import sleep)
sleep(0.5)
# This is not important, it's how I was plotting some random figures
# (from random import seed) (from datetime import datetime)
seed(datetime.now())
n = 10
p1 = [randint(0, 10) for _ in range(n)]
p2 = [randint(0, 10) for _ in range(n)]
t = [i for i in range(n)]
fig = plt.figure(j)
plt.clf()
plt.plot(t, p1, color='blue')
plt.plot(t, p2, color='orange')
plt.xlabel('Time')
plt.ylabel('Value')
# Save the plot
img = BytesIO()
fig.savefig(img, dpi=128)
img.seek(0)
# We finished using everything related to plot, so we free the "lock"
queue[j] = True
# Return the object as a file that can be accessed
return send_file(img, mimetype='image/png')
Finally, when wanting to display this in my flask app, all I had to do was using this <img src="/vis/1"> in my html file.
Edit: I forgot one of the most important part! For some reason, this would still create some unrelated thread issue. I looked it up and that's when I came with the full solution. The threading issue was solved by adding at the beginning of the file:
import matplotlib
import matplotlib.pyplot as plt
matplotlib.use('Agg')
For some reason, using that Agg backend solved the second threading I was having. I don't really have a good explanation for that, but it does work, so it's enough for me.
Alternatively, what also worked was running the app disabling threads by adding:
if __name__ == '__main__':
app.run(threading=False, debug=True)
I don't know however, at the moment, whether this works in production, so I preferred the other solution. :)
I hope this helps if you had the same issue!

Plot really big file in python (5GB) with x axis offset

I am trying to plot a very big file (~5 GB) using python and matplotlib. I am able to load the whole file in memory (the total available in the machine is 16 GB) but when I plot it using simple imshow I get a segmentation fault. This is most probable to the ulimit which I have set to 15000 but I cannot set higher. I have come to the conclusion that I need to plot my array in batches and therefore made a simple code to do that. My main isue is that when I plot a batch of the big array the x coordinates start always from 0 and there is no way I can overlay the images to create a final big one. If you have any suggestion please let me know. Also I am not able to install new packages like "Image" on this machine due to administrative rights. Here is a sample of the code that reads the first 12 lines of my array and make 3 plots.
import os
import sys
import scipy
import numpy as np
import pylab as pl
import matplotlib as mpl
import matplotlib.cm as cm
from optparse import OptionParser
from scipy import fftpack
from scipy.fftpack import *
from cmath import *
from pylab import *
import pp
import fileinput
import matplotlib.pylab as plt
import pickle
def readalllines(file1,rows,freqs):
file = open(file1,'r')
sizer = int(rows*freqs)
i = 0
q = np.zeros(sizer,'float')
for i in range(rows*freqs):
s =file.readline()
s = s.split()
#print s[4],q[i]
q[i] = float(s[4])
if i%262144 == 0:
print '\r ',int(i*100.0/(337*262144)),' percent complete',
i += 1
file.close()
return q
parser = OptionParser()
parser.add_option('-f',dest="filename",help="Read dynamic spectrum from FILE",metavar="FILE")
parser.add_option('-t',dest="dtime",help="The time integration used in seconds, default 10",default=10)
parser.add_option('-n',dest="dfreq",help="The bandwidth of each frequency channel in Hz",default=11.92092896)
parser.add_option('-w',dest="reduce",help="The chuncker divider in frequency channels, integer default 16",default=16)
(opts,args) = parser.parse_args()
rows=12
freqs = 262144
file1 = opts.filename
s = readalllines(file1,rows,freqs)
s = np.reshape(s,(rows,freqs))
s = s.T
print s.shape
#raw_input()
#s_shift = scipy.fftpack.fftshift(s)
#fig = plt.figure()
#fig.patch.set_alpha(0.0)
#axes = plt.axes()
#axes.patch.set_alpha(0.0)
###plt.ylim(0,8)
plt.ion()
i = 0
for o in range(0,rows,4):
fig = plt.figure()
#plt.clf()
plt.imshow(s[:,o:o+4],interpolation='nearest',aspect='auto', cmap=cm.gray_r, origin='lower')
if o == 0:
axis([0,rows,0,freqs])
fdf, fdff = xticks()
print fdf
xticks(fdf+o)
print xticks()
#axis([o,o+4,0,freqs])
plt.draw()
#w, h = fig.canvas.get_width_height()
#buf = np.fromstring(fig.canvas.tostring_argb(), dtype=np.uint8)
#buf.shape = (w,h,4)
#buf = np.rol(buf, 3, axis=2)
#w,h,_ = buf.shape
#img = Image.fromstring("RGBA", (w,h),buf.tostring())
#if prev:
# prev.paste(img)
# del prev
#prev = img
i += 1
pl.colorbar()
pl.show()

If you plot any array with more than ~2k pixels across something in your graphics chain will down sample the image in some way to display it on your monitor. I would recommend down sampling in a controlled way, something like
data = convert_raw_data_to_fft(args) # make sure data is row major
def ds_decimate(row,step = 100):
return row[::step]
def ds_sum(row,step):
return np.sum(row[:step*(len(row)//step)].reshape(-1,step),1)
# as per suggestion from tom10 in comments
def ds_max(row,step):
return np.max(row[:step*(len(row)//step)].reshape(-1,step),1)
data_plotable = [ds_sum(d) for d in data] # plug in which ever function you want
or interpolation.

Matplotlib is pretty memory-inefficient when plotting images. It creates several full-resolution intermediate arrays, which is probably why your program is crashing.
One solution is to downsample the image before feeding it into matplotlib, as #tcaswell suggests.
I also wrote some wrapper code to do this downsampling automatically, based on your screen resolution. It's at https://github.com/ChrisBeaumont/mpl-modest-image, if it's useful. It also has the advantage that the image is resampled on the fly, so you can still pan and zoom without sacrificing resolution where you need it.

I think you're just missing the extent=(left, right, bottom, top) keyword argument in plt.imshow.
x = np.random.randn(2, 10)
y = np.ones((4, 10))
x[0] = 0 # To make it clear which side is up, etc
y[0] = -1
plt.imshow(x, extent=(0, 10, 0, 2))
plt.imshow(y, extent=(0, 10, 2, 6))
# This is necessary, else the plot gets scaled and only shows the last array
plt.ylim(0, 6)
plt.colorbar()
plt.show()