We Keep Coding

Mathplotlib graph problems

I'm trying to display data from a weather station with mathplotlib. For some reason that I can't quite figure out my last values on the graph are acting randomly, going back in time on the x axis.
x axis is the dates,
y axis is the water level
y1 axis is the discharge flow
Here's a picture of the result
Graph
import pandas as pd
import matplotlib.pyplot as plt
url_hourly = "https://dd.weather.gc.ca/hydrometric/csv/BC/hourly/BC_08MG005_hourly_hydrometric.csv"
url_daily = "https://dd.weather.gc.ca/hydrometric/csv/BC/daily/BC_08MG005_daily_hydrometric.csv"
fields = ["Date","Water Level / Niveau d'eau (m)", "Discharge / Débit (cms)"]
#Read csv files
hourly_data = pd.read_csv(url_hourly, usecols=fields)
day_data = pd.read_csv(url_daily, usecols=fields)
#Merge csv files
water_data = pd.concat([day_data,hourly_data])
#Convert date to datetime
water_data['Date'] = pd.to_datetime(water_data['Date']).dt.normalize()
water_data['Date'] = water_data['Date'].dt.strftime('%m/%d/%Y')
# CSV files contains 288 data entries per day (12per hour * 24hrs). Selecting every 288th element to represent one day
data_24hr = water_data[::288]
# Assigning columns to x, y, y1 axis
x = data_24hr[fields[0]]
y1 = data_24hr[fields[1]]
y2= data_24hr[fields[2]]
#Ploting the graph
fig, ax1 = plt.subplots()
ax2 = ax1.twinx()
curve1 = ax1.plot(x,y1, label='Water Level', color = 'r', marker="o")
curve2 = ax2.plot(x,y2,label='Discharge Volume', color = 'b',marker="o")
plt.plot()
plt.show()
Any tips would be greatly appreciated as I'm quite new to this
thank you

Okay I went through the code removed the duplicates (as suggest by Arne) by the "Date" column. Oh and I made the graph formatting slightly more readable. This graphed without going back in time:
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import ticker
url_hourly = "https://dd.weather.gc.ca/hydrometric/csv/BC/hourly/BC_08MG005_hourly_hydrometric.csv"
url_daily = "https://dd.weather.gc.ca/hydrometric/csv/BC/daily/BC_08MG005_daily_hydrometric.csv"
fields = ["Date","Water Level / Niveau d'eau (m)", "Discharge / Débit (cms)"]
#Read csv files
hourly_data = pd.read_csv(url_hourly, usecols=fields)
day_data = pd.read_csv(url_daily, usecols=fields)
#Merge csv files
water_data = pd.concat([day_data,hourly_data])
#Convert date to datetime
water_data['Date'] = pd.to_datetime(water_data['Date']).dt.normalize()
water_data['Date'] = water_data['Date'].dt.strftime('%m/%d/%Y')
# CSV files contains 288 data entries per day (12per hour * 24hrs). Selecting every 288th element to represent one day
data_24hr = water_data.iloc[::288]
data_24hr.drop_duplicates(subset="Date",inplace=True) #remove duplicates according to the date column
# Assigning columns to x, y, y1 axis
x = data_24hr[fields[0]]
y1 = data_24hr[fields[1]]
y2= data_24hr[fields[2]]
print(len(x), len(y1))
#Ploting the graph
fig, ax1 = plt.subplots()
ax2 = plt.twinx()
curve1 = ax1.plot(x, y1, label='Water Level', color = 'r', marker="o")
curve2 = ax2.plot(x, y2, label='Discharge Volume', color = 'b',marker="o")
fig.autofmt_xdate(rotation=90)
plt.show()

Can I set default values with matplotlib and pandas for each x tick?

I have the following code:
# Ratings by day, divided by Staff member
from datetime import datetime as dt
by_staff = df.groupby('User ID')
plt.figure(figsize=(15,8))
# Those are used to calculate xticks and yticks
xmin, xmax = pd.to_datetime(dt.now()), pd.to_datetime(0)
ymin, ymax = 0, 0
for index, data in by_staff:
by_day = data.groupby('Date')
x = pd.to_datetime(by_day.count().index)
y = by_day.count()['Value']
xmin = min(xmin, x.min())
xmax = max(xmax, x.max())
ymin = min(ymin, min(y))
ymax = max(ymax, max(y))
plt.plot_date(x, y, marker='o', label=index, markersize=12)
plt.title('Ratings by day, by Staff member', fontdict = {'fontsize': 25})
plt.xlabel('Day', fontsize=15)
plt.ylabel('n° of ratings for that day', fontsize=15)
ticks = pd.date_range(xmin, xmax, freq='D')
plt.xticks(ticks, rotation=60)
plt.yticks(range(ymin, ymax + 1))
plt.gcf().autofmt_xdate()
plt.grid()
plt.legend([a for a, b in by_staff],
title="Ratings given",
loc="center left",
bbox_to_anchor=(1, 0, 0.5, 1))
plt.show()
I'd like to set the value shown at a specific xtick to 0 if there's no data for the day. Currently, this is the plot shown:
I tried some Google searches, but I can't seem to explain my problem correctly. How could I solve this?
My dataset: https://cdn.discordapp.com/attachments/311932890017693700/800789506328100934/sample-ratings.csv

Let's try to simplify the task by letting pandas aggregate the data. We group by Date and User ID simultaneously and then unstack the dataframe. This allows us to fill the missing data points with a preset value like 0.The form x = df.groupby(["Date",'User ID']).count().Value.unstack(fill_value=0) is compact chaining for a= df.groupby(["Date",'User ID']), b=a.count(), c=b.Value, x=c.unstack(fill_value=0). You can print out each intermediate result of these chained pandas operations to see what it does.
from matplotlib import pyplot as plt
import pandas as pd
df = pd.read_csv("test.csv", sep=",", parse_dates=["Date"])
#by_staff = df.groupby(["Date",'User ID']) - group entries by date and ID
#.count - count identical date-ID pairs
#.Value - use only this column
#.unstack(fill_value=0) bring resulting data from long to wide form
#and fill missing data with zero
by_staff = df.groupby(["Date",'User ID']).count().Value.unstack(fill_value=0)
ax = by_staff.plot(marker='o', markersize=12, linestyle="None", figsize=(15,8))
plt.title('Ratings by day, by Staff member', fontdict = {'fontsize': 25})
plt.xlabel('Day', fontsize=15)
plt.ylabel('n° of ratings for that day', fontsize=15)
#labeling only the actual rating values shown in the grid
plt.yticks(range(df.Value.max() + 1))
#this is not really necessary, it just labels zero differently
#labels = ["No rating"] + [str(i) for i in range(1, df.Value.max() + 1)]
#ax.set_yticklabels(labels)
plt.gcf().autofmt_xdate()
plt.grid()
plt.show()
Sample output:
Obviously, you don't see multiple entries.

Skipping certain values in Python with Matplotlib

I am currently working on an intra-day stock chart using the Alpha Vantage API. The data frame contains values from 4:00 to 20:00. In my matplotlib.pyplot chart however, the x-Axis also includes values from 20:00 to 4:00 over night. I dont want this as it messes up the aesthetics and also the Volume subplot.
Q: Is there any way to skip x-Axis values which dont exist in the actual Data Frame (the values from 20:00 to 04:00)?
As you can see, the Data Frame clearly jumps from 20:00 to 04:00
However in the Matplotlib chart, the x-Axis contains the values from 20:00 to 4:00, messing with the chart
Code so far. I believe so far everything is right:
import pandas as pd
import matplotlib.pyplot as plt
from alpha_vantage.timeseries import TimeSeries
import time
import datetime as dt
from datetime import timedelta as td
from dateutil.relativedelta import relativedelta
#Accessing and Preparing API
ts = TimeSeries(key=api_key, output_format='pandas')
ticker_input = "TSLA"
interval_input = "15min"
df, meta_data = ts.get_intraday(symbol = ticker_input, interval = interval_input, outputsize = 'full')
slice_date = 16*4*5
df = df[0:slice_date]
df = df.iloc[::-1]
df["100ma"] = df["4. close"].rolling(window = 50, min_periods = 0).mean()
df["Close"] = df["4. close"]
df["Date"] = df.index
#Plotting all as 2 different subplots
ax1 = plt.subplot2grid((7,1), (0,0), rowspan = 5, colspan = 1)
ax1.plot(df["Date"], df['Close'])
ax1.plot(df["Date"], df["100ma"], linewidth = 0.5)
plt.xticks(rotation=45)
ax2 = plt.subplot2grid((6,1), (5,0), rowspan = 2, colspan = 2, sharex = ax1)
ax2.bar(df["Date"], df["5. volume"])
ax2.axes.xaxis.set_visible(False)
plt.tight_layout()
plt.show()
It would be great if anybody could help. Im still a complete beginner and only started Python 2 weeks ago.

We got the data from the same place, although the data acquisition method is different. After extracting it in 15 units, I created a graph by excluding the data after 8pm and before 4pm. I created the code with the understanding that your skip would open up the pause. What you want it to skip is skipped once the NaN is set.
import datetime
import pandas as pd
import numpy as np
import pandas_datareader.data as web
import mplfinance as mpf
# import matplotlib.pyplot as plt
with open('./alpha_vantage_api_key.txt') as f:
api_key = f.read()
now_ = datetime.datetime.today()
start = datetime.datetime(2019, 1, 1)
end = datetime.datetime(now_.year, now_.month, now_.day - 1)
symbol = 'TSLA'
df = web.DataReader(symbol, 'av-intraday', start, end, api_key=api_key)
df.columns = ['Open', 'High', 'Low', 'Close', 'Volume']
df.index = pd.to_datetime(df.index)
df["100ma"] = df["Close"].rolling(window = 50, min_periods = 0).mean()
df["Date"] = df.index
df_15 = df.asfreq('15min')
df_15 = df_15[(df_15.index.hour >= 4)&(df_15.index.hour <= 20) ]
import matplotlib.pyplot as plt
fig = plt.figure(figsize=(8,4.5),dpi=144)
#Plotting all as 2 different subplots
ax1 = plt.subplot2grid((7,1), (0,0), rowspan = 5, colspan = 1)
ax1.plot(df_15["Date"], df_15['Close'])
ax1.plot(df_15["Date"], df_15["100ma"], linewidth = 0.5)
plt.xticks(rotation=20)
ax2 = plt.subplot2grid((6,1), (5,0), rowspan = 2, colspan = 2, sharex = ax1)
ax2.bar(df_15["Date"], df_15["Volume"])
ax2.axes.xaxis.set_visible(False)
# plt.tight_layout()
plt.show()

I fixed it using matplotlib.ticker.formatter.
I first created a class and using:
class MyFormatter(Formatter):
def __init__(self, dates, fmt='%Y-%m-%d %H:%M'):
self.dates = dates
self.fmt = fmt
def __call__(self, x, pos=0):
'Return the label for time x at position pos'
ind = int(np.round(x))
if ind >= len(self.dates) or ind < 0:
return ''
return self.dates[ind].strftime(self.fmt)
formatter = MyFormatter(df.index)
ax1 = plt.subplot2grid((7,1), (0,0), rowspan = 5, colspan = 1)
ax1.xaxis.set_major_formatter(formatter)
ax1.plot(np.arange(len(df)), df["Close"])
ax1.plot(np.arange(len(df)), df["100ma"], linewidth = 0.5)
ax1.xticks(rotation=45)
ax1.axis([xmin,xmax,ymin,ymax])
ax2 = plt.subplot2grid((6,1), (5,0), rowspan = 2, colspan = 2, sharex = ax1)
ax2.bar(np.arange(len(df)), df["5. volume"])
plt.show()
This gave me a smoother graph than the one before and also that recommended by r-beginner.
The only issue that I have is that if I zoom in the x-axis doesnt really change. it always has teh year, month, date, hour, and minute. Obviously I only want hour and minute when Im zoomed in further. I am yet to figure out how to do that

Cannot prepare proper labels in Matplotlib

I have very simple code:
from matplotlib import dates
import matplotlib.ticker as ticker
my_plot=df_h.boxplot(by='Day',figsize=(12,5), showfliers=False, rot=90)
I've got:
but I would like to have fewer labels on X axis. To do this I've add:
my_plot.xaxis.set_major_locator(ticker.MaxNLocator(12))
It generates fewer labels but values of labels have wrong values (=first of few labels from whole list)
What am I doing wrong?
I have add additional information:
I've forgoten to show what is inside DataFrame.
I have three columns:
reg_Date - datetime64 (index)
temperature - float64
Day - date converted from reg_Date to string, it looks like '2017-10' (YYYY-MM)
Box plot group date by 'Day' and I would like to show values 'Day" as a label but not all values
, for example every third one.

You were almost there. Just set ticker.MultipleLocator.
The pandas.DataFrame.boxplot also returns axes, which is an object of class matplotlib.axes.Axes. So you can use this code snippet to customize your labels:
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
center = np.random.randint(50,size=(10, 20))
spread = np.random.rand(10, 20) * 30
flier_high = np.random.rand(10, 20) * 30 + 30
flier_low = np.random.rand(10, 20) * -30
y = np.concatenate((spread, center, flier_high, flier_low))
fig, ax = plt.subplots(figsize=(10, 5))
ax.boxplot(y)
x = ['Label '+str(i) for i in range(20)]
ax.set_xticklabels(x)
ax.set_xlabel('Day')
# Set a tick on each integer multiple of a base within the view interval.
ax.xaxis.set_major_locator(ticker.MultipleLocator(5))
plt.xticks(rotation=90)

I think there is a compatibility issue with Pandas plots and Matplotlib formatters.
With the following code:
df = pd.read_csv('lt_stream-1001-full.csv', header=0, encoding='utf8')
df['reg_date'] = pd.to_datetime(df['reg_date'] , format='%Y-%m-%d %H:%M:%S')
df.set_index('reg_date', inplace=True)
df_h = df.resample(rule='H').mean()
df_h['Day']=df_h.index.strftime('%Y-%m')
print(df_h)
f, ax = plt.subplots()
my_plot = df_h.boxplot(by='Day',figsize=(12,5), showfliers=False, rot=90, ax=ax)
locs, labels = plt.xticks()
i = 0
new_labels = list()
for l in labels:
if i % 3 == 0:
label = labels[i]
i += 1
new_labels.append(label)
else:
label = ''
i += 1
new_labels.append(label)
ax.set_xticklabels(new_labels)
plt.show()
You get this chart:
But I notice that this is grouped by month instead of by day. It may not be what you wanted.
Adding the day component to the string 'Day' messes up the chart as there seems to be too many boxes.
df = pd.read_csv('lt_stream-1001-full.csv', header=0, encoding='utf8')
df['reg_date'] = pd.to_datetime(df['reg_date'] , format='%Y-%m-%d %H:%M:%S')
df.set_index('reg_date', inplace=True)
df_h = df.resample(rule='H').mean()
df_h['Day']=df_h.index.strftime('%Y-%m-%d')
print(df_h)
f, ax = plt.subplots()
my_plot = df_h.boxplot(by='Day',figsize=(12,5), showfliers=False, rot=90, ax=ax)
locs, labels = plt.xticks()
i = 0
new_labels = list()
for l in labels:
if i % 15 == 0:
label = labels[i]
i += 1
new_labels.append(label)
else:
label = ''
i += 1
new_labels.append(label)
ax.set_xticklabels(new_labels)
plt.show()
The for loop creates the tick labels every as many periods as desired. In the first chart they were set every 3 months. In the second one, every 15 days.
If you would like to see less grid lines:
df = pd.read_csv('lt_stream-1001-full.csv', header=0, encoding='utf8')
df['reg_date'] = pd.to_datetime(df['reg_date'] , format='%Y-%m-%d %H:%M:%S')
df.set_index('reg_date', inplace=True)
df_h = df.resample(rule='H').mean()
df_h['Day']=df_h.index.strftime('%Y-%m-%d')
print(df_h)
f, ax = plt.subplots()
my_plot = df_h.boxplot(by='Day',figsize=(12,5), showfliers=False, rot=90, ax=ax)
locs, labels = plt.xticks()
i = 0
new_labels = list()
new_locs = list()
for l in labels:
if i % 3 == 0:
label = labels[i]
loc = locs[i]
i += 1
new_labels.append(label)
new_locs.append(loc)
else:
i += 1
ax.set_xticks(new_locs)
ax.set_xticklabels(new_labels)
ax.grid(axis='y')
plt.show()
I've read about x_compat in Pandas plot in order to apply Matplotlib formatters, but I get an error when trying to apply it. I'll give it another shot later.
Old unsuccesful answer
The tick labels seem to be dates. If they are set as datetime in your dataframe, you can:
months = mdates.MonthLocator(1,4,7,10) #Choose the months you like the most
ax.xaxis.set_major_locator(months)
Otherwise, you can let Matplotlib know they are dates by:
ax.xaxis_date()
Your comment:
I have add additional information:
I've forgoten to show what is inside DataFrame.
I have three columns:
reg_Date - datetime64 (index)
temperature - float64
Day - date converted from reg_Date to string, it looks like '2017-10' *(YYYY-MM) *
Box plot group date by 'Day' and I would like to show values 'Day" as a label but not all values
, for example every third one.
Based on your comment in italic above, I would use reg_Date as the input and the following lines:
days = mdates.DayLocator(interval=3)
daysFmt = mdates.DateFormatter('%Y-%m') #to format display
ax.xaxis.set_major_locator(days)
ax.xaxis.set_major_formatter(daysFmt)
I forgot to mention that you will need to:
import matplotlib.dates as mdates
Does this work?

Set index values for python plot

I am graphing three lines on a single plot. I want the x-axis to display the date the data was taken on and the time from 00:00 to 24:00. Right now my code displays the time of day correctly but for the date, instead of the date that the data was recorded on being displayed, the current date is shown (12-18). I am unsure how to correct this. Also it would be acceptable for my plot to show only time from 00:00 to 24:00 with out the date on the x-axis. Thank you for your help!!
# set index as time for graphing
monAverages['Time'] = monAverages['Time'].apply(lambda x: pd.to_datetime(str(x)))
index = monAverages['Time']
index = index.apply(lambda x: pd.to_datetime(str(x)))
averagePlot = dfSingleDay
predictPlot = predictPlot[np.isfinite(predictPlot)]
datasetPlot = datasetPlot[np.isfinite(datasetPlot)]
predictPlot1 = pd.DataFrame(predictPlot)
datasetPlot1 = pd.DataFrame(datasetPlot)
averagePlot.set_index(index, drop=True,inplace=True)
datasetPlot1.set_index(index, drop=True,inplace=True)
predictPlot1.set_index(index, drop=True,inplace=True)
plt.rcParams["figure.figsize"] = (10,10)
plt.plot(datasetPlot1,'b', label='Real Data')
plt.plot(averagePlot, 'y', label='Average for this day of the week')
plt.plot(predictPlot1, 'g', label='Predictions')
plt.title('Power Consumption')
plt.xlabel('Date (00-00) and Time of Day(00)')
plt.ylabel('kW')
plt.legend()
plt.show()

You need to be sure that you get only the time:
import matplotlib.dates as mdates
# set index as time for graphing
monAverages['Time'] = monAverages['Time'].apply(lambda x: pd.to_datetime(str(x)))
index = monAverages['Time']
#index = index.apply(lambda x: pd.to_datetime(str(x)))
dates= [dt.datetime.strptime(d,'%Y-%m-%d %H:%M:%S').time() for d in index]
averagePlot = dfSingleDay
predictPlot = predictPlot[np.isfinite(predictPlot)]
datasetPlot = datasetPlot[np.isfinite(datasetPlot)]
predictPlot1 = pd.DataFrame(predictPlot)
datasetPlot1 = pd.DataFrame(datasetPlot)
plt.rcParams["figure.figsize"] = (10,10)
plt.plot(dates,datasetPlot1,'b', label='Real Data')
plt.plot(dates,averagePlot, 'y', label='Average for this day of the week')
plt.plot(dates,predictPlot1, 'g', label='Predictions')
plt.title('Power Consumption')
plt.xlabel('Date (00-00) and Time of Day(00)')
plt.ylabel('kW')
plt.legend()
plt.show()
This code here explains how you can run it
import datetime as dt
import matplotlib.pyplot as plt
dates = ['2019-12-18 00:00:00','2019-12-18 12:00:00','2019-12-18 13:00:00']
x = [dt.datetime.strptime(d,'%Y-%m-%d %H:%M:%S').time() for d in dates]
y = range(len(x))
plt.plot(x,y)
plt.gcf().autofmt_xdate()
plt.show()