i am using PandasModel(QAbstractTableModel) to show data in a qtableView. it's working fine with less data. but whenever i tried with large datas it became slow. for 5000 rows data it almost took 20-25 seconds .it just take a seconds in mssql mgmt studio to run the query. i am not getting where i did wrong with my codes.
class PandasModel(QtCore.QAbstractTableModel):
def __init__(self, df = pd.DataFrame(), parent=None):
QtCore.QAbstractTableModel.__init__(self, parent=parent)
self._df = df
def headerData(self, section, orientation, role=QtCore.Qt.DisplayRole):
if role != QtCore.Qt.DisplayRole:
return QtCore.QVariant()
if orientation == QtCore.Qt.Horizontal:
try:
return self._df.columns.tolist()[section]
except (IndexError, ):
return QtCore.QVariant()
elif orientation == QtCore.Qt.Vertical:
try:
return self._df.index.tolist()[section]+1
except (IndexError, ):
return QtCore.QVariant()
def data(self, index, role=QtCore.Qt.DisplayRole):
if role != QtCore.Qt.DisplayRole:
return QtCore.QVariant()
if not index.isValid():
return QtCore.QVariant()
return QtCore.QVariant(str(self._df.iloc[index.row(), index.column()]))
def setData(self, index, value, role):
if not index.isValid():
return False
if role != QtCore.Qt.EditRole:
return False
row = index.row()
if row < 0 or row >= len(self._data.values):
return False
column = index.column()
if column < 0 or column >= self._data.columns.size:
return False
self._data.values[row][column] = value
self.dataChanged.emit(index, index)
return True
def rowCount(self, parent=QtCore.QModelIndex()):
return len(self._df.index)
def columnCount(self, parent=QtCore.QModelIndex()):
return len(self._df.columns)
def sort(self, column, order):
colname = self._df.columns.tolist()[column]
self.layoutAboutToBeChanged.emit()
self._df.sort_values(colname, ascending= order == QtCore.Qt.AscendingOrder, inplace=True)
self._df.reset_index(inplace=True, drop=True)
self.layoutChanged.emit()
i tried many approaches like st the chunksize , but not working. please help me out
sql_conn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER=' +
server+';DATABASE='+database+';UID='+username+';PWD=' + password)
cursor = sql_conn.cursor()
def loadTranForACC(AccountID):
query_string="Select * from user.dbo.Transaction(nolock) where accID =" + AccountID
data=pd.read_sql_query(query_string, sql_conn)
return data
Since you are working with the data from Database, I'd like to point out that PyQt has
QSqlTableModel and QSqlQueryModel to work with the data from database.
Looking at your model, to me, the model looks fine, try to see exactly what part of your program is eating up time.
You can look at these threads for more information and details regarding these:
qtablewidget becomes slow for large tables
pyqt qtablewidget extremely slow
The ultimate solution though would be to use pagination, show only 500,1000 rows at a time, it will run within a fraction of seconds.
You can visit these threads to implement pagination: QT Forum | Pagination in QTableWidget or Paginated Display of Table Data in PyQt
Related
i want to set a comboBox and show my table's column1 data as items and associated value column2 is for selected item id and want to setText in a qLabel.
i am using a model to view the items in comboBox and working fine. i can get the currentText value but how to get the associated value for the items.
in my case :
my sql table like:
type id
------------
DIV 2
TRANS 33
FEE 41
EXP 89
now , i can set the column1(type) values into comboBox successfully. now, if user selected value 'FEE' , than qlable should be updated as its associate id : 41. how to do that!
df=loadData()
model=PandasModel(df)
self.comboBox.setModel(model)
self.comboBox.setModelColumn(0)
content=self.comboBox.currentText()
self.label.setText(content) # content should be ID instead of currentText
pandasMode:
class PandasModel(QtCore.QAbstractTableModel):
def __init__(self, df = pd.DataFrame(), parent=None):
QtCore.QAbstractTableModel.__init__(self, parent=parent)
self._df = df
def headerData(self, section, orientation, role=QtCore.Qt.DisplayRole):
if role != QtCore.Qt.DisplayRole:
return QtCore.QVariant()
if orientation == QtCore.Qt.Horizontal:
try:
return self._df.columns.tolist()[section]
except (IndexError, ):
return QtCore.QVariant()
elif orientation == QtCore.Qt.Vertical:
try:
return self._df.index.tolist()[section]+1
except (IndexError, ):
return QtCore.QVariant()
def data(self, index, role=QtCore.Qt.DisplayRole):
if index.isValid():
if role == QtCore.Qt.DisplayRole:
return str(self._df.iloc[index.row(), index.column()])
return None
def setData(self, index, value, role):
if not index.isValid():
return False
if role != QtCore.Qt.EditRole:
return False
row = index.row()
if row < 0 or row >= len(self._df.values):
return False
column = index.column()
if column < 0 or column >= self._df.columns.size:
return False
self._df.values[row][column] = value
self.dataChanged.emit(index, index)
return True
# def rowCount(self, parent=QtCore.QModelIndex()):
# return len(self._df.index)
def rowCount(self, parent=None):
return len(self._df.index)
def columnCount(self, parent=QtCore.QModelIndex()):
return len(self._df.columns)
def sort(self, column, order):
colname = self._df.columns.tolist()[column]
self.layoutAboutToBeChanged.emit()
self._df.sort_values(colname, ascending= order == QtCore.Qt.AscendingOrder, inplace=True)
self._df.reset_index(inplace=True, drop=True)
self.layoutChanged.emit()
Assuming that the model stores the id in the second column then it has to obtain the associated index:
ID_COLUMN = 1
index = self.comboBox.model().index(
self.comboBox.currentIndex(), ID_COLUMN, self.comboBox.rootModelIndex()
)
id_ = index.data()
print(id_)
self.label.setText(id_)
I have created a model class of type QAbstractTableModel to which I have added a number of methods as shown here:
class resultsModel(QAbstractTableModel):
def __init__(self, parent, headerData, arraydata, *args):
QAbstractTableModel.__init__(self, parent, *args)
self.arraydata = arraydata
self.headerdata = headerData #['Timestamp', 'Force (N)', 'Diplacement (mm)']
def rowCount(self, parent):
return len(self.arraydata)
def columnCount(self, parent):
if len(self.arraydata) > 0:
return len(self.arraydata[0])
return 0
def headerData(self, col, orientation, role):
if orientation == Qt.Horizontal and role == Qt.DisplayRole:
return self.headerdata[col]
return None
#Make table cells non-editable
def flags(self, index):
return Qt.ItemIsEnabled | Qt.ItemIsSelectable
def data(self, index, role):
if not index.isValid():
return None
value = self.arraydata[index.row()][index.column()]
if role == Qt.EditRole:
return value
elif role == Qt.DisplayRole:
return value
def setData(self, index, value, role):
if not index.isValid():
return None
row = index.row()
column = index.column()
self.arraydata[row][column] = value
self.dataChanged.emit(index, index)
return True
def setHeaderData(self, col, orientation, value, role):
if role != Qt.DisplayRole or orientation != Qt.Horizontal:
return False
self.headerdata[col] = value
result = self.headerdata[col]
if result:
self.headerDataChanged.emit(orientation, col, col)
return result
When the application starts I use a QTableview with the model above behind it:
resultsHeaders = ['Timestamp', 'Force (N)', 'Diplacement (mm)']
resultsData = [['','','']]
self.resultsTableModel = resultsModel(self, resultsHeaders, resultsData)
self.resultsTable = QTableView()
self.resultsTable.setModel(self.resultsTableModel)
During runtime, if a serial device is connected, I want to add some additional columns to the model but I am having difficulty implementing an 'insertColumns' method for the model before I add the new header values.
#insert columns
???
#update header values
for i in range(len(resultsHeaders)):
self.resultsTable.model().setHeaderData(i, Qt.Horizontal, str(resultsHeaders[int(i)]), Qt.DisplayRole)
An appropriate "insert column(s)" function must be implemented, as the model should correctly be updated to reflect the actual column count, so that the new headers are also correctly shown.
In order to correctly add columns, both beginInsertColumn() (before adding new column data) and endInsertColumn() (at the end of the operation) should be properly called.
Normally, adding more columns would require to implement insertColumns(), but considering the situation and the fact that they would call both methods anyway, there's no need to do that, and a custom function will be fine.
The following only inserts a single column, if you want to add more columns at once, just ensure that the third argument of beginInsertColumns correctly reflects that (newColumn + len(columnsToAdd)) and that the header data and new empty values for the array correspond to the new column count.
class ResultsModel(QAbstractTableModel):
# ...
def addColumn(self, name):
newColumn = self.columnCount()
self.beginInsertColumns(QModelIndex(), newColumn, newColumn)
self.headerdata.append(name)
for row in self.arraydata:
row.append('')
self.endInsertColumns()
# ...
class Whatever(QWidget):
# ...
def addColumn(self):
name = 'New column {}'.format(self.resultsTableModel.columnCount())
self.resultsTableModel.addColumn(name)
Note that:
both rowCount() and columnCount() should have a default keyword parent argument as the base implementation does (accepted practice involves parent=QModelIndex());
class names should always be capitalized, so you should rename your model class to ResultsModel; read more on the topic on the official Style Guide for Python Code;
So, Im currently taking just limited data from a ZeroMQ Messaging system appending it all to a Dataframe and then turning the Dataframe into a pandas model with code I acquired from someone on stack. Then running that model through a PyQt5 Tableview.
Every time I run the tableview code, after converting my Dataframe to a model for Tableview the kernal just dies. I tried to at least handle a exception but it wont even raise a runtimeerror that is super general. It just dies every time....
For the purpose of the test im using a CSV with data to try to get this into a workable model. You can use any csv or data you have on your end to test this as theres no hard coded formatting.
from PyQt5 import QtCore, QtGui, QtWidgets
import pandas as pd
class PandasModel(QtCore.QAbstractTableModel):
def __init__(self, df = pd.DataFrame(), parent=None):
QtCore.QAbstractTableModel.__init__(self, parent=parent)
self._df = df
def headerData(self, section, orientation, role=QtCore.Qt.DisplayRole):
if role != QtCore.Qt.DisplayRole:
return QtCore.QVariant()
if orientation == QtCore.Qt.Horizontal:
try:
return self._df.columns.tolist()[section]
except (IndexError, ):
return QtCore.QVariant()
elif orientation == QtCore.Qt.Vertical:
try:
# return self.df.index.tolist()
return self._df.index.tolist()[section]
except (IndexError, ):
return QtCore.QVariant()
def data(self, index, role=QtCore.Qt.DisplayRole):
if role != QtCore.Qt.DisplayRole:
return QtCore.QVariant()
if not index.isValid():
return QtCore.QVariant()
return QtCore.QVariant(str(self._df.ix[index.row(), index.column()]))
def setData(self, index, value, role):
row = self._df.index[index.row()]
col = self._df.columns[index.column()]
if hasattr(value, 'toPyObject'):
# PyQt4 gets a QVariant
value = value.toPyObject()
else:
# PySide gets an unicode
dtype = self._df[col].dtype
if dtype != object:
value = None if value == '' else dtype.type(value)
self._df.set_value(row, col, value)
return True
def rowCount(self, parent=QtCore.QModelIndex()):
return len(self._df.index)
def columnCount(self, parent=QtCore.QModelIndex()):
return len(self._df.columns)
def sort(self, column, order):
colname = self._df.columns.tolist()[column]
self.layoutAboutToBeChanged.emit()
self._df.sort_values(colname, ascending= order == QtCore.Qt.AscendingOrder, inplace=True)
self._df.reset_index(inplace=True, drop=True)
self.layoutChanged.emit()
def createview(title, model):
try:
view = QtWidgets.QTableView()
view.setWindowview
except:
raise RuntimeError("I know python!")
if __name__=="__main__":
df=pd.read_csv("C:\Excel Sheets\Test_CSV_6-18-18.csv")
model = PandasModel(df)
createview("Model", model)
----How The Problem Looks Like----
after I clicked on the Record button, it becomes this
As you can see, the table just shifted to the left, hiding the vertical headers.
----Related Codes----
My tableivew is called tableViewTransaction, it uses a model to link to data in pandas' dataframe format. Here is how I connected them together inside the __init__ function of a QMainWindow.
self.data = pd.read_csv('transactions.txt', sep='\t', header=None)
self.data.columns = ["Name", "Price", "Action"]
self.model = PandasModel(self.data)
self.tableViewTransaction.setModel(self.model)
Here is my Model for the TableView
class PandasModel(QtCore.QAbstractTableModel):
def __init__(self, df = pd.DataFrame(), parent=None):
QtCore.QAbstractTableModel.__init__(self, parent=parent)
self._df = df
def headerData(self, section, orientation, role=QtCore.Qt.DisplayRole):
if role != QtCore.Qt.DisplayRole:
return QtCore.QVariant()
if orientation == QtCore.Qt.Horizontal:
try:
return self._df.columns.tolist()[section]
except (IndexError, ):
return QtCore.QVariant()
elif orientation == QtCore.Qt.Vertical:
try:
# return self.df.index.tolist()
return self._df.index.tolist()[section]
except (IndexError, ):
return QtCore.QVariant()
def data(self, index, role=QtCore.Qt.DisplayRole):
if role != QtCore.Qt.DisplayRole:
return QtCore.QVariant()
if not index.isValid():
return QtCore.QVariant()
return QtCore.QVariant(str(self._df.ix[index.row(), index.column()]))
def setData(self, index, value, role):
row = self._df.index[index.row()]
col = self._df.columns[index.column()]
if hasattr(value, 'toPyObject'):
# PyQt4 gets a QVariant
value = value.toPyObject()
else:
# PySide gets an unicode
dtype = self._df[col].dtype
if dtype != object:
value = None if value == '' else dtype.type(value)
self._df.set_value(row, col, value)
return True
def rowCount(self, parent=QtCore.QModelIndex()):
return len(self._df.index)
def columnCount(self, parent=QtCore.QModelIndex()):
return len(self._df.columns)
def sort(self, column, order):
colname = self._df.columns.tolist()[column]
self.layoutChanged.emit()
self._df.sort_values(colname, ascending= order == QtCore.Qt.AscendingOrder, inplace=True)
self._df.reset_index(inplace=True, drop=True)
self.layoutChanged.emit()
When the Record button is pressed, the following function is called:
def recordTransaction(self):
name = self.comboBoxStock.currentText()
price = self.lineEditMoney.text()
action = self.comboBoxAction.currentText()
self.data.loc[len(self.data.index)] = [name, price, action]
self.tableViewTransaction.model().layoutChanged.emit()
I know that name, price, action here is storing the correct information i.e. "stock1", "2", "Buy" in my above example.
----Full Code----
https://drive.google.com/file/d/1rU66yLqlQu0bTdINkSWxJe2E_OdMtS0i/view?usp=sharing
How to use:
first unzip it, then use python3 to run StockSim.py.
On a mac, just run python3 StockSim.py when you have went to the Stock Sim directory using terminal. Make sure you have python3 and pyqt5 installed first.
----------------------
All I want is that the TableView not shift to the left, any of your help would be very much appreciated!
I'm relatively new to Python and especially PyQt and model-view programming. I want to have it so that someone can only enter integers in my table and not letters/symbols etc. Here's the model I've made so far for my tableView widget:
class PixelTableModel(QtCore.QAbstractTableModel):
def __init__(self):
super(PixelTableModel, self).__init__()
self.pixel_coordinate = [[None, None, None, None]]
def rowCount(self, parent):
return 1
def columnCount(self, parent):
return 4
def flags(self, index):
return QtCore.Qt.ItemIsEditable | QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable
def setData(self, index, value, role = QtCore.Qt.EditRole):
if role == QtCore.Qt.EditRole:
row = index.row()
column = index.column()
self.pixel_coordinate[row][column] = value
print(self.pixel_coordinate) #testing
return True
return False
def data(self, index, role):
if role == QtCore.Qt.DisplayRole:
row = index.row()
column = index.column()
value = self.pixel_coordinate[row][column]
return value
def headerData(self, section, orientation, role): # section = row column, orientation = vertical/horizontal
if role == QtCore.Qt.DisplayRole:
if orientation == QtCore.Qt.Horizontal:
dict = {0: "Xstart", 1: "Ystart", 2: "Xmax", 3: "Ymax"}
for key in dict:
if section == key:
return dict[key]
else:
return "Pixel coordinate"
It seems to work except obviously for the part where it still can take letters/symbols as input in the tableView. I've tried a couple of things in the setData() method but can't seem to get it work, always get some type of error or it won't even change the box at all. Thanks to anyone that can help me with this. Also sorry for bad English.
For anyone still interested, after going through it again fixed it with simple try except block:
def setData(self, index, value, role = QtCore.Qt.EditRole):
if role == QtCore.Qt.EditRole:
try:
row = index.row()
column = index.column()
string_to_int = int(value)
self.pixel_coordinate[row][column] = value
print(self.pixel_coordinate) #testing
return True
except ValueError:
print("Not a number")
return False
return False