I'm trying to create a sqlite database that takes inputs from a CSV file using flask-sqlalchelmy.
Here is the config file:
app = Flask(__name__)
basedir = os.path.abspath(os.path.dirname(__file__))
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+os.path.join(basedir,'applicant.db')
app.config['SQLALCHEMY_TRACK _MODIFICATIONS']=False
cors = CORS(app)
db = SQLAlchemy(app)
app.run(debug=True)
Here are the models:
class attributes(db.Model):
__tablename__ = 'attributes'
id = Column(Integer, primary_key=True)
URL = Column(Text)
Gender = Column(String(10))
SAT =Column(Integer)
ACT = Column(Integer)
major =db.relationship('Major',backref = 'attributes')
ecs = db.relationship('Ecs', backref = 'attributes')
Race = db.relationship('Race',backref = 'attributes')
Acceptances = db.relationship('Acceptances', backref = 'attributes')
Rejections = db.relationship('Rejections', backref = 'attributes')
class Ecs(db.Model):
id = Column(Integer, primary_key = True)
listofecs = Column(Text)
Attributeid = Column(Integer, db.ForeignKey('attributes.id'))
def __repr__(self):
return f'<Ecs "{self.title}">'
class Major(db.Model):
id = Column(Integer, primary_key = True)
majorlist = Column(Text)
Attributeid = Column(Integer, db.ForeignKey('attributes.id'))
def __repr__(self):
return f'<major "{self.title}">'
class Race(db.Model):
id = Column(Integer, primary_key = True)
racelist = Column(Text)
Attributeid = Column(Integer, db.ForeignKey('attributes.id'))
def __repr__(self):
return f'<race "{self.title}">'
class Acceptances(db.Model):
id = Column(Integer, primary_key = True)
acceptlist = Column(Text)
Attributeid = Column(Integer, db.ForeignKey('attributes.id'))
def __repr__(self):
return f'<acceptances "{self.title}">'
class Rejections(db.Model):
id = Column(Integer, primary_key = True)
rejectlist = Column(Text)
Attributeid = Column(Integer, db.ForeignKey('attributes.id'))
def __repr__(self):
return f'<rejections "{self.title}">'
They are mostly one-to-many relationships - but i'm not sure if thats the issue.
I'm trying to create the database by using this script:
import pandas as pd
from app import attributes, Major, Ecs, Race, Acceptances,Rejections, db, df
from ast import literal_eval
df = pd.read_csv('..\csvfiles\processeddata.csv')
for i in range(len(df['Gender'])):
applicant = attributes(Gender = df['Gender'].iloc[i],SAT = df['SAT'].iloc[i], ACT = df['ACT'].iloc[i])
db.session.add(applicant)
for racevalues in df['Race'].apply(literal_eval).iloc[i]:
race = Race(racelist = racevalues, Attributeid = i+1)
db.session.add(race)
for majorvalues in df['Major'].apply(literal_eval).iloc[i]:
major = Major(majorlist = majorvalues,Attributeid = i+1 )
db.session.add(major)
for ecvalues in df['Extracurriculars'].apply(literal_eval).iloc[i]:
ecs = Ecs(listofecs = ecvalues , Attributeid = i+1)
db.session.add(ecs)
for valuesacceptances in df["Acceptances"].apply(literal_eval).iloc[i]:
Accepts = Acceptances(acceptlist = valuesacceptances, Attributeid = i+1)
db.session.add(Accepts)
for valuesreject in df["Rejections"].apply(literal_eval).iloc[i]:
Rejects = Rejections(rejectlist = valuesreject, Attributeid = i+1)
db.session.add(Rejects)
db.create_all()
db.session.commit()
df is the data that is taken from the csv file that I want to be inserted into the database. I have confirmed that it is the correct file and has data by printing df. I have also tried to manually create the database using db.create_all() in the command line through the flask shell. This creates the scheme for the database but even after running the script, there is no data inserted into the database. Further, when I try to query the data onto a route the route indeed works but returns an empty array.
The strange thing is that this worked perfectly fine for a few days. It stopped working after my computer crashed and I wonder if that has something to do with it.
Related
I have a one to many relationship between a Project and a Note (i.e. a project will have many notes but a note belongs to a single project) in my Flask app:
class BaseDocument(db.Model):
__abstract__ = True
created_at = db.Column(db.DateTime, default=datetime.now)
updated_at = db.Column(db.DateTime, onupdate=datetime.now)
archived = db.Column(db.Boolean, default=False)
def __repr__(self):
return str(self.__dict__)
class Project(BaseDocument):
__tablename__ = "project"
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String)
parent_id = db.Column(db.Integer, db.ForeignKey("project.id"))
notes = db.relationship(
"Note",
backref="project",
lazy=True,
order_by="Note.updated_at",
cascade="all, delete, delete-orphan",
)
class Note(BaseDocument):
__tablename__ = "note"
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String)
content = db.relationship(
"Bullet", backref="note", lazy=True, order_by="Bullet.order"
)
project_id = db.Column(db.Integer, db.ForeignKey("project.id"))
I would like to do something that seems to be very simple but I can't figure out how: I want to update the archived property of all the child notes in a project to True if the parent project archived property is also set to True.
I can only find answers here in StackOverfow about how to update the parent object if a child is updated (the oposite of what I am trying to do), so I am assuming that what I want to do is trivial and I am just bad at sqlalchemy. How can I set this up? Do I need to use a after_update event on Project? If yes, how can I access all the child Notes and set archived=True for all of them?
I have tried to setup the following event listener with no success, I get the following error AttributeError: 'InstrumentedList' object has no attribute 'update':
#db.event.listens_for(Project, "after_update")
def archive_notes(mapper, connection, target):
obj = target.object
connection.execute(target.notes.update(archived=True))
Any help will be very appreciated!
You're on the right track by using after_update. Here's a working example:
import sqlalchemy as sa
from sqlalchemy.orm import declarative_base, relationship
connection_uri = (
"mssql+pyodbc://#localhost:49242/myDb?driver=ODBC+Driver+17+for+SQL+Server"
)
engine = sa.create_engine(
connection_uri,
future=True,
echo=False,
)
Base = declarative_base()
class Project(Base):
__tablename__ = "project"
id = sa.Column(sa.Integer, primary_key=True)
title = sa.Column(sa.Unicode(100), nullable=False)
archived = sa.Column(sa.Boolean, nullable=False, default=False)
class ProjectNote(Base):
__tablename__ = "project_note"
id = sa.Column(sa.Integer, primary_key=True)
project_id = sa.Column(sa.Integer, sa.ForeignKey("project.id"))
project = relationship(Project)
note_text = sa.Column(sa.Unicode(255), nullable=False)
archived = sa.Column(sa.Boolean, nullable=False, default=False)
#sa.event.listens_for(Project, "after_update")
def archive_remaining_project_notes(mapper, connection, target):
if target.archived:
sql = """\
UPDATE project_note SET archived = :yes
WHERE project_id = :proj_id
AND archived = :no
"""
connection.execute(
sa.text(sql),
{"yes": True, "no": False, "proj_id": target.id},
)
# <just for testing>
Base.metadata.drop_all(engine, checkfirst=True)
Base.metadata.create_all(engine)
# </just for testing>
p1 = Project(title="project 1")
p1n1 = ProjectNote(
project=p1, note_text="project 1, note 1, archived", archived=True
)
p1n2 = ProjectNote(project=p1, note_text="project 1, note 2, not archived")
with sa.orm.Session(engine, future=True) as session:
session.add_all([p1, p1n1, p1n2])
session.commit()
print(f"p1n2.archived is: {p1n2.archived}") # p1n2.archived is: False
p1.archived = True
session.commit()
print(f"p1.archived is: {p1.archived}") # p1.archived is: True
print(f"p1n2.archived is: {p1n2.archived}") # p1n2.archived is: True
I have two tables:
class Project(DataBase):
__tablename__ = 'projects'
id = Column(Integer, primary_key=True, autoincrement=True, nullable=False)
name = Column(String, nullable=False, unique=True)
domain = Column(String, nullable=False)
phrases = relationship("Phrase", backref='proj')
def __init__(self, name, domain):
self.name = name
self.domain = domain
class Phrase(DataBase):
__tablename__ = 'phrases'
query_id = Column(Integer, primary_key=True, autoincrement=True, nullable=False)
query_text = Column(String, nullable=False)
project = Column(Integer, ForeignKey('projects.id'), nullable=False)
enable = Column(Boolean, nullable=False, default=True)
def __init__(self, query_text, project, city):
self.query_text = query_text
self.project = project
And I have a function:
def get_first_query():
session = Session(bind=engine)
q = session.query(Phrase).filter(Phrase.enable == True).first()
session.close()
return q
I want to get an object from table 2 and than get its parrent from first table:
session = Session(bind=engine)
q = get_first_query()
print(q.proj)
It doesn't work and print this error:
sqlalchemy.orm.exc.DetachedInstanceError: Parent instance is not bound to a Session; lazy load operation of
attribute 'proj' cannot proceed
I can do this:
session = Session(bind=engine)
q = get_first_query()
q_project = session.query(Project).filter(Project.id == q.project)
But it's a bad way.
You can assess related object via proj attribute.
session.query(Phrase).filter(
Phrase.enable == True
).first().proj
This way you'll hit database one additional time to get it so you'll need to open session again. To avoid additional queries you can use joined load:
session.query(Phrase).filter(
Phrase.enable == True
).options(
joinedload('proj')
).first().proj
Here is my problem. I have three tables.
One called Project which has only one Column called id (this must be unique throughout the system).
One called ServiceAwarenessProject which has a one to one relationship with Project.id.
One called CorporateVPNProject which has a one to one relationship with Project.id
I'm using sqlalchemy ORM, so the code looks like the one below:
class Project(SqlAlchemyBase):
__tablename__ = 'project'
id = Column(Integer, primary_key=True, autoincrement=True)
class ServiceAwarenessProject(SqlAlchemyBase):
__tablename__ = 'sa_project'
id = Column(Integer, primary_key=True)
project_id = Column(Integer, ForeignKey(Project.id))
mop_url = Column(String, nullable=False)
expiration_date = Column(Datetime, index=True)
class CorporateVPNProject(SqlAlchemyBase):
__tablename__ = 'wvpn_project'
id = Column(Integer, primary_key=True)
project_id = Column(Integer, ForeignKey(Project.id))
mop_url = Column(String, nullable=False)
I designed my tables like that, so I can guarantee I have unique project_ids in the entire system. My problem is that I don't know how to join those tables together to find a project based on the project_id. To solve this problem for now, I'm querying both tables, using the function called get_project_by_id.
Is there a smarter way to solve this issue?
class ProjectService:
#staticmethod
def create_project_id():
session = DbSessionFactory.create_session()
result = session.query(Project.id).order_by(desc(Project.id)).first()
if result:
result = result[0]
if str(result)[:8] == datetime.datetime.now().strftime('%Y%m%d'):
project_id = str(result)[:8] + '{:03d}'.format(int(str(result)[8:]) + 1)
new_project = Project(id=project_id)
session.add(new_project)
session.commit()
return project_id
project_id = datetime.datetime.now().strftime('%Y%m%d') + '001'
new_project = Project(id=project_id)
session.add(new_project)
session.commit()
return project_id
#staticmethod
def get_project_by_id(project_id):
session = DbSessionFactory.create_session()
result = session.query(ServiceAwarenessProject) \
.filter(ServiceAwarenessProject.project_id == project_id) \
.first()
if result:
return result
result = session.query(CorporateVPNProject) \
.filter(CorporateVPNProject.project_id == project_id) \
.first()
if result:
return result
def create_serviceawareness_project(self):
session = DbSessionFactory.create_session()
project_id = self.create_project_id()
new_project = ServiceAwarenessProject(project_id=project_id, mop_url='http://www.thepacketwizards.com/1')
session.add(new_project)
session.commit()
return new_project
def create_corporatevpn_project(self):
session = DbSessionFactory.create_session()
project_id = self.create_project_id()
new_project = CorporateVPNProject(project_id=project_id, mop_url='http://www.thepacketwizards.com/wvpn')
session.add(new_project)
session.commit()
return new_project
Thank you!
Following #Ilja Everilä suggestion, I designed the table like so, using only joined table inheritance.
class Project(SqlAlchemyBase):
__tablename__ = 'project'
id = Column(Integer, primary_key=True)
created_on = Column(DateTime, default=datetime.datetime.now)
updated_on = Column(DateTime, default=datetime.datetime.now, onupdate=datetime.datetime.now)
project_url = Column(String(60))
mop_url = Column(String(60))
input_url = Column(String(60))
type = Column(String(60))
__mapper_args__ = {
'polymorphic_identity': 'project',
'polymorphic_on': type
}
class ServiceAwarenessProject(Project):
__tablename__ = 'sa_project'
id = Column(Integer, ForeignKey('project.id'), primary_key=True)
expiration_date = Column(DateTime)
__mapper_args__ = {
'polymorphic_identity': 'ServiceAwareness',
}
class CorporateVPNProject(Project):
__tablename__ = 'wvpn_project'
id = Column(Integer, ForeignKey('project.id'), primary_key=True)
client_name = Column(String(60))
__mapper_args__ = {
'polymorphic_identity': 'CorporateVPN',
}
Now, to query the DB I have to use with_polymorphic, so I can get different instances of Tables per row.
class ProjectService:
#staticmethod
def create_project_id():
session = DbSessionFactory.create_session()
result = session.query(Project.id).order_by(desc(Project.id)).first()
print(result)
if result:
result = result[0]
if str(result)[:8] == datetime.datetime.now().strftime('%Y%m%d'):
project_id = str(result)[:8] + '{:03d}'.format(int(str(result)[8:]) + 1)
return project_id
project_id = datetime.datetime.now().strftime('%Y%m%d') + '001'
return project_id
def create_serviceawareness_project(self):
session = DbSessionFactory.create_session()
project_id = self.create_project_id()
new_project = ServiceAwarenessProject(id=project_id,
project_url='http://project',
expiration_date=datetime.datetime.now() + datetime.timedelta(days=365),
mop_url='http://mop',
input_url='http://url',
type='ServiceAwareness')
session.add(new_project)
session.commit()
session.add(new_project)
return new_project
def create_corporatevpn_project(self):
session = DbSessionFactory.create_session()
project_id = self.create_project_id()
new_project = CorporateVPNProject(id=project_id,
project_url='http://project',
client_name='TIM',
mop_url='http://mop',
input_url='http://url',
type='CorporateVPN')
session.add(new_project)
session.commit()
session.add(new_project)
return new_project
#staticmethod
def get_project_by_id(project_id):
session = DbSessionFactory.create_session()
query = session.query(with_polymorphic(Project, [ServiceAwarenessProject, CorporateVPNProject])).filter(or_(
ServiceAwarenessProject.id == project_id,
CorporateVPNProject.id == project_id
)).first()
return query
I created a Table a Bmarks which has two foreign keys which have relation with same table Url_hash
class Hashed(Base):
__tablename__ = "url_hash"
hash_id = Column(Unicode(22), primary_key=True)
url = Column(UnicodeText)
clicks = Column(Integer, default=0)
def __init__(self, url):
cleaned_url = str(unidecode(url))
self.hash_id = unicode(generate_hash(cleaned_url))
self.url = url
class Bmark(Base):
__tablename__ = "bmarks"
bid = Column(Integer, autoincrement=True, primary_key=True)
hash_id = Column(Unicode(22), ForeignKey('url_hash.hash_id'))
clean_hash_id = Column(Unicode(22), ForeignKey('url_hash.hash_id'))
description = Column(UnicodeText())
extended = Column(UnicodeText())
stored = Column(DateTime, default=datetime.utcnow)
updated = Column(DateTime, onupdate=datetime.utcnow)
clicks = Column(Integer, default=0)
inserted_by = Column(Unicode(255))
username = Column(Unicode(255), ForeignKey('users.username'),
nullable=False,)
tag_str = Column(UnicodeText())
hashed = relation(Hashed,
foreign_keys="Bmark.hash_id",
backref="bmark",
uselist=False
)
clean_hashed = relation(Hashed,
foreign_keys="Bmark.clean_hash_id",
backref="bmark",
uselist=False
)
I am trying to store url after cleaning it a little bit like removing headers,utm parameters etc for indexing purposes
Error is occurring while creating the database
sqlalchemy.exc.ArgumentError: Error creating backref 'bmark' on relationship 'Bmark.clean_hashed': property of that name exists on mapper 'Mapper|Hashed|url_hash'
Actually the error message is very informative.
Just rename one of your backref="bmark" to something else like backref="my_clean_bmark".
I have user who can have his favorite series and there are episodes which have series as foreign key and I am trying to retrieve all episodes from favorite series of user.
I am using Flask-SQLAlchemy.
Database:
db = SQLAlchemy(app)
# cross table for user-series
favorite_series = db.Table('favorite_series',
db.Column('user_id', db.Integer, db.ForeignKey('user.id')),
db.Column('series_id', db.Integer, db.ForeignKey('series.id'))
)
# user
class User(db.Model):
__tablename__ = 'user'
id = db.Column(db.Integer, primary_key=True)
favorite_series = db.relationship('Series', secondary=favorite_series,
backref=db.backref('users', lazy='dynamic'))
# series
class Series(db.Model):
__tablename__ = 'series'
id = db.Column(db.Integer, primary_key=True)
# episode
class Episode(db.Model):
__tablename__ = 'episode'
id = db.Column(db.Integer, primary_key=True)
series_id = db.Column(db.Integer, db.ForeignKey('series.id'))
series = db.relationship('Series',
backref=db.backref('episodes', lazy='dynamic'))
Friend helped me with SQL
select user_id,series.name,episode.name from (favorite_series left join series on favorite_series.series_id = series.id) left join episode on episode.series_id = series.id where user_id=1;
Altough, I want it in SQLAlchemy API, but can't manage to get it working.
EDIT:
My final working result:
episodes = Episode.query.filter(Episode.series_id.in_(x.id for x in g.user.favorite_series)).filter(Episode.air_time!=None).order_by(Episode.air_time)
First of all you don't seem to be declaring your table names?
Also, the whole point of bothering with orm is so you never have to write sql queries:
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import orm
import sqlalchemy as db
Base = declarative_base()
favorite_series = db.Table('favorite_series', Base.metadata,
db.Column('user_id', db.Integer, db.ForeignKey('User.id')),
db.Column('series_id', db.Integer, db.ForeignKey('Series.id'))
)
class Episode(Base):
__tablename__ = 'Episode'
id = db.Column(db.Integer, primary_key=True)
season = db.Column(db.Integer)
episode_num = db.Column(db.Integer)
series_id = db.Column(db.Integer, db.ForeignKey('Series.id'))
def __init__(self, season, episode_num, series_id):
self.season = season
self.episode_num = episode_num
self.series_id = series_id
def __repr__(self):
return self.series.title + \
' S' + str(self.season) + \
'E' + str(self.episode_num)
class Series(Base):
__tablename__ = 'Series'
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String)
episodes = orm.relationship('Episode', backref='series')
def __init__(self, title):
self.title = title
def __repr__(self):
return self.title
class User(Base):
__tablename__ = 'User'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String)
favorite_series = orm.relationship('Series',
secondary=favorite_series, backref='users')
def __init__(self, name):
self.name = name
def __repr__(self):
return self.name
Now you can just access the attributes of your objects and let sql alchemy deal with keeping you DB in sync and issuing queries.
engine = db.create_engine('sqlite:///:memory:')
session = orm.sessionmaker(bind=engine)()
Base.metadata.create_all(engine)
lt = User('Ludovic Tiako')
the_wire = Series('The Wire')
friends = Series('Friends')
session.add_all([lt, the_wire, friends])
session.commit() # need to commit here to generate the id fields
tw_s01e01 = Episode(1,1,the_wire.id)
tw_s01e02 = Episode(1,2,the_wire.id)
f_s01e01 = Episode(1,1,friends.id)
f_s01e02 = Episode(1,2,friends.id)
f_s01e03 = Episode(1,3,friends.id)
session.add_all([tw_s01e01, tw_s01e02,
f_s01e01, f_s01e02, f_s01e03])
session.commit()
the_wire.episodes # > [The Wire S1E1, The Wire S1E2]
friends.episodes # > [Friends S1E1, Friends S1E2, Friends S1E3]
Finally, to answer your question:
lt.favorite_series.append(the_wire)
session.commit()
lt.favorite_series # > [The Wire]
[s.episodes for s in lt.favorite_series] # >> [[The Wire S1E1, The Wire S1E2]]
I don't know about Flask, but from the docs of Flask-SQLAlchemy, it seems it uses declarative, so the ORM. And so, you should have a session. I think it is accessible to you from db.session.
Anyway, if those assumptions are true, this is how you should do it:
query = db.session.query(User.id, Series.name, Episode.name).filter((Episode.series_id == Series.id) & \
(User.id == favorite_series.c.user_id) & (Series.id == favorite_series.c.id) & \
(User.id == 1))
results = query.all();
It might not be the exact query you provided, but should do the same.
UPDATE: I just checked Flask-SQLALchemy code on github, it seems that db is an instance of SQLAlchemy, which has a session attribute, created by self.session = self.create_scoped_session(session_options) which returns a session object. So this should work.
Also, not that by doing that, you won't be using their BaseQuery, although I don't know what that would mean...
Check the documentation to know what to do exactly.