I am trying to build a database with sqlalchemy.
I have two tables : flow and krbr.
from __future__ import print_function
import numpy as np
import pandas as pd
import sqlalchemy
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String
import sqlalchemy_utils
from sqlalchemy_utils.types.ip_address import IPAddressType
Base = declarative_base()
## Define the tables schema
class Flow(Base):
__tablename__ = 'flow'
Id = Column(Integer, primary_key=True)
First = Column(Integer, index=True)
Protocol = Column(String(10))
Src = Column(IPAddressType, index=True)
SrcPort = Column(Integer)
Dst = Column(IPAddressType, index=True)
DstPort = Column(Integer)
GroupId = Column(Integer)
Port = Column(String(10))
VPort = Column(Integer)
IpTos = Column(String)
VlanId = Column(String)
VlanPri = Column(String)
Application = Column(String(100))
Packets = Column(Integer)
Messages = Column(Integer)
Bytes = Column(Integer)
Last = Column(Integer)
#LearnedIPs alertable
#LearnedIPs learned-ip
# u'LearnedIPs new-ips', u'LearnedIPs subnet-name',
# u'LearnedIPs timestamp-sec', u'LearnedIPs total-ips', u'SrcSubnet',
# u'DstSubnet'],
# 'MPLS Exp'
class Krbr(Base):
__tablename__ = 'krbr'
Id = Column(Integer, primary_key=True)
Src = Column(IPAddressType, index=True)
SrcPort = Column(Integer)
Dst = Column(IPAddressType, index=True)
DstPort = Column(Integer)
TimeNs = Column(Integer)
To some of the rows in flow is associated one or more rows of krbr.
A row in krbr is associated with a row in flow if:
1) they have the same values of Src, Dst, SrcPort, DstPort
2) They are close in time. i.e. np.abs(Flow.first - Krbr.TimeNs/1000000000) < threshold
I am wondering what is the right approach to create a link between the two tables. i.e. given a row of one table I want to be able to get the rows of the other table.
I do not know much of sqlalchemy. I guess that I should define a foreign key but I do not know how to enforce such a complex relationship.
Here is a sample of 'Handling Multiple Join Paths':
from sqlalchemy import Integer, ForeignKey, String, Column
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
Base = declarative_base()
class Customer(Base):
__tablename__ = 'customer'
id = Column(Integer, primary_key=True)
name = Column(String)
billing_address_id = Column(Integer, ForeignKey("address.id"))
shipping_address_id = Column(Integer, ForeignKey("address.id"))
billing_address = relationship("Address")
shipping_address = relationship("Address")
class Address(Base):
__tablename__ = 'address'
id = Column(Integer, primary_key=True)
street = Column(String)
city = Column(String)
state = Column(String)
zip = Column(String)
More info about 'Handling Multiple Join Paths' in you can find at
SQLAlchemy 0.9 Documentation
Related
I have three tables
import enum
from sqlalchemy import Column, Enum, Float, String
from sqlalchemy.orm import relationship
#enum.unique
class TypeEnum(str, enum.Enum):
person = "person"
location = "location"
class Person(Base):
name = Column(String)
id = Column(Integer, primary_key=True)
class Location(Base):
id = Column(Integer, primary_key=True)
city = Column(String)
lat = Column(Float)
lon = Column(Float)
class ExternalIdentifier(Base):
name = Column(String)
id = Column(Integer, primary_key=True)
value = Column(String)
type = Column(Enum(TypeEnum))
# internal_id = relationship(...) # Problem here
I would like to have a mapping between ExternalIdentifier and Person/Location:
If ExternalIdentifier.type is person, then ExternalIdentifier.internal_id should refer to an object from table Person
Analogously if the type is location
Unfortunately, I have only been able to make it work by either:
creating fields ExternalIdentifier.person_id/location_id instead of ExternalIdentifier.internal_id/type
or creating tables person/location_to_external_identifier
Suppose I have the following three classes describing a music collection:
from sqlalchemy import Column, Integer, String, ForeignKey, UniqueConstraint
from sqlalchemy import Table
from sqlalchemy.orm import relationship
from sqlalchemy.ext.declarative import declarative_base
TRACK_NAME_LEN = 64
ALBUM_NAME_LEN = 64
TAG_NAME_LEN = 64
URL_LEN = 255
Base = declarative_base()
class Track(Base):
__tablename__ = 'tracks'
id = Column(Integer, primary_key=True)
name = Column(String(TRACK_NAME_LEN), nullable=False)
# many -> one
album_id = Column(Integer, ForeignKey('albums.id'), nullable=True)
album = relationship('Album', back_populates='tracks')
# Auxiliary table for many <--> many relationship
album_tag = Table('album_tag', Base.metadata,
Column('album_id', Integer, ForeignKey('albums.id')),
Column('tag_id', Integer, ForeignKey('tags.id')))
class Album(Base):
__tablename__ = 'albums'
id = Column(Integer, primary_key=True)
name = Column(String(ALBUM_NAME_LEN), nullable=False)
# one -> many
tracks = relationship('Track', back_populates='album')
# many -> many
tags = relationship(
'Tag',
secondary=album_tag,
back_populates='albums')
class Tag(Base):
__tablename__ = 'tags'
id = Column(Integer, primary_key=True)
name = Column(String(TAG_NAME_LEN), nullable=False, unique=True)
# many -> many
albums = relationship(
'Album',
secondary=album_tag,
back_populates='tags')
An album may have many different tags.
I'd like a query returning all tracks whose album has a given tag.
Here's something that doesn't work:
rock = session.query(Tag).filter(name='rock').one()
session.query(Track).join(Album).filter(Album.tags.any(rock))
There are any number of other failed attempts.
How do we achieve this?
This works:
session.query(Track).filter(Track.album.has(Album.tags.any(name='tango'))).all()
I'm trying to use SQLAlchemy on my python app but I have a problem with the many to many relationship.
I have 4 tables:
users, flags, commandes, channels, and commandes_channels_flags
commandes_channels_flags contain a foreign key for each concerned table (commandes, channels and flags)
An user has a flag_id as foreign key too.
So I try to link commandes, channels and flag. the objective is to know that a command can run on a channel for a flag.
I did this:
from sqlalchemy import Column, Integer, String, ForeignKey
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
Base = declarative_base()
class User(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
pseudo = Column(String(50), unique=True, nullable=False)
flag_id = Column(ForeignKey('flags.id'))
class Flag(Base):
__tablename__ = 'flags'
id = Column(Integer, primary_key=True)
irc_flag = Column(Integer)
nom = Column(String(50))
users = relationship("User", backref="flag", order_by="Flag.irc_flag")
commande = relationship("Commande", secondary="commandes_channels_flags", back_populates="flags")
channel = relationship("Channel", secondary="commandes_channels_flags", back_populates="flags")
class Channel(Base):
__tablename__ = 'channels'
id = Column(Integer, primary_key=True)
uri = Column(String(50))
topic = Column(String(255))
commande = relationship("Commande", secondary="commandes_channels_flags", back_populates="channels")
flag = relationship("Flag", secondary="commandes_channels_flags", back_populates="channels")
class Commande(Base):
__tablename__ = 'commandes'
id = Column(Integer, primary_key=True)
pattern = Column(String(50))
channel = relationship("Channel", secondary="commandes_channels_flags", back_populates="commandes")
flag = relationship("Flag", secondary="commandes_channels_flags", back_populates="commandes")
class CommandeChannelFlag(Base):
__tablename__ = 'commandes_channels_flags'
id = Column(Integer, primary_key=True)
commande_id = Column(ForeignKey('commandes.id'))
channel_id = Column(ForeignKey('channels.id'))
flag_id = Column(ForeignKey('flags.id'))
But I have this error:
sqlalchemy.exc.InvalidRequestError: Mapper 'Mapper|Commande|commandes' has no property 'channels'
I understand that I have an error in my tables linking but I can't find it.
back_populates needs to match the exact name of the related property on the other model. In Channel, you have back_populates="channels", but in Commande, you have:
channel = relationship("Channel", secondary="commandes_channels_flags", back_populates="commandes")
Instead, change channel = relationship to channels = relationship.
You'll also need to change the other relationship properties to Flag.commandes, Flag.channels, Channel.commandes, Channel.flags, and Commande.flags to match your back_populates arguments.
I have problem with separating tables with relationships in different files. I want the tables below to be in three separate files and to import TableA in third party page, but I can not manage the load order.
In most of the time I'm receiving the following error.
sqlalchemy.exc. InvalidRequestError: When initializing mapper Mapper|TableA|tablea, expression 'TableB' failed to locate a name ("name 'TableB' is not defined"). If this is a class
name, consider adding this relationship() to the class after both dependent classes have been defined.
class TableA(Base):
__tablename__ = "tablea"
id = Column(Integer, primary_key=True)
name = Column(String)
tableB = relationship("TableB", secondary = TableC.__table__)
class TableB(Base):
__tablename__ = "tableb"
id = Column(Integer, primary_key=True)
name = Column(String)
class TableC(Base):
__tablename__ = "tableab"
tableAId = Column("table_a_id", Integer, ForeignKey("TableA.id"), primary_key=True)
tableBId = Column("table_b_id", Integer, ForeignKey("TableB.id"), primary_key=True)
This should work (note that the TableC.table is replaced with the name of the table to avoid circular module loading):
### base.py
engine = create_engine('sqlite:///:memory:', echo=True)
Session = sessionmaker(bind=engine)
Base = declarative_base(bind=engine)
### classA.py
from base import Base
from classB import TableB
class TableA(Base):
__tablename__ = 'tablea'
id = Column(Integer, primary_key=True)
name = Column(String(50))
tableBs = relationship("TableB", secondary="tableab")
#tableBs = relationship("TableB", secondary=TableC.__table__)
### classB.py
from base import Base
class TableB(Base):
__tablename__ = 'tableb'
id = Column(Integer, primary_key=True)
name = Column(String(50))
### classC.py
from base import Base
from classA import TableA
from classB import TableB
class TableC(Base):
__tablename__ = 'tableac'
tableAId = Column(Integer, ForeignKey("tablea.id"), primary_key=True, )
tableBId = Column(Integer, ForeignKey("tableb.id"), primary_key=True, )
### main.py
from base import Base, Session, engine
from classA import TableA
from classB import TableB
from classC import TableC
Base.metadata.create_all(engine)
Also I believe that the ForeignKey parameter is case sensitive, so you code might not work because "TableA.id" doe snot match "tablea" name when case-sensitive.
from sqlalchemy import Column, String, Integer
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class Parent(Base):
__tablename__ = 'Parent'
ParentID = Column(Integer, primary_key=True)
Description = Column(String)
def __init__(self, ParentID, Description):
self.ParentID = ParentID
self.Description = Description
----------------------------------------------------------------------
from sqlalchemy import Column, String, Integer, ForeignKey
import Parent
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class Child(Base):
__tablename__ = "Child"
ChildID = Column(Integer, primary_key=True)
Description = Column(String)
ParentID = Column('CompanyID', Integer, ForeignKey(Parent.ParentID))
def __init__(self, ChildID, Description,ParentID):
self.ChildID = ChildID
self.Description = Description
self.ParentID=ParentID
I have read the SQLAlchemy documentation and tutorial about building many-to-many relation but I could not figure out how to do it properly when the association table contains more than the 2 foreign keys.
I have a table of items and every item has many details. Details can be the same on many items, so there is a many-to-many relation between items and details
I have the following:
class Item(Base):
__tablename__ = 'Item'
id = Column(Integer, primary_key=True)
name = Column(String(255))
description = Column(Text)
class Detail(Base):
__tablename__ = 'Detail'
id = Column(Integer, primary_key=True)
name = Column(String)
value = Column(String)
My association table is (It's defined before the other 2 in the code):
class ItemDetail(Base):
__tablename__ = 'ItemDetail'
id = Column(Integer, primary_key=True)
itemId = Column(Integer, ForeignKey('Item.id'))
detailId = Column(Integer, ForeignKey('Detail.id'))
endDate = Column(Date)
In the documentation, it's said that I need to use the "association object". I could not figure out how to use it properly, since it's mixed declarative with mapper forms and the examples seem not to be complete. I added the line:
details = relation(ItemDetail)
as a member of Item class and the line:
itemDetail = relation('Detail')
as a member of the association table, as described in the documentation.
when I do item = session.query(Item).first(), the item.details is not a list of Detail objects, but a list of ItemDetail objects.
How can I get details properly in Item objects, i.e., item.details should be a list of Detail objects?
From the comments I see you've found the answer. But the SQLAlchemy documentation is quite overwhelming for a 'new user' and I was struggling with the same question. So for future reference:
ItemDetail = Table('ItemDetail',
Column('id', Integer, primary_key=True),
Column('itemId', Integer, ForeignKey('Item.id')),
Column('detailId', Integer, ForeignKey('Detail.id')),
Column('endDate', Date))
class Item(Base):
__tablename__ = 'Item'
id = Column(Integer, primary_key=True)
name = Column(String(255))
description = Column(Text)
details = relationship('Detail', secondary=ItemDetail, backref='Item')
class Detail(Base):
__tablename__ = 'Detail'
id = Column(Integer, primary_key=True)
name = Column(String)
value = Column(String)
items = relationship('Item', secondary=ItemDetail, backref='Detail')
Like Miguel, I'm also using a Declarative approach for my junction table. However, I kept running into errors like
sqlalchemy.exc.ArgumentError: secondary argument <class 'main.ProjectUser'> passed to to relationship() User.projects must be a Table object or other FROM clause; can't send a mapped class directly as rows in 'secondary' are persisted independently of a class that is mapped to that same table.
With some fiddling, I was able to come up with the following. (Note my classes are different than OP's but the concept is the same.)
Example
Here's a full working example
from sqlalchemy import create_engine, Column, Integer, String, ForeignKey
from sqlalchemy.orm import declarative_base, relationship, Session
# Make the engine
engine = create_engine("sqlite+pysqlite:///:memory:", future=True, echo=False)
# Make the DeclarativeMeta
Base = declarative_base()
class User(Base):
__tablename__ = "users"
id = Column(Integer, primary_key=True)
name = Column(String)
projects = relationship('Project', secondary='project_users', back_populates='users')
class Project(Base):
__tablename__ = "projects"
id = Column(Integer, primary_key=True)
name = Column(String)
users = relationship('User', secondary='project_users', back_populates='projects')
class ProjectUser(Base):
__tablename__ = "project_users"
id = Column(Integer, primary_key=True)
notes = Column(String, nullable=True)
user_id = Column(Integer, ForeignKey('users.id'))
project_id = Column(Integer, ForeignKey('projects.id'))
# Create the tables in the database
Base.metadata.create_all(engine)
# Test it
with Session(bind=engine) as session:
# add users
usr1 = User(name="bob")
session.add(usr1)
usr2 = User(name="alice")
session.add(usr2)
session.commit()
# add projects
prj1 = Project(name="Project 1")
session.add(prj1)
prj2 = Project(name="Project 2")
session.add(prj2)
session.commit()
# map users to projects
prj1.users = [usr1, usr2]
prj2.users = [usr2]
session.commit()
with Session(bind=engine) as session:
print(session.query(User).where(User.id == 1).one().projects)
print(session.query(Project).where(Project.id == 1).one().users)
Notes
reference the table name in the secondary argument like secondary='project_users' as opposed to secondary=ProjectUser
use back_populates instead of backref
I made a detailed writeup about this here.
Previous Answer worked for me, but I used a Class base approach for the table ItemDetail. This is the Sample code:
class ItemDetail(Base):
__tablename__ = 'ItemDetail'
id = Column(Integer, primary_key=True, index=True)
itemId = Column(Integer, ForeignKey('Item.id'))
detailId = Column(Integer, ForeignKey('Detail.id'))
endDate = Column(Date)
class Item(Base):
__tablename__ = 'Item'
id = Column(Integer, primary_key=True)
name = Column(String(255))
description = Column(Text)
details = relationship('Detail', secondary=ItemDetail.__table__, backref='Item')
class Detail(Base):
__tablename__ = 'Detail'
id = Column(Integer, primary_key=True)
name = Column(String)
value = Column(String)
items = relationship('Item', secondary=ItemDetail.__table__, backref='Detail')