We Keep Coding

Sql alchemy using connection string with kerberos and impala

Actually I use sql alchemy and impala to connect database with creator like :
def conn():
return connect(host=CONNECTION_PARAMETERS["HOST"],
port=CONNECTION_PARAMETERS["PORT"],
database=CONNECTION_PARAMETERS["DATABASE"],
use_ssl=CONNECTION_PARAMETERS["USE_SSL"],
auth_mechanism=CONNECTION_PARAMETERS["AUTH_MECANISM"])
def get_bde_db():
engine = create_engine('impala://', creator=conn,
pool_size=2,
pool_pre_ping=True,
pool_recycle=3600)
user_session_local = sessionmaker(autocommit=False, autoflush=False, bind=engine)
db = user_session_local()
try:
yield db
finally:
db.close()
But I want to change it, stop use the creator parameter and build a connection string in the create_engine. Is it possible to generate a url, and use url parameter ?
I use impala, kerberos, ssl

Insert data to a stored procedure sqlalchemy + MSSQL server in python

I am building some RestAPI's using python FastAPI and getting some data from MSSQL server using SQLAlchemy.
I am trying to insert data into existing stored procedure in MSSQL server.
The stored procedure AcctsCostCentersAddV001 will take ArDescription, EngDescription, ErrMsg as parameters and return Output as an output.
MSSQL Server Code:
CREATE Proc [dbo].[AcctsCostCentersAddV001] #ArDescription nvarchar(100), #EngDescription varchar(100), #ErrMsg nvarchar(100) Output
As
Insert Into AcctsCostCenters (ArDescription, EngDescription) Values(#ArDescription, #EngDescription)
Set #ErrMsg = 'Test'
GO
My Python code:
from fastapi import APIRouter, Request, Depends, HTTPException
from fastapi.responses import JSONResponse
# # # # # # # # # # SQL # # # # # # # # # #
from sqlalchemy import text
from sqlalchemy.exc import ProgrammingError
# # # # # # # # # # Files # # # # # # # # # #
from dependencies import get_db
from sqlalchemy.engine import create_engine
from internal.config import username, password, SQL_SERVER, database_name
#router.get("/create/CostCenter/")
async def forms(request: Request, db=Depends(get_db)):
try:
connection_string = f'mssql://{username}:{password}#{SQL_SERVER}/{database_name}?
driver=ODBC+Driver+17+for+SQL+Server'
engine = create_engine(connection_string, pool_pre_ping=True)
connection = engine.raw_connection()
try:
cursor_obj = connection.cursor()
query = "Declare #ErrMsg nvarchar(100) Exec AcctsCostCentersAddV001 'Test', 'Test', #ErrMsg Output Print #ErrMsg"
cursor_obj.execute(text(query))
results = list(cursor_obj.fetchall())
cursor_obj.close()
connection.commit()
print(results)
finally:
connection.close()
except IndexError:
raise HTTPException(status_code=404, detail="Not found")
except ProgrammingError as e:
print(e)
raise HTTPException(status_code=400, detail="Invalid Entry")
except Exception as e:
print(e)
return UnknownError(
"unknown error caused by CostCenter API request handler", error=e)
return JSONResponse(results)
For some reason, this code doesn't raise any exceptions but yet i keep getting
('The SQL contains 0 parameter markers, but 3 parameters were supplied', 'HY000')
I have even tried this
cursor_obj.execute("Declare #ErrMsg nvarchar(100) Exec AcctsCostCentersAddV001 'Test', 'Test', #ErrMsg")
but i get back
No results. Previous SQL was not a query.
i tried wrapping the query in text() but i got The first argument to execute must be a string or unicode query.
but when i go into MSSQL Server and run
Declare #ErrMsg nvarchar(100)
Exec AcctsCostCentersAddV001 'Test', 'Test', #ErrMsg Output
Print #ErrMsg
It runs without any problem.
My Env:
Ubuntu 21.04 VPS by OVH
I hope i am providing everything you guys need and let me know if missed anything, Thanks!
btw I know i am connecting to the db twice :3
(Edited):
I am actually connecting to the DB from a datapase.py file, and i am connecting again in the function just for testing.
My database.py:
import time
from colorama import Fore
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.engine import create_engine
from internal.config import username, password, SQL_SERVER, database_name
try:
connection_string = f'mssql://{username}:{password}#{SQL_SERVER}/{database_name}?driver=ODBC+Driver+17+for+SQL+Server'
engine = create_engine(connection_string, pool_pre_ping=True)
connection = engine.raw_connection()
print(f"{Fore.CYAN}DB{Fore.RESET}: Database Connected. ")
SessionLocal = sessionmaker(autocommit=True, autoflush=False, bind=engine)
Base = declarative_base()
except Exception as e:
print(f"{Fore.RED}DB{Fore.RESET}: Couldn't connect to Database.", e)

How to use PostgreSQL test database in async FastAPI tests?

I'm working on an async FastAPI project and I want to connect to the database during tests. Coming from Django, my instinct was to create pytest fixtures that take care of creating/dropping the test database. However, I couldn't find much documentation on how to do this. The most complete instructions I could find were in this tutorial, but they don't work for me because they are all synchronous. I'm somewhat new to async development so I'm having trouble adapting the code to work async. This is what I have so far:
import pytest
from sqlalchemy.ext.asyncio import create_async_engine, session
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy_utils import database_exists, create_database
from fastapi.testclient import TestClient
from app.core.db import get_session
from app.main import app
Base = declarative_base()
#pytest.fixture(scope="session")
def db_engine():
default_db = (
"postgresql+asyncpg://postgres:postgres#postgres:5432/postgres"
)
test_db = "postgresql+asyncpg://postgres:postgres#postgres:5432/test"
engine = create_async_engine(default_db)
if not database_exists(test_db): # <- Getting error on this line
create_database(test_db)
Base.metadata.create_all(bind=engine)
yield engine
#pytest.fixture(scope="function")
def db(db_engine):
connection = db_engine.connect()
# begin a non-ORM transaction
connection.begin()
# bind an individual Session to the connection
Session = sessionmaker(bind=connection)
db = Session()
# db = Session(db_engine)
yield db
db.rollback()
connection.close()
#pytest.fixture(scope="function")
def client(db):
app.dependency_overrides[get_session] = lambda: db
PREFIX = "/api/v1/my-endpoint"
with TestClient(PREFIX, app) as c:
yield c
And this is the error I'm getting:
E sqlalchemy.exc.MissingGreenlet: greenlet_spawn has not been called; can't call await_() here. Was IO attempted in an unexpected place? (Background on this error at: https://sqlalche.me/e/14/xd2s)
/usr/local/lib/python3.9/site-packages/sqlalchemy/util/_concurrency_py3k.py:67: MissingGreenlet
Any idea what I have to do to fix it?

You try to use sync engine with async session. Try to use:
from sqlalchemy.ext.asyncio import AsyncSession
Session = sessionmaker(bind= connection, class_=AsyncSession)
https://docs.sqlalchemy.org/en/14/orm/extensions/asyncio.html

How to fix '(sqlite3.ProgrammingError) SQLite objects created in a thread can only be used in that same thread.' Issue?

I'm utilizing Flask and SqlAlchemy. The database I've created for SqlAlchemy seems to mess up when I try to run my website and will pop up with the error stating that there's a thread error. I'm wondering if it's because I haven't dropped my table from my previous schema. I'm using a linux server to try and run the "python3" and the file to set up my database.
I've tried to physically delete the table from my local drive and the re run it but I still up this error.
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.orm import scoped_session
from database_setup import Base, Category, Item
engine = create_engine('sqlite:///database_tables.db')
Base.metadata.bind = engine
Session = sessionmaker()
Session.bind = engine
session = Session()
brushes = Category(id = 1, category_name = 'Brushes')
session.add(brushes)
session.commit()
pencils = Category(id = 2, category_name = 'Pencils')
session.add(pencils)
session.commit()
When I am in debug mode using Flask, I click the links I've made using these rows, but after three clicks I get the error
"(sqlite3.ProgrammingError) SQLite objects created in a thread can only be used in that same thread.The object was created in thread id 140244909291264 and this is thread id 140244900898560 [SQL: SELECT category.id AS category_id, category.category_name AS category_category_name FROM category] [parameters: [{}]] (Background on this error at: http://sqlalche.me/e/f405)"

you can use for each thread a session, by indexing them using the thread id _thread.get_ident():
import _thread
engine = create_engine('sqlite:///history.db', connect_args={'check_same_thread': False})
...
Base.metadata.create_all(engine)
sessions = {}
def get_session():
thread_id = _thread.get_ident() # get thread id
if thread_id in sessions:
return sessions[thread_id]
session_factory = sessionmaker(bind=engine)
Session = scoped_session(session_factory)
sessions[thread_id] = Session()
return sessions[thread_id]
then use get_session() where it is needed, in your case:
get_session().add(brushes)
get_session().commit()

MySQL Connection not available when use SQLAlchemy(MySQL) and Flask

I'm getting this error sometime (sometime is ok, sometime is wrong):
sqlalchemy.exc.OperationalError: (OperationalError) MySQL Connection not available.
while using session.query
I'm writing a simple server with Flask and SQLAlchemy (MySQL). My app.py like this:
Session = sessionmaker(bind=engine)
session = Session()
#app.route('/foo')
def foo():
try:
session.query(Foo).all()
except Exception:
session.rollback()
Update
I also create new session in another file and call it in app.py
Session = sessionmaker(bind=engine)
session = Session()
def foo_helper(): #call in app.py
session.query(Something).all()
Update 2
My engine:
engine = create_engine('path')
How can I avoid that error?
Thank you!

Make sure the value of ‘pool_recycle option’ is less than your MYSQLs wait_timeout value when using SQLAlchemy ‘create_engine’ function.
engine = create_engine("mysql://username:password#localhost/myDatabase", pool_recycle=3600)
Try to use scoped_session to make your session:
from sqlalchemy.orm import scoped_session, sessionmaker
session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=engine))
and close/remove your session after retrieving your data.
session.query(Foo).all()
session.close()