We Keep Coding

SQLITE - NOT NULL Constraint Failed - transferring a row from one table to another

I'm building a to-do list API with sqlite and flask. I've got a main.py file, a todo.db, and a helper.py.
I have a function (which is my POST request). I have two database tables called incomplete and complete. My function is supposed to "complete" an action by deleting the corresponding task from incomplete, copying it, and putting it in complete.
I tried a lot of different things, kept getting different errors, from "database is locked" to "error binding parameter 1" and now this:
Error: NOT NULL constraint failed: complete.task
I've gotten the function to at least delete the corresponding task from incomplete, and add something to complete, but it adds it incorrectly. Instead of something like this output when I use the GET request:
[{"complete": [["678", "do dishes" ]]}, {"incomplete": [["756", "Setting up API"]]}]
I get something like this
[{"complete": [["678", "[]"]}, {"incomplete": [["756", "Setting up API"]]}]
I'm not sure what I'm doing wrong. I'm also getting an error when I'm un-completing items (I get "Internal Server Error") :(
Here's the helper file, with parts ommitted. This snippet includes the relevant parts, and the part I'm focusing on is the add_to_completes().
import sqlite3
import random
#for id's because users dont set them
DB_PATH = './todo.db'
# connect to database
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute("""CREATE TABLE IF NOT EXISTS "complete" ("id" TEXT NOT NULL, "task" TEXT NOT NULL, PRIMARY KEY("id"));""")
# save the change
c.execute("""CREATE TABLE IF NOT EXISTS "incomplete" ("id" TEXT NOT NULL, "task" TEXT NOT NULL, PRIMARY KEY("id"));""")
conn.commit()
def add_to_incomplete(task):
id = random.randint(100, 999)
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('insert into incomplete(id, task) values(?,?)', (id, task))
conn.commit()
return {"id": id}
except Exception as e:
print('Error: ', e)
return None
def add_to_complete(inputId):
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('select task from incomplete where id=?', (inputId,))
tasks = c.fetchone()
c.execute('insert into complete values(?,?)', (inputId,tasks))
delete_task(inputId)
conn.commit()
return {"id": id}
except Exception as e:
print('Error: ', e)
return None
def get_all_completes():
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('select * from complete')
rows = c.fetchall()
conn.commit()
return { "complete": rows }
except Exception as e:
print('Error: ', e)
return None
def get_all_incompletes():
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('select * from incomplete')
rows = c.fetchall()
conn.commit()
return { "incomplete": rows }
except Exception as e:
print('Error: ', e)
return None
def uncomplete(inputId):
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('select from complete where id=?', (inputId,))
row = c.fetchall()
c.execute('delete from complete where id=?', (inputId,))
#you have that format (item,) because we need to pass execute() a tuple even if theres only one thing in the tuple
add_to_incomplete(row)
conn.commit()
return {"id":id}
except Exception as e:
print('Error: ', e)
return None
def delete_task(inputId):
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('delete from complete where id=?', (inputId,))
c.execute('delete from incomplete where id=?', (inputId,))
conn.commit()
return {"id":id}
except Exception as e:
print('Error: ', e)
return None
Here are the relevant parts of my main.py file:
import helper
from flask import Flask, request, Response
import json
app = Flask(__name__)
...
#app.route('/tasks/all', methods = ["GET"])
def get_all_items():
res_data = helper.get_all_completes(), helper.get_all_incompletes()
response = Response(json.dumps(res_data), mimetype='application/json')
return response
#app.route('/tasks/complete', methods = ["POST"])
def complete_task():
req_data = request.get_json()
inputId = req_data['id']
res_data = helper.add_to_complete(inputId)
# find matching task to input id
return "completed task" + inputId
#app.route('/tasks/incomplete', methods = ["PATCH"])
def uncomplete_task():
req_data = request.get_json()
inputId = req_data['id']
res_data = helper.uncomplete(inputId)
if res_Data is None:
response = Response("{'error': 'Error uncompleting task - '" + task + ", " + status + "}", status=400 , mimetype='application/json')
response = Response(json.dumps(res_data), mimetype='application/json')
return "uncompleted task" + " " + inputId
...
#app.route('/tasks/remove', methods = ["DELETE"])
def delete():
req_data = request.get_json()
inputId = req_data['id']
res_data = helper.delete_task(inputId)
if res_data is None:
response = Response("{'error': 'Error deleting task - '" + task + "}", status=400 , mimetype='application/json')
return "deleted task id" + " " + inputId

Having trouble autoincrementing in an API

I built a to-do list API with Flask and SQlite, and now I'm trying to use AUTOINCREMENT for incrementing the id's for the tasks. However, I am getting an error ("Error: NOT NULL constraint failed: incomplete.id") when I try to add something to the list. I'm not sure why, I looked at the sqlite documentation, and I seem to be following. I even tried reformatting the create table statements. I'm not sure what else to do, i'd really appreciate some guidance/advice/help. Thanks!
Here is my helper.py
import helper
from flask import Flask, request, jsonify, Response
import json
app = Flask(__name__)
#app.route('/')
def hello_world():
return 'Hello World!'
#app.route('/tasks/new', methods=['PUT'])
def add_task():
# global idCount
# idCount = idCount + 1
# get item from the POST body, request module used to parse request and get HTTP body data. response is used to return response to the client, of type JSON
req_data = request.get_json()
task = req_data['task']
# add task to the list
res_data = helper.add_to_incomplete(task)
# return error if task cant be added
if res_data is None:
response = Response("{'error': 'Task not added - " + task + "'}", mimetype='application/json')
return response;
response = Response(json.dumps(res_data), mimetype='application/json')
return response
#app.route('/tasks/all', methods = ["GET"])
def get_all_items():
res_data = helper.get_all_completes(), helper.get_all_incompletes()
response = Response(json.dumps(res_data), mimetype='application/json')
return response
#app.route('/tasks/complete', methods = ["POST"])
def complete_task():
req_data = request.get_json()
inputId = req_data['id']
res_data = helper.add_to_complete(inputId)
# find matching task to input id
return "completed task" + inputId
#app.route('/tasks/incomplete', methods = ["PATCH"])
def uncomplete_task():
req_data = request.get_json()
inputId = req_data['id']
res_data = helper.uncomplete(inputId)
# find matching task to input id
return "un-completed task" + inputId
#app.route('/tasks/remove', methods = ["DELETE"])
def delete():
req_data = request.get_json()
inputId = req_data['id']
res_data = helper.delete_task(inputId)
if res_data is None:
response = Response("{'error': 'Error deleting task - '" + task + "}", status=400 , mimetype='application/json')
return "deleted task id" + " " + inputId
#app.route('/tasks/empty', methods = ["EMPTY"])
def delete_all():
helper.empty()
return "you deleted everything"
Here is my helper.py:
import sqlite3
import random
#for id's because users dont set them
DB_PATH = './todo.db'
# connect to database
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute("CREATE TABLE IF NOT EXISTS complete (id INTEGER PRIMARY KEY, task TEXT NOT NULL);")
# save the change
c.execute("CREATE TABLE IF NOT EXISTS incomplete (id INTEGER PRIMARY KEY, task TEXT NOT NULL);")
conn.commit()
def add_to_incomplete(task):
try:
# id = str(random.randrange(100,999))
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('insert into incomplete(task) values(?)', (task,))
conn.commit()
return {"id": id}
except Exception as e:
print('Error: ', e)
return None
def add_to_complete(inputId):
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('select task from incomplete where id=?', (inputId,))
tasks = c.fetchone()[0]
c.execute('insert into complete values(?,?)', (inputId,tasks))
delete_task(inputId)
conn.commit()
return {"id": id}
except Exception as e:
print('Error: ', e)
return None
def get_all_completes():
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('select * from complete')
rows = c.fetchall()
conn.commit()
return { "complete": rows }
except Exception as e:
print('Error: ', e)
return None
def get_all_incompletes():
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('select * from incomplete')
rows = c.fetchall()
conn.commit()
return { "incomplete": rows }
except Exception as e:
print('Error: ', e)
return None
def uncomplete(inputId):
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('select task from complete where id=?', (inputId,))
tasks = c.fetchone()[0]
c.execute('insert into incomplete values(?,?)', (inputId,tasks))
delete_task(inputId)
conn.commit()
return {"id": id}
except Exception as e:
print('Error: ', e)
return None
def delete_task(inputId):
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('delete from complete where id=?', (inputId,))
c.execute('delete from incomplete where id=?', (inputId,))
conn.commit()
return {"id":id}
except Exception as e:
print('Error: ', e)
return None
def empty():
try:
conn = sqlite3.connect(DB_PATH)
c = conn.cursor()
c.execute('delete from complete')
c.execute('delete from incomplete')
conn.commit()
return "you deleted everything mwahaha"
except Exception as e:
print('Error: ', e)
return None

I would suggest changing your sql table creation code to:
create table if not exists complete
(
id int auto_increment,
constraint complete_pk
primary key (id)
);
However a better option is to use SQLAlchemy

how to mock python postgres database

I try to mock my db but when I test it the result is None.
try:
con = psycopg2.connect(
host="yhvh",
database="python_db",
user="postgres",
password="pass",
)
except:
print("Unable to connect database")
# Open a cursor to perform database operation
cur = con.cursor()
def read(con):
"""
Read data in Database
"""
print("Read")
# execute the query
data ="SELECT id, name FROM employees"
cur.execute(
data
)
# fetchall - returns all entries
rows = cur.fetchall()
for r in rows:
print(f"id {r[0]} name {r[1]}")
this is the code for my testing
def test_read(self):
expected = [9, 'aaa']
with patch('psycopg2.connect') as mock_connect:
mock_con_cm = mock_connect.return_value
mock_con = mock_con_cm.__enter__.return_value
mock_cur = mock_con.cursor.return_value
mock_cur.fetchall.return_value = expected
result = db.read(mock_connect)
self.assertEqual(expected, result)
I get an assertionError: [9, 'aaa'] != None
How the result to have a value that would result is equal to expected ?

First you need to return the rows which contains the list of data from read function if not it will return None.
Then use assertListEqual(expected, result) to check the elements in the list.
Your final code will look like this.
def read(con):
"""
Read data in Database
"""
print("Read")
# execute the query
data ="SELECT id, name FROM employees"
cur.execute(
data
)
# fetchall - returns all entries
rows = cur.fetchall()
for r in rows:
print(f"id {r[0]} name {r[1]}")
return rows
And assertion should be,
self.assertListEqual(expected, result)

How to fix {"message": "The method is not allowed for the requested URL." } for my post method?

This is the first time I'm creating an API for android retrofit. I modified this code according to the snippet I got online. The main functionality of the post method is to take the given parameters and store it in the sqlite3 database.
My schema of the following two tables:
sqlite> .schema spending
CREATE TABLE spending(
ID INTEGER PRIMARY KEY AUTOINCREMENT,
date TEXT ,
reason TEXT ,
amount INTEGER
);
CREATE TABLE receiving(
ID INTEGER PRIMARY KEY AUTOINCREMENT,
date TEXT ,
from_reason TEXT ,
amount INTEGER
);
from flask import Flask, request
from flask_restful import Resource, Api
from sqlalchemy import create_engine
from flask import jsonify
db_connect = create_engine('sqlite:///api.db')
app = Flask(__name__)
api = Api(app)
class AddSpending(Resource):
def add_spending(self):
try:
_json = request.json
_date = _json['date']
_reason = _json['reason']
_amount = _json['amount']
# validate the received values
if _date and _reason and _amount and request.method == 'POST':
#do not save password as a plain text
#_hashed_password = generate_password_hash(_password)
# save edits
sql = "INSERT INTO spending(date, reason, amount) VALUES(%s, %s, %d)"
data = (_date, _reason, _amount)
#conn = mysql.connect()
conn = db_connect.connect()
cursor = db_connect.cursor()
conn.cursor()
conn.execute(sql, data)
conn.commit()
#resp = jsonify('Spending added successfully!')
#resp.status_code = 200
return
else:
return 404
except Exception as e:
print(e)
finally:
cursor.close()
conn.close()
api.add_resource(AddSpending, '/spending_up',methods=['POST']) # Route_3
When a user passes data through this parameter. The data should be stored in the database

I think the problem is that you called you method as add_spending and shoud be named as post
change def add_spending(self) by def post(self)
the code for your api should look like that, without the methods='POST'
class AddSpending(Resource):
def post(self):
try:
_json = request.json
_date = _json['date']
_reason = _json['reason']
_amount = _json['amount']
# validate the received values
if _date and _reason and _amount and request.method == 'POST':
#do not save password as a plain text
#_hashed_password = generate_password_hash(_password)
# save edits
sql = "INSERT INTO spending(date, reason, amount) VALUES(%s, %s, %d)"
data = (_date, _reason, _amount)
#conn = mysql.connect()
conn = db_connect.connect()
cursor = db_connect.cursor()
conn.cursor()
conn.execute(sql, data)
conn.commit()
#resp = jsonify('Spending added successfully!')
#resp.status_code = 200
return
else:
return 404
except Exception as e:
print(e)
finally:
cursor.close()
conn.close()
api.add_resource(AddSpending, '/spending_up') # Route_3
UPDATE
I just tried with a code similar to yours and worked
ANOTHER UPDATE
your repo code

Trying to understand cx_Oracle`s LOB object

I have a database class in python which I use to query the database.
class Database():
def __init__(self, user, password, host, port, service_name, mode, *args):
#mode should be 0 if not cx_Oracle.SYSDBA
self.user = user
self.password = password
self.host = host
self.port = port
self.user = user
self.service_name = service_name
self.logger = logging.getLogger(__name__)
self.mode = 0
self.connection = None
self.connect_string = self.user + '/' + self.password + '#' + dsn
try:
self.connection = cx_Oracle.connect(self.connect_string, mode=self.mode, threaded=True)
self.connection.stmtcachesize = 1000
self.connection.client_identifier = 'my_app_scheduler'
self.cursor = self.connection.cursor()
self.cursor.arraysize = 10000
self.idVar = self.cursor.var(cx_Oracle.NUMBER)
except cx_Oracle.DatabaseError, exc:
error, = exc
self.logger.exception('Exception occured while trying to create database object : %s', error.message)
raise exc
def query(self, q):
try:
self.cursor.execute(q)
return self.cursor.fetchall(), self.cursor.rowcount
except cx_Oracle.DatabaseError, exc:
raise exc
And this is the code to manipulate the fetched data and convert it.
output, rowcount = db_run_query.query(sql_text)
#self.logger.debug('output : %s, type : %s', output, type(output))
end_time=time.time()
time_taken=end_time - start_time
self.logger.debug('Rowcount : %s, time_taken : %s', rowcount, time_taken)
column_name = [d[0] for d in db_run_query.cursor.description]
result = [dict(zip(column_name, row)) for row in output]
#Convert everything to string : Eg: datetime
try:
for each_dict in result:
for key in each_dict:
if isinstance(each_dict[key], cx_Oracle.LOB):
self.logger.debug('%s', each_dict[key].size())
each_dict[key]=each_dict[key].read()
#self.logger.debug('%s %s %s %s %s %s %s', key, each_dict, type(key), type(each_dict[key]), type(each_dict), temp_each_dict, type(temp_each_dict))
else:
each_dict[key]=str(each_dict[key])
except Exception as e:
self.logger.debug(e)
So without self.cursor.arraysize = 10000
and for a query like select clob_value from table it was able to fetch the data and logged Rowcount : 4901, time_taken : 0.196296930313 but was giving me an error like
LOB variable no longer valid after subsequent fetch
but when I mention the arraysize parameter the error goes away. ( Is arraysize only for lob columns coz it works fine for select other_column from table where rownum<20000 <- other_column in varchar)
Why does that happen?

Turns out CLOBs and fetchall don't place nice together:
Internally, Oracle uses LOB locators which are allocated based on the
cursor array size. Thus, it is important that the data in the LOB
object be manipulated before another internal fetch takes place. The
safest way to do this is to use the cursor as an iterator. In
particular, do not use the fetchall() method.
Avoiding cursor.fetchall() and using it like an iterator (e.g. for row in cursor: ...) and i was able to get around this problem.