My problem is that when I run project or debug, the first query run so fast in about < 1s, but when it comes to run second query, it costs more than 30s. I'm so confused about it. I have already ran it in DB editor, both of them run so fast, doesn't have any problem. First look, two queries is quite similar so I do not know why caused it.
By the way sometimes I debug, it pop up a red notice in the left debug and run tab, but I cannot get screen shot of this. It just appear once or twice .
This is screen shot of sql query
query 1: rows = db.select("SELECT recruiter_id FROM linkedin.candidates WHERE recruiter_id in (" + ",".join(recruiter_ids) + ")")
query 2: rows = db.select("select c.recruiter_id, c.updated from linkedin.candidates c where c.recruiter_id in (" + ",".join(duplicates_rid) + ")")
This is my code
if recruiter_ids:
print("Creating connection to MySQL in recruiter 12")
rows = db.select("SELECT recruiter_id FROM linkedin.candidates WHERE recruiter_id in (" + ",".join(recruiter_ids) + ")")
db_recruiter_ids = [r['recruiter_id'] for r in rows] + [get_recruiter_id(url) for url in duplicates]
print("Recruiter ids in database:", len(db_recruiter_ids), db_recruiter_ids[:5])
duplicates = [url for url in profile_urls if any(get_recruiter_id(url) == rid for rid in db_recruiter_ids)]
duplicates_rid = [ get_recruiter_id(url) for url in duplicates]
if duplicates_rid:
rows = db.select("select c.recruiter_id, c.updated from linkedin.candidates c where c.recruiter_id in (" + ", ".join(duplicates_rid) + ")")
#rows = db.select("select c.recruiter_id, c.updated from linkedin.candidates c where c.recruiter_id in {}".format(tuple(duplicates_rid)))
rows = [r['recruiter_id'] for r in rows if r['updated'] < datetime.datetime.now() - datetime.timedelta(days=90)]
old_resumes = [url for url in profile_urls if any(get_recruiter_id(url) == r for r in rows)]
profile_urls = [url for url in profile_urls if not any(get_recruiter_id(url) == rid for rid in db_recruiter_ids)]
print("Found duplicates in list:", len(duplicates), duplicates[:3])
if db_recruiter_ids:
tag_candidate_by_recruiter_id(db, db_recruiter_ids, project_id, tracker_id)
Thank you guy so much !!
Related
This function always gives me the following errors when run:
CRITICAL >> Not all parameters were used in the SQL statement
CRITICAL >> Exception for errors programming errors
I've spent hours looking at the code but cannot find the problem - What is wrong?
#************ TestCase Table Insertion *********************
def insertUpdateTestCase(prev_max_weeknumber):
log.Log('insertUpdateTestCase START', 'info')
insertUpdateTestCase_start_time = datetime.now()
testcases = """INSERT INTO prtm_testcase (testplan_identifier, testcase_name, testcase_identifier, testcase_longidentifier, testcase_uri, globalconfiguration_identifier, weeknumber, localconfiguration_identifier)
VALUES
(%s,%s,%s,%s,%s,%s,%s,%s)
ON CONFLICT (testplan_identifier, testcase_identifier, globalconfiguration_identifier, localconfiguration_identifier, weeknumber)
DO
UPDATE SET testcase_name = EXCLUDED.testcase_name,
testcase_longidentifier=EXCLUDED.testcase_longidentifier,
testcase_uri=EXCLUDED.testcase_uri,
weeknumber=EXCLUDED.weeknumber """
# Define some variables for executing Select Query based on limits
offset = 0
per_query = 10000
while True:
#execute query based on limits using projects
cursor_remets.execute("select tsdata.ts_moduleid, coalesce(tsdata_extended.ts_objecttext,'Unknown') as ts_objecttext, tsdata.ts_objectidentifier, SUBSTRING_INDEX(tsdata.ts_variant, ',', -1) as after_comma_value, tsdata.weeknumber,SUBSTRING_INDEX(tsdata.ts_variant, ',', -1) as project_id,SUBSTRING_INDEX(tsdata.ts_variant, ',', 1) as gc_id from tsdata left join tsdata_extended on tsdata_extended.ts_objectidentifier = tsdata.ts_objectidentifier and tsdata.ts_moduleid = tsdata_extended.ts_moduleid and tsdata.ts_variant = tsdata_extended.ts_variant where tsdata.weeknumber=%s OFFSET %s", (prev_max_weeknumber, per_query, offset))
rows = cursor_remets.fetchall()
if len(rows) > 0:
for row in rows:
#print(row)
testcond = (row[0] and row[1] and row[2] and row[4] and row[5] and row[6])
#testcond = True
if testcond:
cursor_prtm.execute(testcases,(row[0],row[1].replace("\x00", "\uFFFD").replace('\\', '\\\\'),row[2],None,None,row[6],row[4],row[5]))
conn_prtm.commit()
DataMigration.validateInsertedRecord('insertUpdateTestCase', row)
else:
log.Log('In insertUpdateTestCase, row validation failed ' + str(row), 'info')
else:
break
#print(str(len(rows)) + ' rows written successfully to prtm_testcase')
offset += per_query
log.Log('insertUpdateTestCase completed execution in ' + str(datetime.now()-insertUpdateTestCase_start_time), 'info')
Executed the SQL statement in the source database and rows are returned with no nulls or empty fields in the data.
Compared the list of fields against the defined statement and count the same number of parameters in both
I encountered a problem when trying to send sms using the SMSC service in Django project.
My Celery task for sending email and sms:
def order_created_retail(order_id):
# Task to send an email when an order is successfully created
order = OrderRetail.objects.get(id=order_id)
subject = 'Order №{}.'.format(order_id)
message_mail = 'Hello, {}! You have successfully placed an order{}. Manager will contact you shortly'.format(order.first_name, order.id)
message_sms = 'Your order №{} is accepted! Wait for operator call'
mail_sent = send_mail(
subject,
message_mail,
'email#email.com',
[order.email]
)
smsc = SMSC()
sms_sent = smsc.send_sms(
[order.phone],
str(message_sms)
)
return mail_sent, sms_sent
Email sends correctly, but for sms I get that error:
Task orders.tasks.order_created_retail[f05458b1-65e8-493b-9069-fbaa55083e7a] raised unexpected: TypeError('quote_from_bytes() expected bytes')
function from SMSC library:
def send_sms(self, phones, message, translit=0, time="", id=0, format=0, sender=False, query=""):
formats = ["flash=1", "push=1", "hlr=1", "bin=1", "bin=2", "ping=1", "mms=1", "mail=1", "call=1", "viber=1", "soc=1"]
m = self._smsc_send_cmd("send", "cost=3&phones=" + quote(phones) + "&mes=" + quote(message) + \
"&translit=" + str(translit) + "&id=" + str(id) + ifs(format > 0, "&" + formats[format-1], "") + \
ifs(sender == False, "", "&sender=" + quote(str(sender))) + \
ifs(time, "&time=" + quote(time), "") + ifs(query, "&" + query, ""))
# (id, cnt, cost, balance) или (id, -error)
if SMSC_DEBUG:
if m[1] > "0":
print("Сообщение отправлено успешно. ID: " + m[0] + ", всего SMS: " + m[1] + ", стоимость: " + m[2] + ", баланс: " + m[3])
else:
print("Ошибка №" + m[1][1:] + ifs(m[0] > "0", ", ID: " + m[0], ""))
return m
What am I doing wrong?
Thanks!
to solve this problem, I started investigating the functions that were giving out the error.
It turned out that I was passing an incorrect value. the function was expecting a string. And it took me a long time to figure out why editing didn't help.
It turns out that you have to RESET CELERY every time you make an edit.
I can search a shapefile for an attribute and it works fine but I don't know how to get the other fields in that record once the correct records is found. Don't know if I should use SearchCursor or SelectLayerByAttribute_management.
townlands = r'F:\MyProject\Assignment\townlands.shp'
outpath = r'F:\MyProject\Assignment'
the_townland=str(text_search_townland.get())
selection = str(""" "NAME_TAG" = '""" + the_townland + "'")
selection2 = ????????????????
print selection, selection2
This code is working in that it finds the townland that the user puts in text_search_townland and it prints it as selection. I'm looking to get another field called OSM_USER from that record into selection2.
I got this working after lots of trial and error. It does need SearchCursor or at least that is how I got it working.
def new_record():
#set environment variables.
arcpy.env.workspace = r'F:\MyProject\Assignment\folklore.gdb'
myPath = r'F:\MyProject\Assignment\folklore.gdb'
editRows = arcpy.da.InsertCursor('folklore', '*')
print editRows.fields
# get the centroid of the townland from townland_centroid (fc) based on the
# townland the user enters.
database = r'F:\MyProject\Assignment\folklore.gdb'
fc = database + '/' + 'townland_centroid'
the_townland=str(text_search_townland.get())
fields = ['NAME_TAG', 'X_coord', 'Y_coord']
whereClause = '"NAME_TAG"' + " = '" + the_townland + "'"
with arcpy.da.SearchCursor(fc, fields, whereClause) as cursor:
for row in cursor:
print('{0}, {1}, {2}'.format(row[0], row[1], row[2]))
X_coord = str(row[1])
Y_coord = str(row[2])
del cursor
# Set variables with values that will populate 'folklore' featureclass.
OID = 1
ptShape = arcpy.Point(0,0)
townland = text_search_townland.get()
county = var_county2.get()
category = var_category.get()
URL = text_search_URL.get()
spec_location = "text_search_speclocation.get()"
date_entered = text_search_date_entered.get()
story_year = int(text_search_story_year.get())
X_coord_put = X_coord
Y_coord_put = Y_coord
newRecord = [OID, ptShape, townland, county, URL, spec_location, date_entered, story_year, category, X_coord, Y_coord]
editRows.insertRow(newRecord)
del editRows
Hope this helps someone.
I use python operation postgresql database, the implementation of sql, it removed the quotation marks, resulting in inquiries failed, how to avoid?
def build_sql(self,table_name,keys,condition):
print(condition)
# condition = {
# "os":["Linux","Windows"],
# "client_type":["ordinary"],
# "client_status":'1',
# "offset":"1",
# "limit":"8"
# }
sql_header = "SELECT %s FROM %s" % (keys,table_name)
sql_condition = []
sql_range = []
sql_sort = []
sql_orederby = []
for key in condition:
if isinstance(condition[key],list):
sql_condition.append(key+" in ("+",".join(condition[key])+")")
elif key == 'limit' or key == 'offset':
sql_range.append(key + " " + condition[key])
else:
sql_condition.append(key + " = " + condition[key])
print(sql_condition)
print(sql_range)
sql_condition = [str(i) for i in sql_condition]
if not sql_condition == []:
sql_condition = " where " + " and ".join(sql_condition) + " "
sql = sql_header + sql_condition + " ".join(sql_range)
return sql
Error：
MySQL Error Code : column "winxp" does not exist
LINE 1: ...T * FROM ksc_client_info where base_client_os in (WinXP) and...
Mind you I do not have much Python experience, but basically you don't have single quotes in that sequence, so you either need to add those before passing it to function or for example during join(), like that:
sql_condition.append(key+" in ("+"'{0}'".format("','".join(condition[key]))+")")
You can see other solutions in those questions:
Join a list of strings in python and wrap each string in quotation marks
Add quotes to every list elements
Below is a function that extracts information from a database which holds information about events. Everything works except that when I try and iterate through times in rows in HTML it is apparently empty. I will therefore assume that rows.append(time) is not doing what it should be doing. I tried rows.append((time)) and that did not work either.
def extractor(n):
date = (datetime.datetime.now() + datetime.timedelta(days=n)).date()
rows = db.execute("SELECT * FROM events WHERE date LIKE :date ORDER BY date", date = str(date) + '%')
printed_day = date.strftime('%A') + ", " + date.strftime('%B') + " " + str(date.day) + ", " + str(datetime.datetime.now().year)
start_time = time.strftime("%H:%M:%S")
for row in rows:
date_split = str.split(row['date'])
just_time = date_split[1]
if just_time == '00:00:00':
just_time = 'All Day'
else:
just_time = just_time[0:5]
times.append((just_time))
rows.append(times)
results.append((rows, printed_day, start_time, times))
Solved it:
replace
times.append((just_time))
rows.append(times)
with
row['times'] = just_time