I'm trying to import 5000+ rows in Odoo 12 it's basically a mapping from a CSV developed in a custom method in a module, the problem I'm getting timeout in the request, that's happening when writing to the database, I'm using the standard ERP methods create and write.
How can I work around a solution to this? I know bulk insert is not possible to this, any other solution to this?
is a SQL command for insertion OK to use?
class file_reader(models.TransientModel):
_name = "rw.file.reader"
csv_file = fields.Binary(string='CSV File', required=True)
#api.multi
def import_csv(self):
# csv importer handler
file = base64.b64decode(self.csv_file).decode().split('\n')
reader = csv.DictReader(file)
# account.analytic.line
ignored = []
time1 = datetime.now()
self._cr.execute('select id, name from project_project where active = true')
projects = self._cr.fetchall()
self._cr.execute('select id, login from res_users')
users = self._cr.fetchall()
self._cr.execute('select id, work_email from hr_employee')
employees = self._cr.fetchall()
LOG_EVERY_N = 100
for row in reader:
project_name = row['Project - Name']
email = row['User - Email Address']
project = [item for item in projects if item[1] == project_name]
if len(project) >0:
user = [item for item in users if item[1] == email]
employee = [item for item in employees if item[1] == email]
if len(user)>0 and len(employee)>0:
task = self.env['project.task'].search([['user_id','=',user[0][0]],
['project_id','=',project[0][0] ]],limit=1)
if task:
y = row['Duration'].split(':')
i, j = y[0], y[1]
model = {
'project_id': project[0][0],
'task_id': task['id'],
'employee_id':employee[0][0],
'user_id': user[0][0],
'date': row['Date'],
'unit_amount': int(i) + (float(j) / 60), # Time Spent convertion to float
'is_timesheet': True,
'billable': True if row['Billable'] == 'Yes' else False,
'nexonia_id':row['ID']
}
time_sheet = self.env['account.analytic.line'].search([['nexonia_id','=', row['ID']]],limit=1)
if time_sheet:
model.update({'id':time_sheet.id})
self.env['account.analytic.line'].sudo().write(model)
else:
self.env['account.analytic.line'].sudo().create(model)
else:
if email not in ignored:
ignored.append(email)
else:
if project_name not in ignored:
ignored.append(project_name)
all_text = 'Nothing ignored'
if ignored is not None:
all_text = "\n".join(filter(None, ignored))
message_id = self.env['message.wizard'].create({
'message': "Import data completed",
'ignored': all_text
})
time2 = datetime.now()
logging.info('total time ------------------------------------------ %s',time2-time1)
return {
'name': 'Successfull',
'type': 'ir.actions.act_window',
'view_mode': 'form',
'res_model': 'message.wizard',
# pass the id
'res_id': message_id.id,
'target': 'new'
}
I Enhanced your code a litle bit because you are searching for each project, user and employee using loop
for each row and for 5000+ row.
Using ORM method is always good because, they handle the stored compute fields and python constrains, but this will take time too
if you don't have any complex compute you can use INSERT or UPDATE query this will speed up the importion 100 times.
#api.multi
def import_csv(self):
# when you use env[model] for more than ones extract it to variable its better
# notice how I added sudo to the name of variable
AccountAnalyticLine_sudo =self.env['account.analytic.line'].sudo()
# csv importer handler
file = base64.b64decode(self.csv_file).decode().split('\n')
reader = csv.DictReader(file)
# account.analytic.line
ignored = []
time1 = datetime.now()
# convert result to dictionary for easy access later
self._cr.execute('select id, name from project_project where active = true order by name')
projects = {p[1]: p for p in self._cr.fetchall()}
self._cr.execute('select id, login from res_users order by login')
users = {u[1]: u for u in self._cr.fetchall()}
self._cr.execute('select id, work_email from hr_employee order by work_email')
employees = {emp[1]: emp for emp in self._cr.fetchall()}
LOG_EVERY_N = 100
for row in reader:
project_name = row['Project - Name']
email = row['User - Email Address']
# no need for loop and the dicionary loopkup is so fast
project = projects.get(project_name)
if project:
user = user.get(email)
employee = employees.get(email)
if user and employee:
task = self.env['project.task'].search([('user_id','=',user[0]),
('project_id','=',project[0])],
limit=1)
if task:
y = row['Duration'].split(':')
i, j = y[0], y[1]
# by convention dictionary that are passed to create or write should be named vals or values
vals = {
'project_id': project[0],
'task_id': task['id'],
'employee_id':employee[0],
'user_id': user[0],
'date': row['Date'],
'unit_amount': int(i) + (float(j) / 60), # Time Spent convertion to float
'is_timesheet': True,
'billable': True if row['Billable'] == 'Yes' else False,
'nexonia_id':row['ID']
}
time_sheet = AccountAnalyticLine_sudo.search([('nexonia_id','=', row['ID'])],limit=1)
# I think adding logger message here will be or create and update counters to know how much record record were updated or created
if time_sheet:
# I think you want to update the existing time sheet record so do this
# record.write(vals)
time_sheet.write(vals)
# you are updating an empty RecordSet
#self.env['account.analytic.line'].sudo().write(model)
else:
# create new one
AccountAnalyticLine_sudo.create(model)
else:
if email not in ignored:
ignored.append(email)
else:
if project_name not in ignored:
ignored.append(project_name)
all_text = 'Nothing ignored'
# ignored is not None is always True because ignored is a list
if ignored:
all_text = "\n".join(filter(None, ignored))
message_id = self.env['message.wizard'].create({
'message': "Import data completed",
'ignored': all_text
})
time2 = datetime.now()
logging.info('total time ------------------------------------------ %s',time2-time1)
return {
'name': 'Successfull',
'type': 'ir.actions.act_window',
'view_mode': 'form',
'res_model': 'message.wizard',
# pass the id
'res_id': message_id.id,
'target': 'new'
}
I hope this will help you a little bit even that the question is meant for somethinng else but I'm confused Odoo usually allow request to be handled
for 60 minutes.
While you are importing records through script, code optimization is very important.Try to reduce the number of search/read calls by using dictionary to save each result or use the SQL which i don't recommend.
Related
I'm getting record does not exist or has been deleted.\n(record: account.move.line(5398,), user: 7) error while updating data to odoo. Following is my code can anyone help to solve this problem.
import xmlrpc.client
endpoint_url = "/api/account.move/"
obj = get_object_or_404(OrderItem, order__id=order_id)
invoice_date = obj.order.created_on
name = obj.product.varient_name
price = obj.total
quantity = obj.quantity
payment_source = obj.order.payment_method
payment_reference = obj.order.order_number
common = xmlrpc.client.ServerProxy('{}/xmlrpc/2/common'.format(url))
uid = common.authenticate(db, username, password, {})
models = xmlrpc.client.ServerProxy('{}/xmlrpc/2/object'.format(url))
ids = models.execute_kw(db, uid, password, 'account.move', 'search_read', [[['source_document', '=', payment_reference]]], {'fields': ['partner_id', 'id']})
invoice_id = ids[0]['id']
partner_id_ = ids[0]['partner_id'][0]
headers = {
"access-token":tokens,
"Content-type":"application/jsonp",
"Cookie":session_id
}
api_invoice_line_id = [(1, invoice_id,{'name':name, 'price_unit':price, 'quantity':quantity})]
data = {
"partner_id":partner_id_,
"invoice_date":str(invoice_date),
"move_type":"out_invoice",
"__api__invoice_line_ids":str(api_invoice_line_id),
"payment_source":payment_source,
"source_document": payment_reference,
"rider":rider_name,
"ref":""
}
datas_ = json.dumps(data, indent=4)
req = requests.put(url+endpoint_url+str(invoice_id), headers=headers, data=datas_)
if req.status_code == 200:
status = "Update Successful"
else:
status = str(req.text)
return status
It looks like your code is failing because of the tuple you give to insert/update data in __api__invoice_line_ids.
I suppose it's some account.move.line Many2many or One2many field.
I see you are using the command 1 with invoice_id (an account.move id) and some data in a dict.
The problem here is that you are then trying to add that in the field pointing to account.move.line. So your ID 5398 is the account.move id, not an account.move.line id.
Not sure what your goal is to achieve here. If it's to push/update some account.move records with new data, change your __api__invoice_line_ids to point to account.move.
If your goal is to push/update some account.move.line records, then you better loop on the line_ids of your invoice_id :)
If I wasn't clear on something or you have any other question don't hesitate asking !
I'm creating a customer in square and getting the results as follows. What I need is to get the id of customer.
My code :
from square.client import Client
client = Client(
access_token=settings.SQUARE_ACCESS_TOKEN,
environment=settings.SQUARE_ENVIRONMENT,
)
api_customers = client.customers
request_body = {'idempotency_key': idempotency_key, 'given_name': name, 'company_name': company,'phone_number':phone}
result = api_customers.create_customer(request_body)
And this is the output:
<ApiResponse [{"customer":
{"id": "F8M9KDHWPMYGK2108RMQVQ6FHC",
"created_at": "2020-10-22T09:14:50.159Z",
"updated_at": "2020-10-22T09:14:50Z",
"given_name": "mkv5",
"phone_number": "900000066666",
"company_name": "codesvera",
"preferences": {"email_unsubscribed": false},
"creation_source": "THIRD_PARTY"}
}
]>
Are you using this library ?
https://github.com/square/square-python-sdk/blob/master/square/http/api_response.py
if yes result is an array and APiResponse object.
so first you should do that : result = result.body
then to get the ID: result['customer']['id']
Ps : You have exemple in the github doc :
https://github.com/square/square-python-sdk
# Initialize the customer count
total_customers = 0
# Initialize the cursor with an empty string since we are
# calling list_customers for the first time
cursor = ""
# Count the total number of customers using the list_customers method
while True:
# Call list_customers method to get all customers in this Square account
result = api_customers.list_customers(cursor)
if result.is_success():
# If any customers are returned, the body property
# is a list with the name customers.
# If there are no customers, APIResponse returns
# an empty dictionary.
if result.body:
customers = result.body['customers']
total_customers += len(customers)
# Get the cursor if it exists in the result else set it to None
cursor = result.body.get('cursor', None)
print(f"cursor: {cursor}")
else:
print("No customers.")
break
# Call the error method to see if the call failed
elif result.is_error():
print(f"Errors: {result.errors}")
break
# If there is no cursor, we are at the end of the list.
if cursor == None:
break
print(f"Total customers: {total_customers}")
I am Overloading the get_mail_values method of mail.compose.message. I have added email_cc field only and I am expecting the mail should go to both recipients.
#api.multi
def get_mail_values(self, res_ids):
"""Generate the values that will be used by send_mail to create mail_messages
or mail_mails. """
self.ensure_one()
results = dict.fromkeys(res_ids, False)
rendered_values = {}
mass_mail_mode = self.composition_mode == 'mass_mail'
# render all template-based value at once
if mass_mail_mode and self.model:
rendered_values = self.render_message(res_ids)
# compute alias-based reply-to in batch
reply_to_value = dict.fromkeys(res_ids, None)
if mass_mail_mode and not self.no_auto_thread:
# reply_to_value = self.env['mail.thread'].with_context(thread_model=self.model).browse(res_ids).message_get_reply_to(default=self.email_from)
reply_to_value = self.env['mail.thread'].with_context(thread_model=self.model).message_get_reply_to(res_ids, default=self.email_from)
for res_id in res_ids:
# static wizard (mail.message) values
mail_values = {
'subject': self.subject,
'body': self.body or '',
'parent_id': self.parent_id and self.parent_id.id,
'partner_ids': [partner.id for partner in self.partner_ids],
**'email_cc' : self.partner_ids_cc,**
'attachment_ids': [attach.id for attach in self.attachment_ids],
'author_id': self.author_id.id,
'email_from': self.email_from,
'record_name': self.record_name,
'no_auto_thread': self.no_auto_thread,
'mail_server_id': self.mail_server_id.id,
}
# mass mailing: rendering override wizard static values
if mass_mail_mode and self.model:
# keep a copy unless specifically requested, reset record name (avoid browsing records)
mail_values.update(notification=not self.auto_delete_message, model=self.model, res_id=res_id, record_name=False)
# auto deletion of mail_mail
if self.auto_delete or self.template_id.auto_delete:
mail_values['auto_delete'] = True
# rendered values using template
email_dict = rendered_values[res_id]
mail_values['partner_ids'] += email_dict.pop('partner_ids', [])
mail_values.update(email_dict)
if not self.no_auto_thread:
mail_values.pop('reply_to')
if reply_to_value.get(res_id):
mail_values['reply_to'] = reply_to_value[res_id]
if self.no_auto_thread and not mail_values.get('reply_to'):
mail_values['reply_to'] = mail_values['email_from']
# mail_mail values: body -> body_html, partner_ids -> recipient_ids
mail_values['body_html'] = mail_values.get('body', '')
mail_values['recipient_ids'] = [(4, id) for id in mail_values.pop('partner_ids', [])]
# process attachments: should not be encoded before being processed by message_post / mail_mail create
mail_values['attachments'] = [(name, base64.b64decode(enc_cont)) for name, enc_cont in email_dict.pop('attachments', list())]
attachment_ids = []
for attach_id in mail_values.pop('attachment_ids'):
new_attach_id = self.env['ir.attachment'].browse(attach_id).copy({'res_model': self._name, 'res_id': self.id})
attachment_ids.append(new_attach_id.id)
mail_values['attachment_ids'] = self.env['mail.thread']._message_preprocess_attachments(
mail_values.pop('attachments', []),
attachment_ids, 'mail.message', 0)
results[res_id] = mail_values
return results
As per my conclusion mail.compose.message passing control to mail.mail object and send method of mail.mail object is responsible to send mail to normal recipients and CC recipient.
Please suggest me where i am doing mistakes?
I need to override the create method in my model on odoo 10 :
in my module i have Three Models :
Asset With
validated = fields.Boolean("Is validated")
survey2_ids = fields.One2many('mymodule.survey2', 'asset_id', string='Survey2')
Survey2 with :
name = fields.Char()
asset_id = fields.Many2one('asset.asset', description='Asset')
survey1_id = fields.Many2one('mymodule.survey1', description="Survey1")
description = fields.Text(description="description")
Survey1 with :
name = fields.Char(description="Name")
ok = fields.Boolean("Is ok")
description = fields.Text()
The goal in here is when creating a new asset, and if validated = True: all records in mymodule.survey1 with ok==True should be copied in survey2_ids, i tried this function but it doesn't seem to be working:
#api.model
def create(self, vals):
survey1_ids = self.env['mymodule.survey1'].search([('ok', '=', True)])
if self.validated:
for rec in survey1_ids:
vals['survey2_ids'] = [(0, False, {'asset_id': self.id, 'survey2_id': rec.id,'name':rec.name,'description':})]
return super(asset_asset, self).create(vals)
Any help will be aappreciated
There are two problems in your code :
Create is kind of a "class method" (it is tied to the model, no to the record). So when you ask for the value of self.validated, this will always be false because self is not the record you're creating, it's the model. You should check vals.get('validated') instead. Or create the record before-hand and use it instead of self (in my example, res in the newly created record).
You're not really copying survey 1 into survey 2. You just have to create survey 2 using the data in survey 1.
The solution that I think is best :
#api.model
def create(self, vals):
res = super(asset_asset, self).create(vals)
if vals.get('validated'):
survey1_ids = self.env['mymodule.survey1'].search([('ok', '=', True)])
for s in survey1_ids:
v = {
'name': s.name,
'description': s.description,
'survey1_id': s.id,
'asset_id': res.id
}
self.env['mymodule.survey2'].create(v)
return res
Assuming that there are no errors in the logs, you are not getting what you intended to do. Once the code has executed, you are only getting 1 survey attached to the asset.
This is because inside the create function you wrote:
vals['survey2_ids'] = [(0, False, {'asset_id': self.id, 'survey2_id': rec.id,'name':rec.name,'description':})]
This will override the survey2_id in the vals each and every time in the for loop.
What you should do here is:
survey_2_list = []
for rec in survey1_ids:
survey_2_list.append((0, False, {'asset_id': self.id, 'survey2_id': rec.id,'name':rec.name,'description':rec.description}))
vals['survey2_ids'] = survey_2_list
Try the following:
#api.model
def create(self, vals):
survey_2_list = []
if self.validated:
survey1_ids = self.env['mymodule.survey1'].search([('ok', '=', True)])
if survey1_ids:
for rec in survey1_ids:
values = {
'asset_id': self.id,
'survey2_id': rec.id,
'name':rec.name,
'description':rec.description,
}
survey_2_list.append((0, False, values))
vals['survey2_ids'] = survey_2_list
return super(asset_asset, self).create(vals)
I have a flask application which is receiving a request from dataTables Editor. Upon receipt at the server, request.form looks like (e.g.)
ImmutableMultiDict([('data[59282][gender]', u'M'), ('data[59282][hometown]', u''),
('data[59282][disposition]', u''), ('data[59282][id]', u'59282'),
('data[59282][resultname]', u'Joe Doe'), ('data[59282][confirm]', 'true'),
('data[59282][age]', u'27'), ('data[59282][place]', u'3'), ('action', u'remove'),
('data[59282][runnerid]', u''), ('data[59282][time]', u'29:49'),
('data[59282][club]', u'')])
I am thinking to use something similar to this really ugly code to decode it. Is there a better way?
from collections import defaultdict
# request.form comes in multidict [('data[id][field]',value), ...]
# so we need to exec this string to turn into python data structure
data = defaultdict(lambda: {}) # default is empty dict
# need to define text for each field to be received in data[id][field]
age = 'age'
club = 'club'
confirm = 'confirm'
disposition = 'disposition'
gender = 'gender'
hometown = 'hometown'
id = 'id'
place = 'place'
resultname = 'resultname'
runnerid = 'runnerid'
time = 'time'
# fill in data[id][field] = value
for formkey in request.form.keys():
exec '{} = {}'.format(d,repr(request.form[formkey]))
This question has an accepted answer and is a bit old but since the DataTable module seems being pretty popular among jQuery community still, I believe this approach may be useful for someone else. I've just wrote a simple parsing function based on regular expression and dpath module, though it appears not to be quite reliable module. The snippet may be not very straightforward due to an exception-relied fragment, but it was only one way to prevent dpath from trying to resolve strings as integer indices I found.
import re, dpath.util
rxsKey = r'(?P<key>[^\W\[\]]+)'
rxsEntry = r'(?P<primaryKey>[^\W]+)(?P<secondaryKeys>(\[' \
+ rxsKey \
+ r'\])*)\W*'
rxKey = re.compile(rxsKey)
rxEntry = re.compile(rxsEntry)
def form2dict( frmDct ):
res = {}
for k, v in frmDct.iteritems():
m = rxEntry.match( k )
if not m: continue
mdct = m.groupdict()
if not 'secondaryKeys' in mdct.keys():
res[mdct['primaryKey']] = v
else:
fullPath = [mdct['primaryKey']]
for sk in re.finditer( rxKey, mdct['secondaryKeys'] ):
k = sk.groupdict()['key']
try:
dpath.util.get(res, fullPath)
except KeyError:
dpath.util.new(res, fullPath, [] if k.isdigit() else {})
fullPath.append(int(k) if k.isdigit() else k)
dpath.util.new(res, fullPath, v)
return res
The practical usage is based on native flask request.form.to_dict() method:
# ... somewhere in a view code
pars = form2dict(request.form.to_dict())
The output structure includes both, dictionary and lists, as one could expect. E.g.:
# A little test:
rs = jQDT_form2dict( {
'columns[2][search][regex]' : False,
'columns[2][search][value]' : None,
'columns[2][search][regex]' : False,
} )
generates:
{
"columns": [
null,
null,
{
"search": {
"regex": false,
"value": null
}
}
]
}
Update: to handle lists as dictionaries (in more efficient way) one may simplify this snippet with following block at else part of if clause:
# ...
else:
fullPathStr = mdct['primaryKey']
for sk in re.finditer( rxKey, mdct['secondaryKeys'] ):
fullPathStr += '/' + sk.groupdict()['key']
dpath.util.new(res, fullPathStr, v)
I decided on a way that is more secure than using exec:
from collections import defaultdict
def get_request_data(form):
'''
return dict list with data from request.form
:param form: MultiDict from `request.form`
:rtype: {id1: {field1:val1, ...}, ...} [fieldn and valn are strings]
'''
# request.form comes in multidict [('data[id][field]',value), ...]
# fill in id field automatically
data = defaultdict(lambda: {})
# fill in data[id][field] = value
for formkey in form.keys():
if formkey == 'action': continue
datapart,idpart,fieldpart = formkey.split('[')
if datapart != 'data': raise ParameterError, "invalid input in request: {}".format(formkey)
idvalue = int(idpart[0:-1])
fieldname = fieldpart[0:-1]
data[idvalue][fieldname] = form[formkey]
# return decoded result
return data