-
-
Notifications
You must be signed in to change notification settings - Fork 612
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[8.0][IMP] Performance issues in mail_tracking addons #97
Changes from 2 commits
718f884
2afe69c
b301ecb
147f4e3
e00d331
70addf6
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -11,17 +11,40 @@ | |
BLANK = 'R0lGODlhAQABAIAAANvf7wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==' | ||
|
||
|
||
def _env_get(db): | ||
def _env_get(db, callback, tracking_id, event_type, **kw): | ||
res = 'NOT FOUND' | ||
reg = False | ||
try: | ||
reg = registry(db) | ||
except OperationalError: | ||
_logger.warning("Selected BD '%s' not found", db) | ||
except: # pragma: no cover | ||
_logger.warning("Selected BD '%s' connection error", db) | ||
if reg: | ||
return api.Environment(reg.cursor(), SUPERUSER_ID, {}) | ||
return False | ||
current = http.request.db and db == http.request.db | ||
env = current and http.request.env | ||
if not env: | ||
try: | ||
reg = registry(db) | ||
except OperationalError: | ||
_logger.warning("Selected BD '%s' not found", db) | ||
except: # pragma: no cover | ||
_logger.warning("Selected BD '%s' connection error", db) | ||
if reg: | ||
_logger.info("Creating a new environment for database '%s'", db) | ||
env = api.Environment(reg.cursor(), SUPERUSER_ID, {}) | ||
else: | ||
# make sudo when reusing environment | ||
env = env(user=SUPERUSER_ID) | ||
if env: | ||
# This try-except-finally is equivalent to a with statement | ||
# We assure that new created cursor si commit/rollback/close | ||
# in any case | ||
try: | ||
res = callback(env, tracking_id, event_type, **kw) | ||
if not current: | ||
env.cr.commit() | ||
except Exception as e: | ||
if not current: | ||
env.cr.rollback() | ||
raise e | ||
finally: | ||
if not current: | ||
env.cr.close() | ||
return res | ||
|
||
|
||
class MailTrackingController(http.Controller): | ||
|
@@ -35,49 +58,37 @@ def _request_metadata(self): | |
'ua_family': request.user_agent.browser or False, | ||
} | ||
|
||
def _tracking_open(self, env, tracking_id, event_type, **kw): | ||
tracking_email = env['mail.tracking.email'].search([ | ||
('id', '=', tracking_id), | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Use There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I use tracking_email recordset after that |
||
]) | ||
if tracking_email: | ||
metadata = self._request_metadata() | ||
tracking_email.event_create('open', metadata) | ||
else: | ||
_logger.warning( | ||
"MailTracking email '%s' not found", tracking_id) | ||
|
||
def _tracking_event(self, env, tracking_id, event_type, **kw): | ||
metadata = self._request_metadata() | ||
return env['mail.tracking.email'].event_process( | ||
http.request, kw, metadata, event_type=event_type) | ||
|
||
@http.route('/mail/tracking/all/<string:db>', | ||
type='http', auth='none') | ||
def mail_tracking_all(self, db, **kw): | ||
env = _env_get(db) | ||
if not env: | ||
return 'NOT FOUND' | ||
metadata = self._request_metadata() | ||
response = env['mail.tracking.email'].event_process( | ||
http.request, kw, metadata) | ||
env.cr.commit() | ||
env.cr.close() | ||
return response | ||
return _env_get(db, self._tracking_event, None, None, **kw) | ||
|
||
@http.route('/mail/tracking/event/<string:db>/<string:event_type>', | ||
type='http', auth='none') | ||
def mail_tracking_event(self, db, event_type, **kw): | ||
env = _env_get(db) | ||
if not env: | ||
return 'NOT FOUND' | ||
metadata = self._request_metadata() | ||
response = env['mail.tracking.email'].event_process( | ||
http.request, kw, metadata, event_type=event_type) | ||
env.cr.commit() | ||
env.cr.close() | ||
return response | ||
return _env_get(db, self._tracking_event, None, event_type, **kw) | ||
|
||
@http.route('/mail/tracking/open/<string:db>' | ||
'/<int:tracking_email_id>/blank.gif', | ||
type='http', auth='none') | ||
def mail_tracking_open(self, db, tracking_email_id, **kw): | ||
env = _env_get(db) | ||
if env: | ||
tracking_email = env['mail.tracking.email'].search([ | ||
('id', '=', tracking_email_id), | ||
]) | ||
if tracking_email: | ||
metadata = self._request_metadata() | ||
tracking_email.event_create('open', metadata) | ||
else: | ||
_logger.warning( | ||
"MailTracking email '%s' not found", tracking_email_id) | ||
env.cr.commit() | ||
env.cr.close() | ||
_env_get(db, self._tracking_open, tracking_email_id, None, **kw) | ||
|
||
# Always return GIF blank image | ||
response = werkzeug.wrappers.Response() | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,6 +3,7 @@ | |
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html). | ||
|
||
import logging | ||
import threading | ||
import urlparse | ||
import time | ||
import re | ||
|
@@ -23,14 +24,19 @@ class MailTrackingEmail(models.Model): | |
_rec_name = 'display_name' | ||
_description = 'MailTracking email' | ||
|
||
# This table is going to growth fast and to infinite, so we index: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. s/growth/grow |
||
# - name: Search in tree view | ||
# - time: default order fields | ||
# - recipient_address: Used for email_store calculation (non-store) | ||
# - state: Search and group_by in tree view | ||
name = fields.Char(string="Subject", readonly=True, index=True) | ||
display_name = fields.Char( | ||
string="Display name", readonly=True, store=True, | ||
compute="_compute_display_name") | ||
timestamp = fields.Float( | ||
string='UTC timestamp', readonly=True, | ||
digits=dp.get_precision('MailTracking Timestamp')) | ||
time = fields.Datetime(string="Time", readonly=True) | ||
time = fields.Datetime(string="Time", readonly=True, index=True) | ||
date = fields.Date( | ||
string="Date", readonly=True, compute="_compute_date", store=True) | ||
mail_message_id = fields.Many2one( | ||
|
@@ -42,7 +48,7 @@ class MailTrackingEmail(models.Model): | |
recipient = fields.Char(string='Recipient email', readonly=True) | ||
recipient_address = fields.Char( | ||
string='Recipient email address', readonly=True, store=True, | ||
compute='_compute_recipient_address') | ||
compute='_compute_recipient_address', index=True) | ||
sender = fields.Char(string='Sender email', readonly=True) | ||
state = fields.Selection([ | ||
('error', 'Error'), | ||
|
@@ -87,70 +93,49 @@ class MailTrackingEmail(models.Model): | |
string="Tracking events", comodel_name='mail.tracking.event', | ||
inverse_name='tracking_email_id', readonly=True) | ||
|
||
@api.model | ||
def tracking_ids_recalculate(self, model, email_field, tracking_field, | ||
email, new_tracking=None): | ||
objects = self.env[model].search([ | ||
(email_field, '=ilike', email), | ||
]) | ||
for obj in objects: | ||
trackings = obj[tracking_field] | ||
if new_tracking: | ||
trackings |= new_tracking | ||
trackings = trackings._email_score_tracking_filter() | ||
if set(obj[tracking_field].ids) != set(trackings.ids): | ||
if trackings: | ||
obj.write({ | ||
tracking_field: [(6, False, trackings.ids)] | ||
}) | ||
else: | ||
obj.write({ | ||
tracking_field: [(5, False, False)] | ||
}) | ||
return objects | ||
def _email_score_tracking_filter(self, domain, order='time desc', | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
limit=10): | ||
"""Default tracking search. Ready to be inherited.""" | ||
return self.search(domain, limit=limit, order=order) | ||
|
||
@api.model | ||
def _tracking_ids_to_write(self, email): | ||
trackings = self.env['mail.tracking.email'].search([ | ||
('recipient_address', '=ilike', email) | ||
]) | ||
trackings = trackings._email_score_tracking_filter() | ||
if trackings: | ||
return [(6, False, trackings.ids)] | ||
else: | ||
return [(5, False, False)] | ||
|
||
@api.multi | ||
def _email_score_tracking_filter(self): | ||
"""Default email score filter for tracking emails""" | ||
# Consider only last 10 tracking emails | ||
return self.sorted(key=lambda r: r.time, reverse=True)[:10] | ||
def email_is_bounced(self, email): | ||
return len(self._email_score_tracking_filter([ | ||
('recipient_address', '=ilike', email), | ||
('state', 'in', ('error', 'rejected', 'spam', 'bounced')), | ||
])) > 0 | ||
|
||
@api.model | ||
def email_score_from_email(self, email): | ||
trackings = self.env['mail.tracking.email'].search([ | ||
return self._email_score_tracking_filter([ | ||
('recipient_address', '=ilike', email) | ||
]) | ||
return trackings.email_score() | ||
]).email_score() | ||
|
||
@api.multi | ||
def email_score(self): | ||
"""Default email score algorimth""" | ||
"""Default email score algorimth. Ready to be inherited | ||
|
||
Must return a value beetwen 0.0 and 100.0 | ||
- Bad reputation: Value between 0 and 50.0 | ||
- Unknown reputation: Value 50.0 | ||
- Good reputation: Value between 50.0 and 100.0 | ||
""" | ||
score = 50.0 | ||
trackings = self._email_score_tracking_filter() | ||
for tracking in trackings: | ||
for tracking in self: | ||
if tracking.state in ('error',): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This would be cleaner & more efficient in a dictionary:
|
||
score -= 50.0 | ||
elif tracking.state in ('rejected', 'spam', 'bounced'): | ||
score -= 25.0 | ||
elif tracking.state in ('soft-bounced', 'unsub'): | ||
score -= 10.0 | ||
elif tracking.state in ('delivered',): | ||
score += 5.0 | ||
score += 1.0 | ||
elif tracking.state in ('opened',): | ||
score += 10.0 | ||
score += 5.0 | ||
if score > 100.0: | ||
score = 100.0 | ||
if score < 0.0: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You're right |
||
score = 0.0 | ||
return score | ||
|
||
@api.multi | ||
|
@@ -167,7 +152,7 @@ def _compute_recipient_address(self): | |
@api.depends('name', 'recipient') | ||
def _compute_display_name(self): | ||
for email in self: | ||
parts = [email.name] | ||
parts = [email.name or ''] | ||
if email.recipient: | ||
parts.append(email.recipient) | ||
email.display_name = ' - '.join(parts) | ||
|
@@ -179,14 +164,6 @@ def _compute_date(self): | |
email.date = fields.Date.to_string( | ||
fields.Date.from_string(email.time)) | ||
|
||
@api.model | ||
def create(self, vals): | ||
tracking = super(MailTrackingEmail, self).create(vals) | ||
self.tracking_ids_recalculate( | ||
'res.partner', 'email', 'tracking_email_ids', | ||
tracking.recipient_address, new_tracking=tracking) | ||
return tracking | ||
|
||
def _get_mail_tracking_img(self): | ||
base_url = self.env['ir.config_parameter'].get_param('web.base.url') | ||
path_url = ( | ||
|
@@ -202,6 +179,13 @@ def _get_mail_tracking_img(self): | |
'tracking_email_id': self.id, | ||
}) | ||
|
||
@api.multi | ||
def _partners_email_bounced_set(self, reason): | ||
for tracking_email in self: | ||
self.env['res.partner'].search([ | ||
('email', '=ilike', tracking_email.recipient_address) | ||
]).email_bounced_set(tracking_email, reason) | ||
|
||
@api.multi | ||
def smtp_error(self, mail_server, smtp_server, exception): | ||
self.sudo().write({ | ||
|
@@ -210,6 +194,7 @@ def smtp_error(self, mail_server, smtp_server, exception): | |
'error_description': tools.ustr(exception), | ||
'state': 'error', | ||
}) | ||
self.sudo()._partners_email_bounced_set('error') | ||
return True | ||
|
||
@api.multi | ||
|
@@ -228,7 +213,7 @@ def _message_partners_check(self, message, message_id): | |
partners = mail_message.notified_partner_ids | mail_message.partner_ids | ||
if (self.partner_id and self.partner_id not in partners): | ||
# If mail_message haven't tracking partner, then | ||
# add it in order to see his trackking status in chatter | ||
# add it in order to see his tracking status in chatter | ||
if mail_message.subtype_id: | ||
mail_message.sudo().write({ | ||
'notified_partner_ids': [(4, self.partner_id.id)], | ||
|
@@ -287,20 +272,21 @@ def _concurrent_events(self, event_type, metadata): | |
|
||
@api.multi | ||
def event_create(self, event_type, metadata): | ||
testing = getattr(threading.currentThread(), 'testing', False) | ||
event_ids = self.env['mail.tracking.event'] | ||
for tracking_email in self: | ||
other_ids = tracking_email._concurrent_events(event_type, metadata) | ||
if not other_ids: | ||
vals = tracking_email._event_prepare(event_type, metadata) | ||
if vals: | ||
event_ids += event_ids.sudo().create(vals) | ||
partners = self.tracking_ids_recalculate( | ||
'res.partner', 'email', 'tracking_email_ids', | ||
tracking_email.recipient_address) | ||
if partners: | ||
partners.email_score_calculate() | ||
# Commit to DB to release exclusive lock | ||
if not testing: | ||
self.env.cr.commit() # pragma: no cover | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why can we not use savepoint here? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. when using savepoint, if everything is going well (no exceptions), then transaction is committed? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmmm ok looking at the intent here, we may not want a Savepoint. It's not immediately committing the transaction, and will roll back entirely if another transaction afterwards fails (assuming that transaction is also not within a savepoint). Think of it like a save-guard - if the transaction within the Savepoint fails, it will rollback just within that savepoint. If the transaction fails outside of a savepoint, all mutations & savepoints in that transaction will fail. Note this is a Postgres feature, so it's significantly easier to play with in direct SQL if you're curious. Surprisingly, the Postgres docs on Savepoint are pretty nice & the Odoo implementation matches well. Where is the exclusive lock that is mentioned in the comments actually created? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I can see this Odoo code: https://github.com/odoo/odoo/blob/d81258ab8e0dcb6c0df3a47ab8084ec99b7d3e24/openerp/sql_db.py#L390 According to that, RELEASE SAVEPOINT just remove the savepoint. But in this case, I need to have the event (and mail state) visible to other isolated cursors and release exclusivelock to mail_tracking_email and mail_tracking_event tables. With this commit, then other writes (for instance to res_partner table) can be done after reading mail_tracking_email or mail_tracking_event tables. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. After thinking twice, I remove this commit. Now the addon is not reading mail_tracking_email nor email_tracking_event tables after create/write on them, so I don't need this. |
||
else: | ||
_logger.debug("Concurrent event '%s' discarded", event_type) | ||
if event_type in {'hard_bounce', 'spam', 'reject'}: | ||
self.sudo()._partners_email_bounced_set(event_type) | ||
return event_ids | ||
|
||
@api.model | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Not sure if you can use
with env.cr:
orwith env.cr.savepoint():
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This pattern is equivalent to
with
, but in this case if more explicit and easier to debug if we need it in future. See: http://effbot.org/zone/python-with-statement.htmThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't see a functional difference though? This pattern fails Lint & I am not sure that debugging is a good enough reason to keep it.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks @lasley, I didn't read transaction related guidelines before. In a multi-db instance we need to get a new env (and cursor) because ORM can initialize it (several databases, none selected). But I can make it better following guidelines. I'm doing it right now