diff --git a/.gitignore b/.gitignore index 3d0e181..9a2468e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ benchmarks-env/ *.dmg .coverage .DS_Store +scratch/ diff --git a/app/models.py b/app/models.py index fa5f90a..deafad1 100644 --- a/app/models.py +++ b/app/models.py @@ -13,7 +13,7 @@ # Association table for many-to-many relationship between lists and users list_users = db.Table( # pylint: disable=invalid-name 'list_users', - db.Column('list_id', db.String(64), db.ForeignKey('list_stats.list_id'), + db.Column('list_id', db.String(64), db.ForeignKey('email_list.list_id'), primary_key=True), db.Column('user_id', db.Integer, db.ForeignKey('app_user.id'), primary_key=True)) @@ -31,13 +31,9 @@ def __repr__(self): return ''.format(self.id) class ListStats(db.Model): # pylint: disable=too-few-public-methods - """Stores individual MailChimp lists and their associated stats.""" - list_id = db.Column(db.String(64), primary_key=True) - list_name = db.Column(db.String(128)) - org_id = db.Column(db.Integer, db.ForeignKey('organization.id', - name='fk_org_id')) - api_key = db.Column(db.String(64)) - data_center = db.Column(db.String(64)) + """Stores stats associated with a MailChimp list.""" + id = db.Column(db.Integer, primary_key=True) + analysis_timestamp = db.Column(db.DateTime, default=datetime.utcnow) frequency = db.Column(db.Float) subscribers = db.Column(db.Integer) open_rate = db.Column(db.Float) @@ -48,13 +44,28 @@ class ListStats(db.Model): # pylint: disable=too-few-public-methods pending_pct = db.Column(db.Float) high_open_rt_pct = db.Column(db.Float) cur_yr_inactive_pct = db.Column(db.Float) + list_id = db.Column(db.String(64), db.ForeignKey('email_list.list_id', + name='fk_list_id')) + + def __repr__(self): + return ''.format(self.id) + +class EmailList(db.Model): # pylint: disable=too-few-public-methods + """Stores individual MailChimp lists.""" + list_id = db.Column(db.String(64), primary_key=True) + list_name = db.Column(db.String(128)) + api_key = db.Column(db.String(64)) + data_center = db.Column(db.String(64)) store_aggregates = db.Column(db.Boolean) monthly_updates = db.Column(db.Boolean) monthly_update_users = db.relationship( AppUser, secondary=list_users, backref='lists', lazy='subquery') + org_id = db.Column(db.Integer, db.ForeignKey('organization.id', + name='fk_org_id')) + analyses = db.relationship(ListStats, backref='list') def __repr__(self): - return ''.format(self.list_id) + return ''.format(self.list_id) class Organization(db.Model): # pylint: disable=too-few-public-methods """Stores a media or journalism organization.""" @@ -67,7 +78,7 @@ class Organization(db.Model): # pylint: disable=too-few-public-methods employee_range = db.Column(db.String(32)) budget = db.Column(db.String(64)) affiliations = db.Column(db.String(512)) - lists = db.relationship(ListStats, backref='org') + lists = db.relationship(EmailList, backref='org') users = db.relationship(AppUser, secondary=users, backref='orgs') def __repr__(self): diff --git a/app/tasks.py b/app/tasks.py index f93d8c3..c41aea3 100644 --- a/app/tasks.py +++ b/app/tasks.py @@ -1,16 +1,17 @@ """This module contains Celery tasks and functions associated with them.""" import os import json -import random import time +from datetime import datetime, timedelta, timezone import requests import numpy as np +from sqlalchemy import desc from sqlalchemy.sql.functions import func from celery.utils.log import get_task_logger from app import celery, db from app.emails import send_email from app.lists import MailChimpList, MailChimpImportError, do_async_import -from app.models import ListStats +from app.models import EmailList, ListStats from app.dbops import associate_user_with_list from app.visualizations import ( draw_bar, draw_stacked_horizontal_bar, draw_histogram, draw_donuts) @@ -81,12 +82,8 @@ def import_analyze_store_list(list_data, org_id, user_email=None): mailing_list.calc_high_open_rate_pct() mailing_list.calc_cur_yr_stats() - # Create a list object + # Create a set of stats list_stats = ListStats( - list_id=list_data['list_id'], - list_name=list_data['list_name'], - api_key=list_data['key'], - data_center=list_data['data_center'], frequency=mailing_list.frequency, subscribers=mailing_list.subscribers, open_rate=mailing_list.open_rate, @@ -97,13 +94,23 @@ def import_analyze_store_list(list_data, org_id, user_email=None): pending_pct=mailing_list.pending_pct, high_open_rt_pct=mailing_list.high_open_rt_pct, cur_yr_inactive_pct=mailing_list.cur_yr_inactive_pct, - store_aggregates=list_data['store_aggregates'], - monthly_updates=list_data['monthly_updates'], - org_id=org_id) + list_id=list_data['list_id']) - # If the user gave their permission, store the object in the database + # If the user gave their permission, store the stats in the database if list_data['monthly_updates'] or list_data['store_aggregates']: - list_stats = db.session.merge(list_stats) + + # Create a list object to go with the set of stats + email_list = EmailList( + list_id=list_data['list_id'], + list_name=list_data['list_name'], + api_key=list_data['key'], + data_center=list_data['data_center'], + store_aggregates=list_data['store_aggregates'], + monthly_updates=list_data['monthly_updates'], + org_id=org_id) + email_list = db.session.merge(email_list) + + db.session.add(list_stats) try: db.session.commit() except: @@ -113,7 +120,7 @@ def import_analyze_store_list(list_data, org_id, user_email=None): return list_stats def send_report(stats, list_id, list_name, user_email_or_emails): - """Generates charts using Pygal and emails them to the user. + """Generates charts using Plotly and emails them to the user. Args: stats: a dictionary containing analysis results for a list. @@ -122,22 +129,30 @@ def send_report(stats, list_id, list_name, user_email_or_emails): user_email_or_emails: a list of emails to send the report to. """ - # Generate aggregates for the database - # Only include lists where we have permission + # This subquery generates the most recent stats + # For each unique list_id in the database + # Where store_aggregates is True + subquery = ListStats.query.filter( + ListStats.list.has(store_aggregates=True)).order_by('list_id', desc( + 'analysis_timestamp')).distinct(ListStats.list_id).subquery() + + # Generate aggregates within the subquery agg_stats = db.session.query( - func.avg(ListStats.subscribers), - func.avg(ListStats.subscribed_pct), - func.avg(ListStats.unsubscribed_pct), - func.avg(ListStats.cleaned_pct), - func.avg(ListStats.pending_pct), - func.avg(ListStats.open_rate), - func.avg(ListStats.high_open_rt_pct), - func.avg(ListStats.cur_yr_inactive_pct)).filter_by( - store_aggregates=True).first() + func.avg(subquery.columns.subscribers), + func.avg(subquery.columns.subscribed_pct), + func.avg(subquery.columns.unsubscribed_pct), + func.avg(subquery.columns.cleaned_pct), + func.avg(subquery.columns.pending_pct), + func.avg(subquery.columns.open_rate), + func.avg(subquery.columns.high_open_rt_pct), + func.avg(subquery.columns.cur_yr_inactive_pct)).first() # Make sure we have no 'None' values agg_stats = [agg if agg else 0 for agg in agg_stats] + # Convert subscribers average to an integer + agg_stats[0] = int(agg_stats[0]) + # Generate epoch time (to get around image caching in webmail) epoch_time = str(int(time.time())) @@ -202,7 +217,7 @@ def send_report(stats, list_id, list_name, user_email_or_emails): os.environ.get('SES_CONFIGURATION_SET') or None)) def extract_stats(list_object): - """Extracts a stats dictionary from a list object from the database.""" + """Extracts a stats dictionary from a SQLAlchemy ListStats object.""" stats = {'subscribers': list_object.subscribers, 'open_rate': list_object.open_rate, 'hist_bin_counts': json.loads(list_object.hist_bin_counts), @@ -218,10 +233,12 @@ def extract_stats(list_object): def init_list_analysis(user_data, list_data, org_id): """Celery task wrapper for each stage of analyzing a list. - First checks if the list stats are cached, i.e. already in the + First checks if there is a recently cached analysis, i.e. already in the database. If not, calls import_analyze_store_list() to generate - them. Then checks if the user is already associated with the list, - if not, create the relationship. Finally, generates a benchmarking + the ListStats and an associated EmailList. Next updates the user's + privacy options (e.g. store_aggregates, monthly_updates) if the list was + cached. Then checks if the user selected monthly updates, if so, + create the relationship. Finally, generates a benchmarking report with the stats. Args: @@ -230,60 +247,88 @@ def init_list_analysis(user_data, list_data, org_id): org_id: the id of the organization associated with the list. """ - # Try to pull the list stats from database + # Try to pull the most recent ListStats from the database # Otherwise generate them - list_object = (ListStats.query.filter_by( - list_id=list_data['list_id']).first() or - import_analyze_store_list( - list_data, org_id, user_data['email'])) - - # Associate the list with the user who requested the analysis - # If that user requested monthly updates - if list_data['monthly_updates']: - associate_user_with_list(user_data['user_id'], list_object) - - stats = extract_stats(list_object) + most_recent_analysis = (ListStats.query.filter_by( + list_id=list_data['list_id']).order_by(desc( + 'analysis_timestamp')).first() or import_analyze_store_list( + list_data, org_id, user_data['email'])) + + # If the user chose to store their data, there will be an associated + # EmailList object + list_object = EmailList.query.filter_by( + list_id=list_data['list_id']).first() + + if list_object: + + # Update the privacy options if they differ from previous selection + if (list_object.monthly_updates != list_data['monthly_updates'] + or list_object.store_aggregates != list_data['store_aggregates']): + list_object.monthly_updates = list_data['monthly_updates'] + list_object.store_aggregates = list_data['store_aggregates'] + list_object = db.session.merge(list_object) + try: + db.session.commit() + except: + db.session.rollback() + raise + + # Associate the list with the user who requested the analysis + # If that user requested monthly updates + if list_data['monthly_updates']: + associate_user_with_list(user_data['user_id'], list_object) + + # Convert the ListStats object to an easier-to-use dictionary + stats = extract_stats(most_recent_analysis) send_report(stats, list_data['list_id'], list_data['list_name'], [user_data['email']]) @celery.task def update_stored_data(): """Celery task which goes through the database - and updates calculations using the most recent data. + and generates a new set of calculations for each list older than 30 days. This task is called by Celery Beat, see the schedule in config.py. """ - - # Get the logger logger = get_task_logger(__name__) - # Grab what we have in the database - list_objects = ListStats.query.with_entities( - ListStats.list_id, ListStats.list_name, ListStats.org_id, - ListStats.api_key, ListStats.data_center, - ListStats.store_aggregates, ListStats.monthly_updates).all() + # Grab the most recent analyses in the database + list_analyses = ListStats.query.order_by( + 'list_id', desc('analysis_timestamp')).distinct( + ListStats.list_id).all() - if not list_objects: - logger.info('No lists to update!') + if not list_analyses: + logger.warning('No lists in the database!') + return + + # Create a list of analyses which are more than 30 days old + now = datetime.now(timezone.utc) + one_month_ago = now - timedelta(days=30) + analyses_to_update = [ + analysis for analysis in list_analyses + if (analysis.analysis_timestamp.replace( + tzinfo=timezone.utc)) < one_month_ago] + + if not analyses_to_update: + logger.info('No old lists to update!') return # Placeholder for lists which failed during the update process failed_updates = [] - # Update 1/30th of the lists in the database (such that every list - # is updated about once per month, on average). - lists_to_update = random.sample( - list_objects, len(list_objects) // 31 if len(list_objects) // 31 else 1) - # Update each list's calculations in sequence - for list_to_update in lists_to_update: + for analysis in analyses_to_update: + + logger.info('Updating list %s!', analysis.list_id) - logger.info('Updating list %s!', list_to_update.list_id) + # Get the list object associated with the analysis + associated_list_object = analysis.list # Pull information about the list from the API # This may have changed since we originally pulled the list data request_uri = ('https://{}.api.mailchimp.com/3.0/lists/{}'.format( - list_to_update.data_center, list_to_update.list_id)) + associated_list_object.data_center, + associated_list_object.list_id)) params = ( ('fields', 'stats.member_count,' 'stats.unsubscribe_count,' @@ -294,7 +339,7 @@ def update_stored_data(): ) response = requests.get( request_uri, params=params, - auth=('shorenstein', list_to_update.api_key)) + auth=('shorenstein', associated_list_object.api_key)) response_body = response.json() response_stats = response_body['stats'] count = (response_stats['member_count'] + @@ -302,12 +347,12 @@ def update_stored_data(): response_stats['cleaned_count']) # Create a dictionary of list data - list_data = {'list_id': list_to_update.list_id, - 'list_name': list_to_update.list_name, - 'key': list_to_update.api_key, - 'data_center': list_to_update.data_center, - 'monthly_updates': list_to_update.monthly_updates, - 'store_aggregates': list_to_update.store_aggregates, + list_data = {'list_id': analysis.list_id, + 'list_name': associated_list_object.list_name, + 'key': associated_list_object.api_key, + 'data_center': associated_list_object.data_center, + 'monthly_updates': associated_list_object.monthly_updates, + 'store_aggregates': associated_list_object.store_aggregates, 'total_count': count, 'open_rate': response_stats['open_rate'], 'date_created': response_body['date_created'], @@ -315,10 +360,10 @@ def update_stored_data(): # Then re-run the calculations and update the database try: - import_analyze_store_list(list_data, list_to_update.org_id) + import_analyze_store_list(list_data, associated_list_object.org_id) except MailChimpImportError: - logger.error('Error updating list %s.', list_to_update.list_id) - failed_updates.append(list_to_update.list_id) + logger.error('Error updating list %s.', analysis.list_id) + failed_updates.append(analysis.list_id) # If any updates failed, raise an exception to send an error email if failed_updates: @@ -336,7 +381,7 @@ def send_monthly_reports(): logger = get_task_logger(__name__) # Grab info from the database - monthly_report_lists = ListStats.query.filter_by( + monthly_report_lists = EmailList.query.filter_by( monthly_updates=True).all() # Send an email report for each list @@ -352,8 +397,13 @@ def send_monthly_reports(): monthly_report_list.list_name, monthly_report_list.list_id) + # Get the most recent analysis for the list + stats_object = ListStats.query.filter_by( + list_id=monthly_report_list.list_id).order_by( + desc('analysis_timestamp')).first() + # Extract stats from the list object - stats = extract_stats(monthly_report_list) + stats = extract_stats(stats_object) send_report(stats, monthly_report_list.list_id, monthly_report_list.list_name, users_to_email) diff --git a/migrations/versions/0122cf52d3a6_changed_field_names_in_organization_.py b/migrations/versions/0122cf52d3a6_changed_field_names_in_organization_.py deleted file mode 100644 index 10ce96d..0000000 --- a/migrations/versions/0122cf52d3a6_changed_field_names_in_organization_.py +++ /dev/null @@ -1,34 +0,0 @@ -"""changed field names in organization table - -Revision ID: 0122cf52d3a6 -Revises: 09c25a640b79 -Create Date: 2018-09-25 18:28:42.304018 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '0122cf52d3a6' -down_revision = '09c25a640b79' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('organization', schema=None) as batch_op: - batch_op.add_column(sa.Column('affiliations', sa.String(length=512), nullable=True)) - batch_op.drop_column('affiliation') - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('organization', schema=None) as batch_op: - batch_op.add_column(sa.Column('affiliation', sa.VARCHAR(length=64), nullable=True)) - batch_op.drop_column('affiliations') - - # ### end Alembic commands ### diff --git a/migrations/versions/09c25a640b79_add_organizations_table.py b/migrations/versions/09c25a640b79_add_organizations_table.py deleted file mode 100644 index 138fdf1..0000000 --- a/migrations/versions/09c25a640b79_add_organizations_table.py +++ /dev/null @@ -1,79 +0,0 @@ -"""add organizations table - -Revision ID: 09c25a640b79 -Revises: 5faf2d470d12 -Create Date: 2018-09-12 13:55:43.781238 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '09c25a640b79' -down_revision = '5faf2d470d12' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('organization', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('name', sa.String(length=128), nullable=True), - sa.Column('financial_classification', sa.String(length=32), nullable=True), - sa.Column('coverage_scope', sa.String(length=32), nullable=True), - sa.Column('coverage_focus', sa.String(length=64), nullable=True), - sa.Column('platform', sa.String(length=64), nullable=True), - sa.Column('employee_range', sa.String(length=32), nullable=True), - sa.Column('budget', sa.String(length=64), nullable=True), - sa.Column('affiliation', sa.String(length=64), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - with op.batch_alter_table('organization', schema=None) as batch_op: - batch_op.create_index(batch_op.f('ix_organization_name'), ['name'], unique=True) - - op.create_table('users', - sa.Column('org_id', sa.Integer(), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['org_id'], ['organization.id'], ), - sa.ForeignKeyConstraint(['user_id'], ['app_user.id'], ), - sa.PrimaryKeyConstraint('org_id', 'user_id') - ) - with op.batch_alter_table('app_user', schema=None) as batch_op: - batch_op.add_column(sa.Column('name', sa.String(length=64), nullable=True)) - batch_op.drop_column('news_org') - batch_op.drop_column('newsletters') - batch_op.drop_column('contact_person') - - with op.batch_alter_table('list_stats', schema=None) as batch_op: - batch_op.add_column(sa.Column('frequency', sa.Float(), nullable=True)) - batch_op.add_column(sa.Column('org_id', sa.Integer(), nullable=True)) - batch_op.drop_constraint('fk_app_user_id', type_='foreignkey') - batch_op.create_foreign_key('fk_org_id', 'organization', ['org_id'], ['id']) - batch_op.drop_column('user_id') - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('list_stats', schema=None) as batch_op: - batch_op.add_column(sa.Column('user_id', sa.INTEGER(), nullable=True)) - batch_op.drop_constraint('fk_org_id', type_='foreignkey') - batch_op.create_foreign_key('fk_app_user_id', 'app_user', ['user_id'], ['id']) - batch_op.drop_column('org_id') - batch_op.drop_column('frequency') - - with op.batch_alter_table('app_user', schema=None) as batch_op: - batch_op.add_column(sa.Column('contact_person', sa.VARCHAR(length=64), nullable=True)) - batch_op.add_column(sa.Column('newsletters', sa.VARCHAR(length=512), nullable=True)) - batch_op.add_column(sa.Column('news_org', sa.VARCHAR(length=64), nullable=True)) - batch_op.drop_column('name') - - op.drop_table('users') - with op.batch_alter_table('organization', schema=None) as batch_op: - batch_op.drop_index(batch_op.f('ix_organization_name')) - - op.drop_table('organization') - # ### end Alembic commands ### diff --git a/migrations/versions/5faf2d470d12_release_app_v2.py b/migrations/versions/5faf2d470d12_release_app_v2.py deleted file mode 100644 index 24ee235..0000000 --- a/migrations/versions/5faf2d470d12_release_app_v2.py +++ /dev/null @@ -1,67 +0,0 @@ -"""release app v2 - -Revision ID: 5faf2d470d12 -Revises: -Create Date: 2018-08-08 12:06:04.026696 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '5faf2d470d12' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('app_user', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('signup_timestamp', sa.DateTime(), nullable=True), - sa.Column('news_org', sa.String(length=64), nullable=True), - sa.Column('contact_person', sa.String(length=64), nullable=True), - sa.Column('email', sa.String(length=64), nullable=True), - sa.Column('email_hash', sa.String(length=64), nullable=True), - sa.Column('newsletters', sa.String(length=512), nullable=True), - sa.Column('approved', sa.Boolean(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - with op.batch_alter_table('app_user', schema=None) as batch_op: - batch_op.create_index(batch_op.f('ix_app_user_email'), ['email'], unique=True) - batch_op.create_index(batch_op.f('ix_app_user_email_hash'), ['email_hash'], unique=True) - - op.create_table('list_stats', - sa.Column('list_id', sa.String(length=64), nullable=False), - sa.Column('list_name', sa.String(length=128), nullable=True), - sa.Column('user_id', sa.Integer(), nullable=True), - sa.Column('api_key', sa.String(length=64), nullable=True), - sa.Column('data_center', sa.String(length=64), nullable=True), - sa.Column('subscribers', sa.Integer(), nullable=True), - sa.Column('open_rate', sa.Float(), nullable=True), - sa.Column('hist_bin_counts', sa.String(length=512), nullable=True), - sa.Column('subscribed_pct', sa.Float(), nullable=True), - sa.Column('unsubscribed_pct', sa.Float(), nullable=True), - sa.Column('cleaned_pct', sa.Float(), nullable=True), - sa.Column('pending_pct', sa.Float(), nullable=True), - sa.Column('high_open_rt_pct', sa.Float(), nullable=True), - sa.Column('cur_yr_inactive_pct', sa.Float(), nullable=True), - sa.Column('store_aggregates', sa.Boolean(), nullable=True), - sa.Column('monthly_updates', sa.Boolean(), nullable=True), - sa.ForeignKeyConstraint(['user_id'], ['app_user.id'], name='fk_app_user_id'), - sa.PrimaryKeyConstraint('list_id') - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('list_stats') - with op.batch_alter_table('app_user', schema=None) as batch_op: - batch_op.drop_index(batch_op.f('ix_app_user_email_hash')) - batch_op.drop_index(batch_op.f('ix_app_user_email')) - - op.drop_table('app_user') - # ### end Alembic commands ### diff --git a/migrations/versions/a921df1ce5a3_add_relationship_between_lists_and_users.py b/migrations/versions/a921df1ce5a3_add_relationship_between_lists_and_users.py deleted file mode 100644 index 2fbfb10..0000000 --- a/migrations/versions/a921df1ce5a3_add_relationship_between_lists_and_users.py +++ /dev/null @@ -1,34 +0,0 @@ -"""add relationship between lists and users - -Revision ID: a921df1ce5a3 -Revises: 0122cf52d3a6 -Create Date: 2018-09-29 11:37:52.385864 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = 'a921df1ce5a3' -down_revision = '0122cf52d3a6' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('list_users', - sa.Column('list_id', sa.String(length=64), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.ForeignKeyConstraint(['list_id'], ['list_stats.list_id'], ), - sa.ForeignKeyConstraint(['user_id'], ['app_user.id'], ), - sa.PrimaryKeyConstraint('list_id', 'user_id') - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('list_users') - # ### end Alembic commands ### diff --git a/migrations/versions/e1150a91b8d1_release_version_3_0.py b/migrations/versions/e1150a91b8d1_release_version_3_0.py new file mode 100644 index 0000000..21c910d --- /dev/null +++ b/migrations/versions/e1150a91b8d1_release_version_3_0.py @@ -0,0 +1,109 @@ +"""release version 3.0 + +Revision ID: e1150a91b8d1 +Revises: +Create Date: 2019-01-09 15:14:30.861585 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'e1150a91b8d1' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('app_user', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('signup_timestamp', sa.DateTime(), nullable=True), + sa.Column('name', sa.String(length=64), nullable=True), + sa.Column('email', sa.String(length=64), nullable=True), + sa.Column('email_hash', sa.String(length=64), nullable=True), + sa.Column('approved', sa.Boolean(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('app_user', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_app_user_email'), ['email'], unique=True) + batch_op.create_index(batch_op.f('ix_app_user_email_hash'), ['email_hash'], unique=True) + + op.create_table('organization', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=128), nullable=True), + sa.Column('financial_classification', sa.String(length=32), nullable=True), + sa.Column('coverage_scope', sa.String(length=32), nullable=True), + sa.Column('coverage_focus', sa.String(length=64), nullable=True), + sa.Column('platform', sa.String(length=64), nullable=True), + sa.Column('employee_range', sa.String(length=32), nullable=True), + sa.Column('budget', sa.String(length=64), nullable=True), + sa.Column('affiliations', sa.String(length=512), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + with op.batch_alter_table('organization', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_organization_name'), ['name'], unique=True) + + op.create_table('email_list', + sa.Column('list_id', sa.String(length=64), nullable=False), + sa.Column('list_name', sa.String(length=128), nullable=True), + sa.Column('api_key', sa.String(length=64), nullable=True), + sa.Column('data_center', sa.String(length=64), nullable=True), + sa.Column('store_aggregates', sa.Boolean(), nullable=True), + sa.Column('monthly_updates', sa.Boolean(), nullable=True), + sa.Column('org_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['org_id'], ['organization.id'], name='fk_org_id'), + sa.PrimaryKeyConstraint('list_id') + ) + op.create_table('users', + sa.Column('org_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['org_id'], ['organization.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['app_user.id'], ), + sa.PrimaryKeyConstraint('org_id', 'user_id') + ) + op.create_table('list_stats', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('analysis_timestamp', sa.DateTime(), nullable=True), + sa.Column('frequency', sa.Float(), nullable=True), + sa.Column('subscribers', sa.Integer(), nullable=True), + sa.Column('open_rate', sa.Float(), nullable=True), + sa.Column('hist_bin_counts', sa.String(length=512), nullable=True), + sa.Column('subscribed_pct', sa.Float(), nullable=True), + sa.Column('unsubscribed_pct', sa.Float(), nullable=True), + sa.Column('cleaned_pct', sa.Float(), nullable=True), + sa.Column('pending_pct', sa.Float(), nullable=True), + sa.Column('high_open_rt_pct', sa.Float(), nullable=True), + sa.Column('cur_yr_inactive_pct', sa.Float(), nullable=True), + sa.Column('list_id', sa.String(length=64), nullable=True), + sa.ForeignKeyConstraint(['list_id'], ['email_list.list_id'], name='fk_list_id'), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('list_users', + sa.Column('list_id', sa.String(length=64), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['list_id'], ['email_list.list_id'], ), + sa.ForeignKeyConstraint(['user_id'], ['app_user.id'], ), + sa.PrimaryKeyConstraint('list_id', 'user_id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('list_users') + op.drop_table('list_stats') + op.drop_table('users') + op.drop_table('email_list') + with op.batch_alter_table('organization', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_organization_name')) + + op.drop_table('organization') + with op.batch_alter_table('app_user', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_app_user_email_hash')) + batch_op.drop_index(batch_op.f('ix_app_user_email')) + + op.drop_table('app_user') + # ### end Alembic commands ### diff --git a/tests/integration/test_analysis.py b/tests/integration/test_analysis.py index 303084d..e38adf7 100644 --- a/tests/integration/test_analysis.py +++ b/tests/integration/test_analysis.py @@ -3,7 +3,7 @@ import json import requests from app import db -from app.models import AppUser, Organization, ListStats +from app.models import AppUser, Organization, EmailList def test_analysis(client, caplog): """End-to-end test of analyzing a list.""" @@ -78,10 +78,12 @@ def test_analysis(client, caplog): assert any(list_id + '_high_open_rt_pct_' in file for file in chart_files) assert any(list_id + '_cur_yr_inactive_pct_' in file for file in chart_files) assert ('Suppressing an email with the following params: ' - 'Sender: testing@testing.com. Recipients: [\'foo@bar.com\']. ' - 'Subject: Your Email Benchmarking Report is Ready!' - in caplog.text) - list_result = ListStats.query.filter_by(list_id=list_id).first() - assert list_result - assert list_result.org.id == existing_org_id - assert list_result.monthly_update_users[0].id == existing_user_id + 'Sender: testing@testing.com. Recipients: [\'foo@bar.com\']. ' + 'Subject: Your Email Benchmarking Report is Ready!' + in caplog.text) + email_list = EmailList.query.filter_by(list_id=list_id).first() + assert email_list + assert email_list.org.id == existing_org_id + assert email_list.monthly_update_users[0].id == existing_user_id + assert email_list.analyses[0] + assert email_list.analyses[0].open_rate == data['stats']['open_rate'] diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index cd0a461..1e8716d 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -88,8 +88,7 @@ def mocked_mailchimp_list(mocker, fake_calculation_results): """Mocks the MailChimp list class from app/lists.py and attaches fake calculation results to the mock attributes.""" mocked_mailchimp_list = mocker.patch('app.tasks.MailChimpList') - for k, v in fake_calculation_results.items(): - setattr(mocked_mailchimp_list.return_value, k, v) + mocked_mailchimp_list.return_value = MagicMock(**fake_calculation_results) yield mocked_mailchimp_list @pytest.fixture diff --git a/tests/unit/test_tasks.py b/tests/unit/test_tasks.py index e2ea2cf..5322621 100644 --- a/tests/unit/test_tasks.py +++ b/tests/unit/test_tasks.py @@ -1,4 +1,5 @@ import logging +from datetime import datetime, timezone from unittest.mock import MagicMock, ANY, call import json import pytest @@ -67,12 +68,10 @@ def test_import_analyze_store_list( mocked_mailchimp_list_instance.calc_high_open_rate_pct.assert_called() mocked_mailchimp_list_instance.calc_cur_yr_stats.assert_called() assert isinstance(list_stats, ListStats) - _, kwargs = mocked_list_stats.call_args - for k, v in fake_calculation_results.items(): - if k != 'hist_bin_counts': - assert kwargs[k] == v - else: - assert kwargs[k] == json.dumps(v) + mocked_list_stats.assert_called_with( + **{k: (v if k != 'hist_bin_counts' else json.dumps(v)) + for k, v in fake_calculation_results.items()}, + list_id=fake_list_data['list_id']) def test_import_analyze_store_list_store_results_in_db( mocker, fake_list_data, mocked_mailchimp_list): @@ -80,10 +79,20 @@ def test_import_analyze_store_list_store_results_in_db( is stored in the db.""" mocker.patch('app.tasks.do_async_import') mocked_list_stats = mocker.patch('app.tasks.ListStats') + mocked_email_list = mocker.patch('app.tasks.EmailList') mocked_db = mocker.patch('app.tasks.db') fake_list_data['monthly_updates'] = True import_analyze_store_list(fake_list_data, 'foo') - mocked_db.session.merge.assert_called_with(mocked_list_stats.return_value) + mocked_email_list.assert_called_with( + list_id=fake_list_data['list_id'], + list_name=fake_list_data['list_name'], + api_key=fake_list_data['key'], + data_center=fake_list_data['data_center'], + store_aggregates=fake_list_data['store_aggregates'], + monthly_updates=fake_list_data['monthly_updates'], + org_id='foo') + mocked_db.session.merge.assert_called_with(mocked_email_list.return_value) + mocked_db.session.add.assert_called_with(mocked_list_stats.return_value) mocked_db.session.commit.assert_called() def test_import_analyze_store_list_store_results_in_db_exception( @@ -92,6 +101,7 @@ def test_import_analyze_store_list_store_results_in_db_exception( is stored in the db and an exception occurs.""" mocker.patch('app.tasks.do_async_import') mocker.patch('app.tasks.ListStats') + mocker.patch('app.tasks.EmailList') mocked_db = mocker.patch('app.tasks.db') mocked_db.session.commit.side_effect = Exception() fake_list_data['monthly_updates'] = True @@ -104,8 +114,7 @@ def test_send_report(mocker, fake_calculation_results): mocked_db = mocker.patch('app.tasks.db') agg_stats_length = 8 mocked_agg_stats = [1 for x in range(agg_stats_length)] - (mocked_db.session.query.return_value.filter_by.return_value - .first.return_value) = mocked_agg_stats + mocked_db.session.query.return_value.first.return_value = mocked_agg_stats mocked_draw_bar = mocker.patch('app.tasks.draw_bar') mocked_draw_stacked_horizontal_bar = mocker.patch( 'app.tasks.draw_stacked_horizontal_bar') @@ -114,8 +123,6 @@ def test_send_report(mocker, fake_calculation_results): mocked_send_email = mocker.patch('app.tasks.send_email') send_report(fake_calculation_results, '1', 'foo', ['foo@bar.com']) assert len(mocked_db.session.query.call_args[0]) == agg_stats_length - mocked_db.session.query.return_value.filter_by.assert_called_with( - store_aggregates=True) mocked_draw_bar.assert_has_calls([ call( ANY, [fake_calculation_results['subscribers'], mocked_agg_stats[0]], @@ -157,82 +164,134 @@ def test_send_report(mocker, fake_calculation_results): def test_extract_stats(fake_calculation_results): """Tests the extract_stats function.""" - fake_list_object = MagicMock() fake_calculation_results.pop('frequency') - for k, v in fake_calculation_results.items(): - if k != 'hist_bin_counts': - setattr(fake_list_object, k, v) - else: - setattr(fake_list_object, k, json.dumps(v)) + fake_list_object = MagicMock( + **{k: (json.dumps(v) if k == 'hist_bin_counts' else v) + for k, v in fake_calculation_results.items()} + ) stats = extract_stats(fake_list_object) assert stats == fake_calculation_results -def test_init_list_analysis_existing_list(mocker, fake_list_data): - """Tests the init_list_analysis function when the list exists in the - database.""" +def test_init_list_analysis_existing_list_update_privacy_options( + mocker, fake_list_data): + """Tests the init_list_analysis function when the list exists in + the database. Also tests that monthly_updates and store_aggregates + are updated if they differ from that stored in the database.""" mocked_list_stats = mocker.patch('app.tasks.ListStats') + mocked_recent_analysis = ( + mocked_list_stats.query.filter_by.return_value.order_by + .return_value.first.return_value) + mocked_desc = mocker.patch('app.tasks.desc') + mocked_email_list = mocker.patch('app.tasks.EmailList') mocked_list_object = ( - mocked_list_stats.query.filter_by.return_value.first.return_value) + mocked_email_list.query.filter_by.return_value.first.return_value) + mocked_list_object.monthly_updates = True + mocked_list_object.store_aggregates = False + mocked_db = mocker.patch('app.tasks.db') mocked_extract_stats = mocker.patch('app.tasks.extract_stats') mocked_send_report = mocker.patch('app.tasks.send_report') init_list_analysis({'email': 'foo@bar.com'}, fake_list_data, 1) mocked_list_stats.query.filter_by.assert_called_with( list_id=fake_list_data['list_id']) - mocked_extract_stats.assert_called_with(mocked_list_object) + mocked_list_stats.query.filter_by.return_value.order_by.assert_called_with( + mocked_desc.return_value) + mocked_email_list.query.filter_by.assert_called_with( + list_id=fake_list_data['list_id']) + mocked_db.session.merge.assert_called_with(mocked_list_object) + mocked_db.session.commit.assert_called() + mocked_extract_stats.assert_called_with(mocked_recent_analysis) mocked_send_report.assert_called_with( mocked_extract_stats.return_value, fake_list_data['list_id'], fake_list_data['list_name'], ['foo@bar.com']) -def test_init_list_analysis_new_list(mocker, fake_list_data): - """Tests the init_list_analysis function when the list does not exist in - the database.""" +def test_init_analysis_existing_list_db_error(mocker, fake_list_data): + """Tests the init_list_analysis function when the list exists in the + database and a database error occurs.""" + mocker.patch('app.tasks.ListStats') + mocked_email_list = mocker.patch('app.tasks.EmailList') + mocked_list_object = ( + mocked_email_list.query.filter_by.return_value.first.return_value) + mocked_list_object.monthly_updates = True + mocked_list_object.store_aggregates = False + mocked_db = mocker.patch('app.tasks.db') + mocked_db.session.commit.side_effect = Exception() + with pytest.raises(Exception): + init_list_analysis({'email': 'foo@bar.com'}, fake_list_data, 1) + mocked_db.session.rollback.assert_called() + +def test_init_list_analysis_new_list_no_store(mocker, fake_list_data): + """Tests the init_list_analysis function when the list does not exist + in the database and the user chose not to store their data.""" mocked_list_stats = mocker.patch('app.tasks.ListStats') - mocked_list_stats.query.filter_by.return_value.first.return_value = None + (mocked_list_stats.query.filter_by.return_value.order_by + .return_value.first.return_value) = None mocked_import_analyze_store_list = mocker.patch( 'app.tasks.import_analyze_store_list') - mocked_list_object = mocked_import_analyze_store_list.return_value - mocked_extract_stats = mocker.patch('app.tasks.extract_stats') + mocked_email_list = mocker.patch('app.tasks.EmailList') + mocked_email_list.query.filter_by.return_value.first.return_value = None + mocker.patch('app.tasks.extract_stats') mocker.patch('app.tasks.send_report') init_list_analysis({'email': 'foo@bar.com'}, fake_list_data, 1) mocked_import_analyze_store_list.assert_called_with( fake_list_data, 1, 'foo@bar.com') - mocked_extract_stats.assert_called_with(mocked_list_object) -def test_init_list_analysis_monthly_updates(mocker, fake_list_data): - """Tests that the init_list_analysis function correctly associates a user - with a list if the user requested monthly updates.""" +def test_init_list_analysis_new_list_monthly_updates(mocker, fake_list_data): + """Tests the init_list_analysis function when the list does not + exist in the database and the user chose to store their data and + requested monthly updates.""" mocked_list_stats = mocker.patch('app.tasks.ListStats') + (mocked_list_stats.query.filter_by.return_value.order_by + .return_value.first.return_value) = None + mocker.patch('app.tasks.import_analyze_store_list') + mocked_email_list = mocker.patch('app.tasks.EmailList') mocked_list_object = ( - mocked_list_stats.query.filter_by.return_value.first.return_value) + mocked_email_list.query.filter_by.return_value.first.return_value) + mocked_list_object.monthly_updates = True + mocked_list_object.store_aggregates = False mocked_associate_user_with_list = mocker.patch( 'app.tasks.associate_user_with_list') mocker.patch('app.tasks.extract_stats') mocker.patch('app.tasks.send_report') fake_list_data['monthly_updates'] = True - init_list_analysis({'user_id': 1, 'email': 'foo@bar.com'}, fake_list_data, 2) - mocked_associate_user_with_list.assert_called_with(1, mocked_list_object) - + init_list_analysis( + {'email': 'foo@bar.com', 'user_id': 2}, fake_list_data, 1) + mocked_associate_user_with_list.assert_called_with(2, mocked_list_object) def test_update_stored_data_empty_db(mocker, caplog): """Tests the update_stored_data function when there are no lists stored in the database.""" mocked_list_stats = mocker.patch('app.tasks.ListStats') - mocked_list_stats.query.with_entities.return_value.all.return_value = None + (mocked_list_stats.query.order_by.return_value.distinct + .return_value.all.return_value) = None + update_stored_data() + assert 'No lists in the database!' in caplog.text + +def test_update_stored_data_no_old_analyses(mocker, caplog): + """Tests the update_stored_data function when there are no analyses older + than 30 days.""" + mocked_list_stats = mocker.patch('app.tasks.ListStats') + mocked_analysis = MagicMock( + analysis_timestamp=datetime.now(timezone.utc)) + (mocked_list_stats.query.order_by.return_value.distinct + .return_value.all.return_value) = [mocked_analysis] caplog.set_level(logging.INFO) update_stored_data() - assert 'No lists to update!' in caplog.text + assert 'No old lists to update' in caplog.text + def test_update_stored_data(mocker, fake_list_data): """Tests the update_stored_data function.""" mocked_list_stats = mocker.patch('app.tasks.ListStats') - fake_list_to_update = MagicMock() - for k, v in fake_list_data.items(): - if k == 'key': - setattr(fake_list_to_update, 'api_key', v) - else: - setattr(fake_list_to_update, k, v) - mocked_list_stats.query.with_entities.return_value.all.return_value = ( - [fake_list_to_update]) + mocked_list_to_update = MagicMock( + **{('api_key' if k == 'key' else k): v + for k, v in fake_list_data.items()} + ) + mocked_analysis = MagicMock( + analysis_timestamp=datetime(2000, 1, 1, tzinfo=timezone.utc), + list=mocked_list_to_update, + list_id=fake_list_data['list_id']) + (mocked_list_stats.query.order_by.return_value.distinct + .return_value.all.return_value) = [mocked_analysis] mocked_requests = mocker.patch('app.tasks.requests') mocked_import_analyze_store_list = mocker.patch( 'app.tasks.import_analyze_store_list') @@ -271,11 +330,20 @@ def test_update_stored_data(mocker, fake_list_data): 'campaign_count': 10}, 1) -def test_update_stored_data_import_error(mocker, caplog): + +def test_update_stored_data_import_error(mocker, fake_list_data, caplog): """Tests the update_stored_data function when the list import raises an error.""" mocked_list_stats = mocker.patch('app.tasks.ListStats') - mocked_list_stats.query.with_entities.return_value.all.return_value = ( - [MagicMock(list_id='foo')]) + mocked_list_to_update = MagicMock( + **{('api_key' if k == 'key' else k): v + for k, v in fake_list_data.items()} + ) + mocked_analysis = MagicMock( + analysis_timestamp=datetime(2000, 1, 1, tzinfo=timezone.utc), + list=mocked_list_to_update, + list_id=fake_list_data['list_id']) + (mocked_list_stats.query.order_by.return_value.distinct + .return_value.all.return_value) = [mocked_analysis] mocked_requests = mocker.patch('app.tasks.requests') mocked_import_analyze_store_list = mocker.patch( 'app.tasks.import_analyze_store_list') @@ -295,21 +363,23 @@ def test_update_stored_data_import_error(mocker, caplog): update_stored_data() assert 'Error updating list foo.' in caplog.text + def test_send_monthly_reports(mocker, fake_list_data, caplog): """Tests the send_monthly_reports function.""" - mocked_list_stats = mocker.patch('app.tasks.ListStats') - fake_list = MagicMock() - for k, v in fake_list_data.items(): - setattr(fake_list, k, v) - fake_list.monthly_update_users = [MagicMock(email='foo@bar.com')] - mocked_list_stats.query.filter_by.return_value.all.return_value = ( - [fake_list]) + mocked_email_list = mocker.patch('app.tasks.EmailList') + mocked_list = MagicMock(**fake_list_data) + mocked_list.monthly_update_users = [MagicMock(email='foo@bar.com')] + mocked_email_list.query.filter_by.return_value.all.return_value = [mocked_list] caplog.set_level(logging.INFO) + mocked_list_stats = mocker.patch('app.tasks.ListStats') + mocked_stats_object = ( + mocked_list_stats.query.filter_by.return_value.order_by + .return_value.first.return_value) mocked_extract_stats = mocker.patch('app.tasks.extract_stats') mocked_send_report = mocker.patch('app.tasks.send_report') send_monthly_reports() assert ('Emailing foo@bar.com an updated report. List: bar (foo).' in caplog.text) - mocked_extract_stats.assert_called_with(fake_list) + mocked_extract_stats.assert_called_with(mocked_stats_object) mocked_send_report.assert_called_with( mocked_extract_stats.return_value, 'foo', 'bar', ['foo@bar.com'])