diff --git a/.flake8 b/.flake8 index 45555e7a0d..55fb4e173f 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] max-line-length = 88 select = C,E,F,W,B,B950 -extend-ignore = E203, E501 +extend-ignore = E203,E501 per-file-ignores = __init__.py:F401 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1eda51018..7d993a785f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,6 +75,14 @@ jobs: pip install mock pip install requests-mock + - name: Python Static Analysis + env: + DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test + run: | + pip install prospector + pip install -r requirements/azure.pip + prospector -X -s veryhigh onadata + - name: Run tests run: | python manage.py test ${{ matrix.testfolder }} --noinput --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..5feb00ad06 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,12 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files +- repo: https://github.com/psf/black + rev: 22.6.0 + hooks: + - id: black diff --git a/.prospector.yaml b/.prospector.yaml index 27ee1e3a30..ef872b4a17 100644 --- a/.prospector.yaml +++ b/.prospector.yaml @@ -5,10 +5,14 @@ autodetect: true member-warnings: false max-line-length: 88 +pycodestyle: + disable: + - E203 # Whitespace before ':' pylint: options: extension-pkg-allow-list: - ujson + - lxml.etree mccabe: run: false diff --git a/.pylintrc b/.pylintrc index 686428e7b6..056fb09489 100644 --- a/.pylintrc +++ b/.pylintrc @@ -3,7 +3,7 @@ # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code -extension-pkg-allow-list=ujson +extension-pkg-allow-list=ujson,lxml.etree # Add files or directories to the blacklist. They should be base names, not # paths. @@ -50,7 +50,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call,too-few-public-methods +disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call,too-few-public-methods,django-not-configured # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/onadata/__init__.py b/onadata/__init__.py index 6a6c8dff5c..fc79f9a9a0 100644 --- a/onadata/__init__.py +++ b/onadata/__init__.py @@ -11,6 +11,6 @@ # This will make sure the app is always imported when # Django starts so that shared_task will use this app. -from .celery import app as celery_app +from .celeryapp import app as celery_app __all__ = ("celery_app",) diff --git a/onadata/apps/api/management/commands/apply_can_add_project_perms.py b/onadata/apps/api/management/commands/apply_can_add_project_perms.py index b351c3fdca..f2a377ccaa 100644 --- a/onadata/apps/api/management/commands/apply_can_add_project_perms.py +++ b/onadata/apps/api/management/commands/apply_can_add_project_perms.py @@ -9,6 +9,7 @@ """ from django.core.management.base import BaseCommand from django.utils.translation import gettext as _ + from guardian.shortcuts import assign_perm from onadata.apps.api.models import OrganizationProfile @@ -24,9 +25,10 @@ def org_can_add_project_permission(): for organization in organizations.iterator(): permissions = organization.orgprofileuserobjectpermission_set.filter( - permission__codename='can_add_xform') + permission__codename="can_add_xform" + ) for permission in permissions: - assign_perm('can_add_project', permission.user, organization) + assign_perm("can_add_project", permission.user, organization) def user_can_add_project_permission(): @@ -38,9 +40,10 @@ def user_can_add_project_permission(): for user in users.iterator(): permissions = user.userprofileuserobjectpermission_set.filter( - permission__codename='can_add_xform') + permission__codename="can_add_xform" + ) for permission in permissions: - assign_perm('can_add_project', permission.user, user) + assign_perm("can_add_project", permission.user, user) class Command(BaseCommand): @@ -48,7 +51,8 @@ class Command(BaseCommand): Command apply_can_add_preject_perms - applys can_add_project permission to all users who have can_add_xform permission to a user/organization profile. """ - help = _(u"Apply can_add_project permissions") + + help = _("Apply can_add_project permissions") def handle(self, *args, **options): user_can_add_project_permission() diff --git a/onadata/apps/api/management/commands/assign_team_member_permission.py b/onadata/apps/api/management/commands/assign_team_member_permission.py index 9057244f3e..b2fe8d6a13 100644 --- a/onadata/apps/api/management/commands/assign_team_member_permission.py +++ b/onadata/apps/api/management/commands/assign_team_member_permission.py @@ -1,20 +1,26 @@ -from django.core.management.base import BaseCommand +# -*- coding: utf-8 -*- +""" +Assign permission to the member team +""" from django.core.exceptions import ObjectDoesNotExist +from django.core.management.base import BaseCommand from django.utils.translation import gettext as _ -from onadata.apps.api.models.team import Team +from guardian.shortcuts import assign_perm, get_perms_for_model + from onadata.apps.api.models.organization_profile import OrganizationProfile +from onadata.apps.api.models.team import Team from onadata.libs.utils.model_tools import queryset_iterator -from guardian.shortcuts import assign_perm, get_perms_for_model - class Command(BaseCommand): - args = '' - help = _(u"Assign permission to the member team") + """Assign permission to the member team""" + + args = "" + help = _("Assign permission to the member team") def handle(self, *args, **options): - self.stdout.write("Assign permission to the member team", ending='\n') + self.stdout.write("Assign permission to the member team", ending="\n") count = 0 fail = 0 @@ -24,38 +30,41 @@ def handle(self, *args, **options): org_name = args[0] org = OrganizationProfile.objects.get(user__username=org_name) - team = Team.objects.get(organization=org.user, - name=u'%s#%s' % ( - org.user.username, - 'members')) + team = Team.objects.get( + organization=org.user, name=f"{org.user.username}#members" % ("") + ) self.assign_perm(team, org) count += 1 total += 1 except ObjectDoesNotExist as e: fail += 1 - self.stdout.write(str(e), ending='\n') + self.stdout.write(str(e), ending="\n") else: # Get all the teams for team in queryset_iterator( - Team.objects.filter(name__contains='members')): + Team.objects.filter(name__contains="members") + ): self.assign_perm(team, team.organization) count += 1 total += 1 - self.stdout.write("Assigned {} of {} records. failed: {}". - format(count, total, fail), ending='\n') + self.stdout.write( + f"Assigned {count} of {total} records. failed: {fail}", ending="\n" + ) def assign_perm(self, team, org): + """Assign a team org permissions""" for perm in get_perms_for_model(Team): - org = org.user \ - if isinstance(org, OrganizationProfile) else org + org = org.user if isinstance(org, OrganizationProfile) else org assign_perm(perm.codename, org, team) if team.created_by: assign_perm(perm.codename, team.created_by, team) - if hasattr(org.profile, 'creator') and \ - org.profile.creator != team.created_by: - assign_perm(perm.codename, org.profile.creator, team) + if ( + hasattr(org.profile, "creator") + and org.profile.creator != team.created_by + ): + assign_perm(perm.codename, org.profile.creator, team) if org.profile.created_by != team.created_by: assign_perm(perm.codename, org.profile.created_by, team) diff --git a/onadata/apps/api/management/commands/cleanup_permissions.py b/onadata/apps/api/management/commands/cleanup_permissions.py index 8b34b59953..29ff362e3e 100644 --- a/onadata/apps/api/management/commands/cleanup_permissions.py +++ b/onadata/apps/api/management/commands/cleanup_permissions.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Cleanup permissions +""" from django.core.management.base import BaseCommand from django.utils.translation import gettext as _ from guardian.models import UserObjectPermission @@ -7,32 +11,28 @@ class Command(BaseCommand): - help = _(u"Cleanup permissions") + """Cleanup permissions""" + + help = _("Cleanup permissions") def handle(self, *args, **options): deleted = 0 self.stdout.write("Starting UserObject") - for perm in queryset_iterator( - UserObjectPermission.objects.select_related()): + for perm in queryset_iterator(UserObjectPermission.objects.select_related()): try: perm.content_object except AttributeError: perm.delete() deleted += 1 - self.stdout.write( - "deleted {} stale permission".format(deleted) - ) + self.stdout.write(f"deleted {deleted} stale permission") self.stdout.write("Starting GroupObject") - for perm in queryset_iterator( - GroupObjectPermission.objects.select_related()): + for perm in queryset_iterator(GroupObjectPermission.objects.select_related()): try: perm.content_object except AttributeError: perm.delete() deleted += 1 - self.stdout.write( - "deleted {} stale permission".format(deleted) - ) + self.stdout.write(f"deleted {deleted} stale permission") self.stdout.write( - "Total removed orphan object permissions instances: %d" % deleted + f"Total removed orphan object permissions instances: {deleted}" ) diff --git a/onadata/apps/api/management/commands/create_default_project.py b/onadata/apps/api/management/commands/create_default_project.py index 58ffbf029c..d9000ac093 100644 --- a/onadata/apps/api/management/commands/create_default_project.py +++ b/onadata/apps/api/management/commands/create_default_project.py @@ -1,18 +1,26 @@ +# -*- coding: utf-8 -*- +""" +Check for forms not in a project and move them to the default project +""" from django.conf import settings +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand -from django.contrib.auth.models import User from django.utils.translation import gettext as _ from onadata.apps.logger.models.project import Project from onadata.libs.utils.model_tools import queryset_iterator +User = get_user_model() XFORM_DEFAULT_PROJECT_ID = 1 class Command(BaseCommand): - help = _(u"Check for forms not in a project" - u" and move them to the default project") + """ + Check for forms not in a project and move them to the default project + """ + + help = _("Check for forms not in a project and move them to the default project") def handle(self, *args, **options): self.stdout.write("Task started ...") @@ -27,16 +35,19 @@ def handle(self, *args, **options): self.stdout.write("Task completed ...") def set_project_to_user_forms(self, user): - default_project_name = user.username + '\'s Project' + """Set default project for all user forms.""" + default_project_name = user.username + "'s Project" try: project = Project.objects.get(name=default_project_name) except Project.DoesNotExist: - metadata = {'description': 'Default Project'} - project = Project.objects.create(name=default_project_name, - organization=user, - created_by=user, - metadata=metadata) - self.stdout.write("Created project %s" % project.name) + metadata = {"description": "Default Project"} + project = Project.objects.create( + name=default_project_name, + organization=user, + created_by=user, + metadata=metadata, + ) + self.stdout.write(f"Created project {project.name}") finally: xforms = user.xforms.filter(project=XFORM_DEFAULT_PROJECT_ID) diff --git a/onadata/apps/api/management/commands/create_user_profiles.py b/onadata/apps/api/management/commands/create_user_profiles.py index bc89b3618c..b1df518a51 100644 --- a/onadata/apps/api/management/commands/create_user_profiles.py +++ b/onadata/apps/api/management/commands/create_user_profiles.py @@ -1,15 +1,18 @@ # -*- coding: utf-8 -*- """Management Command to add missing user profiles to users.""" +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand -from django.contrib.auth.models import User from django.utils.translation import gettext as _ from onadata.apps.main.models.user_profile import UserProfile from onadata.libs.utils.model_tools import queryset_iterator +User = get_user_model() + class Command(BaseCommand): """Create missing user profiles management command.""" + help = _("Build out missing user profiles") def handle(self, *args, **options): diff --git a/onadata/apps/api/management/commands/delete_users.py b/onadata/apps/api/management/commands/delete_users.py index c30e467883..6339f27e77 100644 --- a/onadata/apps/api/management/commands/delete_users.py +++ b/onadata/apps/api/management/commands/delete_users.py @@ -1,89 +1,80 @@ +# -*- coding: utf-8 -*- """ Delete users management command. """ import sys -from django.contrib.auth.models import User -from onadata.apps.logger.models import XForm -from onadata.apps.logger.models import Instance -from onadata.apps.logger.models import Project + +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError from django.utils import timezone +from onadata.apps.logger.models import Instance, Project, XForm + +User = get_user_model() -def get_user_object_stats( - username): # pylint: disable=R0201 + +def get_user_object_stats(username): """ Get User information. """ # Get the number of projects for this user - user_projects = Project.objects.filter( - created_by__username=username).count() + user_projects = Project.objects.filter(created_by__username=username).count() # Get the number of forms - user_forms = XForm.objects.filter( - user__username=username).count() + user_forms = XForm.objects.filter(user__username=username).count() # Get the number of submissions - user_sumbissions = Instance.objects.filter( - user__username=username).count() + user_sumbissions = Instance.objects.filter(user__username=username).count() user_response = input( - "User account '{}' has {} projects, " - "{} forms and {} submissions. " + f"User account '{username}' has {user_projects} projects, " + f"{user_forms} forms and {user_sumbissions} submissions. " "Do you wish to continue " - "deleting this account?".format( - username, - user_projects, - user_forms, - user_sumbissions - )) + "deleting this account?" + ) return user_response def inactivate_users(users, user_input): """ - Soft deletes the user termporarily. + Soft deletes the user. """ if users: for user in users: - username, email = user.split(':') - user_response = 'True' - if user_input == 'False': + username, email = user.split(":") + user_response = "True" + if user_input == "False": # If the --user_input flag is not provided. # Get acknowledgement from the user on this user_response = get_user_object_stats(username) - if user_response == 'True': + if user_response == "True": try: user = User.objects.get(username=username, email=email) # set inactive status on user account user.is_active = False # append a timestamped suffix to the username # to make the initial username available - deletion_suffix = timezone.now().strftime('-deleted-at-%s') + deletion_suffix = timezone.now().strftime("-deleted-at-%s") user.username += deletion_suffix user.email += deletion_suffix user.save() - sys.stdout.write( - 'User {} deleted successfully.'.format(username)) + sys.stdout.write(f"User {username} deleted successfully.") # confirm too that no user exists with provided email - if len(User.objects.filter( - email=email, is_active=True)) > 1: + if len(User.objects.filter(email=email, is_active=True)) > 1: other_accounts = [ - user.username for user in User.objects.filter( - email=email)] + user.username for user in User.objects.filter(email=email) + ] sys.stdout.write( - 'User accounts {} have the same ' - 'email address with this User'.format( - other_accounts)) + f"User accounts {other_accounts} have the same " + "email address with this User" + ) - except User.DoesNotExist: - raise CommandError( - 'User {} does not exist.'.format(username)) + except User.DoesNotExist as exc: + raise CommandError(f"User {username} does not exist.") from exc else: - sys.stdout.write( - 'No actions taken') + sys.stdout.write("No actions taken") else: - raise CommandError('No User Account provided!') + raise CommandError("No User Account provided!") class Command(BaseCommand): @@ -101,18 +92,17 @@ class Command(BaseCommand): To change this, pass this in with the value True i.e --user_input True """ - help = 'Delete users' + + help = "Delete users" def add_arguments(self, parser): - parser.add_argument('--user_details', nargs='*') + parser.add_argument("--user_details", nargs="*") parser.add_argument( - '--user_input', - help='Confirm deletion of user account', - default='False' + "--user_input", help="Confirm deletion of user account", default="False" ) def handle(self, *args, **kwargs): - users = kwargs.get('user_details') - user_input = kwargs.get('user_input') + users = kwargs.get("user_details") + user_input = kwargs.get("user_input") inactivate_users(users, user_input) diff --git a/onadata/apps/api/management/commands/fix_readonly_role_perms.py b/onadata/apps/api/management/commands/fix_readonly_role_perms.py index 4e2eab79c4..c0e8692c35 100644 --- a/onadata/apps/api/management/commands/fix_readonly_role_perms.py +++ b/onadata/apps/api/management/commands/fix_readonly_role_perms.py @@ -2,29 +2,27 @@ """ fix_readonly_role_perms - Reassign permission to the model when permissions are changed """ -from guardian.shortcuts import get_perms - -from django.core.management.base import BaseCommand, CommandError +from django.conf import settings from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand, CommandError from django.utils.translation import gettext as _ -from django.conf import settings -from onadata.apps.api.models import Team +from guardian.shortcuts import get_perms +from onadata.apps.api.models import Team from onadata.libs.permissions import ( - ReadOnlyRole, + DataEntryMinorRole, + DataEntryOnlyRole, DataEntryRole, + EditorMinorRole, EditorRole, ManagerRole, OwnerRole, + ReadOnlyRole, ReadOnlyRoleNoDownload, - DataEntryOnlyRole, - DataEntryMinorRole, - EditorMinorRole, ) from onadata.libs.utils.model_tools import queryset_iterator - # pylint: disable=invalid-name User = get_user_model() @@ -83,7 +81,7 @@ def reassign_perms(user, model, new_perm): for perm_obj in objects: obj = perm_obj.content_object - ROLES = [ + roles = [ ReadOnlyRoleNoDownload, ReadOnlyRole, DataEntryOnlyRole, @@ -96,7 +94,7 @@ def reassign_perms(user, model, new_perm): ] # For each role reassign the perms - for role_class in reversed(ROLES): + for role_class in reversed(roles): not_readonly = role_class.user_has_role(user, obj) or role_class not in [ ReadOnlyRoleNoDownload, ReadOnlyRole, diff --git a/onadata/apps/api/management/commands/migrate_group_permissions.py b/onadata/apps/api/management/commands/migrate_group_permissions.py index 5bdc24d8ab..90f1478e8a 100644 --- a/onadata/apps/api/management/commands/migrate_group_permissions.py +++ b/onadata/apps/api/management/commands/migrate_group_permissions.py @@ -1,9 +1,16 @@ +# -*- coding: utf-8 -*- +""" +Migrate group permissions +""" +import sys + +from django.apps import apps from django.contrib.contenttypes.models import ContentType from django.core.management.base import BaseCommand from django.db.models import Count -from django.db.models.loading import get_model from django.db.utils import IntegrityError from django.utils.translation import gettext as _ + from guardian.models import GroupObjectPermissionBase from onadata.apps.api.models import Team @@ -11,70 +18,83 @@ class Command(BaseCommand): - help = _(u"Migrate group permissions") + """Migrate group permissions""" + + help = _("Migrate group permissions") def add_arguments(self, parser): parser.add_argument( - '--model', - '-m', - action='store_true', - dest='app_model', + "--model", + "-m", + action="store_true", + dest="app_model", default=False, - help='The model the permission belong too.' - ' (app.model format)') + help="The model the permission belong too." " (app.model format)", + ) parser.add_argument( - '--perm-table', - '-p', - action='store_true', - dest='perms_tbl', + "--perm-table", + "-p", + action="store_true", + dest="perms_tbl", default=False, - help='The new model permission are stored in' - ' (app.model format)') + help="The new model permission are stored in" " (app.model format)", + ) def handle(self, *args, **options): - self.stdout.write("Migrate group permissions started", ending='\n') + self.stdout.write("Migrate group permissions started", ending="\n") if len(args) < 2: - self.stdout.write("This command takes two argument -m and -p " - "Example: " - "-m logger.Team " - "-p logger.TeamUserObjectPermission") - exit() + self.stdout.write( + "This command takes two argument -m and -p " + "Example: " + "-m logger.Team " + "-p logger.TeamUserObjectPermission" + ) + sys.exit() - if options['app_model']: + if options["app_model"]: app_model = args[0] else: self.stdout.write("-m , should be set as the first argument") - exit() + sys.exit() - if options['perms_tbl']: + if options["perms_tbl"]: perms_tbl = args[1] else: self.stdout.write("-p , should be set as the second argument") - exit() + sys.exit() - model = get_model(app_model) - perms_model = get_model(perms_tbl) + model = apps.get_model(app_model) + perms_model = apps.get_model(perms_tbl) if not issubclass(perms_model, GroupObjectPermissionBase): - self.stdout.write("-p , should be a model of a class that is " - "a subclass of GroupObjectPermissionBase") - exit() + self.stdout.write( + "-p , should be a model of a class that is " + "a subclass of GroupObjectPermissionBase" + ) + sys.exit() - ct = ContentType.objects.get( - model=model.__name__.lower(), app_label=model._meta.app_label) - teams = Team.objects.filter().annotate( - c=Count('groupobjectpermission')).filter(c__gt=0) + content_type = ContentType.objects.get( + model=model.__name__.lower(), app_label=model._meta.app_label + ) + teams = ( + Team.objects.filter() + .annotate(c=Count("groupobjectpermission")) + .filter(c__gt=0) + ) for team in queryset_iterator(teams): - self.stdout.write("Processing: {} - {}".format(team.pk, team.name)) - for gop in team.groupobjectpermission_set.filter(content_type=ct)\ - .select_related('permission', 'content_type')\ - .prefetch_related('permission', 'content_type'): + self.stdout.write(f"Processing: {team.pk} - {team.name}") + for gop in ( + team.groupobjectpermission_set.filter(content_type=content_type) + .select_related("permission", "content_type") + .prefetch_related("permission", "content_type") + ): try: perms_model( content_object=gop.content_object, group=team, - permission=gop.permission).save() + permission=gop.permission, + ).save() except IntegrityError: continue except ValueError: diff --git a/onadata/apps/api/management/commands/migrate_permissions.py b/onadata/apps/api/management/commands/migrate_permissions.py index e9e7993456..5cdfcd08ce 100644 --- a/onadata/apps/api/management/commands/migrate_permissions.py +++ b/onadata/apps/api/management/commands/migrate_permissions.py @@ -1,85 +1,103 @@ +# -*- coding: utf-8 -*- +""" +Migrate permissions +""" +import sys + +from django.apps import apps from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType from django.core.management.base import BaseCommand -from django.db.models.loading import get_model from django.db.utils import IntegrityError from django.utils.translation import gettext as _ + from guardian.models import UserObjectPermissionBase from onadata.libs.utils.model_tools import queryset_iterator +User = get_user_model() + class Command(BaseCommand): - help = _(u"Migrate permissions") + """Migrate permissions""" + + help = _("Migrate permissions") def add_arguments(self, parser): parser.add_argument( - '--model', - '-m', - action='store_true', - dest='app_model', + "--model", + "-m", + action="store_true", + dest="app_model", default=False, - help='The model the permission belong too.' - ' (app.model format)') + help="The model the permission belong too." " (app.model format)", + ) parser.add_argument( - '--perm-table', - '-p', - action='store_true', - dest='perms_tbl', + "--perm-table", + "-p", + action="store_true", + dest="perms_tbl", default=False, - help='The new model permission are stored in' - ' (app.model format)') + help="The new model permission are stored in" " (app.model format)", + ) def handle(self, *args, **options): - self.stdout.write("Migrate permissions started", ending='\n') + self.stdout.write("Migrate permissions started", ending="\n") if len(args) < 2: - self.stdout.write("This command takes two argument -m and -p " - "Example: " - "-m logger.Team " - "-p logger.TeamUserObjectPermission") - exit() + self.stdout.write( + "This command takes two argument -m and -p " + "Example: " + "-m logger.Team " + "-p logger.TeamUserObjectPermission" + ) + sys.exit() - if options['app_model']: + if options["app_model"]: app_model = args[0] else: self.stdout.write("-m , should be set as the first argument") - exit() + sys.exit() - if options['perms_tbl']: + if options["perms_tbl"]: perms_tbl = args[1] else: self.stdout.write("-p , should be set as the second argument") - exit() + sys.exit() - model = get_model(app_model) - perms_model = get_model(perms_tbl) + model = apps.get_model(app_model) + perms_model = apps.get_model(perms_tbl) if not issubclass(perms_model, UserObjectPermissionBase): - self.stdout.write("-p , should be a model of a class that is " - "a subclass of UserObjectPermissionBase") - exit() + self.stdout.write( + "-p , should be a model of a class that is " + "a subclass of UserObjectPermissionBase" + ) + sys.exit() - ct = ContentType.objects.get( - model=model.__name__.lower(), app_label=model._meta.app_label) + content_type = ContentType.objects.get( + model=model.__name__.lower(), app_label=model._meta.app_label + ) # Get all the users users = User.objects.exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME).order_by( - 'username') + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ).order_by("username") for user in queryset_iterator(users): - self.stdout.write( - "Processing: {} - {}".format(user.pk, user.username)) - for uop in user.userobjectpermission_set.filter(content_type=ct)\ - .select_related('permission', 'content_type')\ - .prefetch_related('permission', 'content_type'): + self.stdout.write(f"Processing: {user.pk} - {user.username}") + for uop in ( + user.userobjectpermission_set.filter(content_type=content_type) + .select_related("permission", "content_type") + .prefetch_related("permission", "content_type") + ): try: perms_model( content_object=uop.content_object, user=user, - permission=uop.permission).save() + permission=uop.permission, + ).save() except IntegrityError: continue except ValueError: diff --git a/onadata/apps/api/management/commands/print_profiler_results.py b/onadata/apps/api/management/commands/print_profiler_results.py deleted file mode 100644 index 09c4fc725c..0000000000 --- a/onadata/apps/api/management/commands/print_profiler_results.py +++ /dev/null @@ -1,13 +0,0 @@ -from hotshot import stats -from django.core.management.base import BaseCommand - - -class Command(BaseCommand): - args = '' - - def handle(self, *args, **options): - self.stdout.write("Show profiler log file output..", ending='\n') - - _stats = stats.load(args[0]) - _stats.sort_stats('time', 'calls') - _stats.print_stats(20) diff --git a/onadata/apps/api/management/commands/reassign_permission.py b/onadata/apps/api/management/commands/reassign_permission.py index 38ee347f10..065361e841 100644 --- a/onadata/apps/api/management/commands/reassign_permission.py +++ b/onadata/apps/api/management/commands/reassign_permission.py @@ -119,7 +119,6 @@ def reassign_perms(self, user, app, model, new_perm): role_class.add(user, obj) break - # pylint: disable=no-self-use def check_role(self, role_class, user, obj, new_perm=None): """ Test if the user has the role for the object provided diff --git a/onadata/apps/api/management/commands/regenerate_auth_tokens.py b/onadata/apps/api/management/commands/regenerate_auth_tokens.py index 16b7f992c6..38ebaf29ed 100644 --- a/onadata/apps/api/management/commands/regenerate_auth_tokens.py +++ b/onadata/apps/api/management/commands/regenerate_auth_tokens.py @@ -1,58 +1,73 @@ -from django.contrib.auth.models import User +# -*- coding: utf-8 -*- +""" +Regenerate Authentication Tokens +""" +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand from django.utils.translation import gettext as _ from rest_framework.authtoken.models import Token +ALL = "all" + +User = get_user_model() + class Command(BaseCommand): - help = _(u"Regenerate Authentication Tokens") + """Regenerate Authentication Tokens""" + + help = _("Regenerate Authentication Tokens") def add_arguments(self, parser): parser.add_argument( - '--users', - '-u', - action='store_true', - dest='users', + "--users", + "-u", + action="store_true", + dest="users", default=False, - help='Users to Regenerate Tokens') + help="Users to Regenerate Tokens", + ) def handle(self, *args, **options): - ALL = "all" # check if the users option has been used and # at least one argument passed - if options['users'] and len(args) > 0: + if options["users"] and len(args) > 0: # check if keyword 'all' has been included in the list of arguments if len(args) > 1 and ALL in args: - self.stdout.write("Keyword 'all' should be passed as single " - "argument and not be part of a list") + self.stdout.write( + "Keyword 'all' should be passed as single " + "argument and not be part of a list" + ) else: # check if the single argument passed in 'all' if len(args) == 1 and args[0] == ALL: Token.objects.all().delete() for user in User.objects.all(): Token.objects.create(user=user) - self.stdout.write("All users' api tokens have " - "been updated") + self.stdout.write("All users' api tokens have been updated") else: users = User.objects.filter(username__in=args) usernames = [a.username for a in users] # check if ALL usernames provided were valid if len(users) == 0: - self.stdout.write("The usernames provided were " - "invalid") + self.stdout.write("The usernames provided were invalid") else: # check some of the usernames passed were invalid if len(users) != len(args): users_not_found = list(set(args) - set(usernames)) - self.stdout.write("The following usernames don't " - "exist: %s" % users_not_found) + self.stdout.write( + "The following usernames don't exist: " + f"{users_not_found}" + ) for user in users: Token.objects.get(user=user).delete() Token.objects.create(user=user) - self.stdout.write("The API tokens for the users " - "provided have been updated") + self.stdout.write( + "The API tokens for the users provided have been updated" + ) else: - self.stdout.write("This command takes at least one argument with " - "the '--users' or '-u' option e.g -u ") + self.stdout.write( + "This command takes at least one argument with " + "the '--users' or '-u' option e.g -u " + ) diff --git a/onadata/apps/api/management/commands/retrieve_org_or_project_list.py b/onadata/apps/api/management/commands/retrieve_org_or_project_list.py index 5d992d2c93..cdd343d1a0 100644 --- a/onadata/apps/api/management/commands/retrieve_org_or_project_list.py +++ b/onadata/apps/api/management/commands/retrieve_org_or_project_list.py @@ -1,79 +1,92 @@ +# -*- coding: utf-8 -*- +""" +Retrieve collaborators list from all/a specific project(s) or organization(s) +""" import json -from django.core.management.base import ( - BaseCommand, CommandError, CommandParser) + +from django.core.management.base import BaseCommand, CommandError, CommandParser from django.utils.translation import gettext as _ -from onadata.apps.logger.models import Project from onadata.apps.api.models import OrganizationProfile -from onadata.libs.utils.project_utils import get_project_users +from onadata.apps.logger.models import Project from onadata.libs.utils.organization_utils import get_organization_members +from onadata.libs.utils.project_utils import get_project_users class Command(BaseCommand): + """ + Retrieve collaborators list from all/a specific project(s) or organization(s) + """ + help = _( "Retrieve collaborators list from all/a specific" - " project(s) or organization(s)") + " project(s) or organization(s)" + ) def add_arguments(self, parser: CommandParser): parser.add_argument( - '--project-ids', - '-p', + "--project-ids", + "-p", default=None, - dest='project_ids', - help='Comma separated list of project ID(s) to' - ' retrieve collaborators/members from.' + dest="project_ids", + help="Comma separated list of project ID(s) to" + " retrieve collaborators/members from.", ) parser.add_argument( - '--organization-ids', - '-oid', + "--organization-ids", + "-oid", default=None, - dest='organization_ids', - help='Comma separated list of organization ID(s) to retrieve' - ' collaborators/members from.' + dest="organization_ids", + help="Comma separated list of organization ID(s) to retrieve" + " collaborators/members from.", ) parser.add_argument( - '--output-file', - '-o', - dest='output_file', + "--output-file", + "-o", + dest="output_file", default=None, - help='JSON file to output the collaborators/members list too' + help="JSON file to output the collaborators/members list too", ) + # pylint: disable=too-many-branches def handle(self, *args, **options): result = {} - project_ids = options.get('project_ids') - organization_ids = options.get('organization_ids') - output_file = options.get('output_file') + project_ids = options.get("project_ids") + organization_ids = options.get("organization_ids") + output_file = options.get("output_file") if project_ids or organization_ids: if project_ids: - project_ids = project_ids.split(',') + project_ids = project_ids.split(",") for project_id in project_ids: try: project = Project.objects.get(id=int(project_id)) - except Project.DoesNotExist: + except Project.DoesNotExist as exc: raise CommandError( - f'Project with ID {project_id} does not exist.') - except ValueError: + f"Project with ID {project_id} does not exist." + ) from exc + except ValueError as exc: raise CommandError( - f'Invalid project ID input "{project_id}"') + f'Invalid project ID input "{project_id}"' + ) from exc else: result[project.name] = get_project_users(project) if organization_ids: - organization_ids = organization_ids.split(',') + organization_ids = organization_ids.split(",") for org_id in organization_ids: try: - org = OrganizationProfile.objects.get( - id=int(org_id)) - except OrganizationProfile.DoesNotExist: + org = OrganizationProfile.objects.get(id=int(org_id)) + except OrganizationProfile.DoesNotExist as exc: raise CommandError( - f'Organization with ID {org_id} does not exist.') - except ValueError: + f"Organization with ID {org_id} does not exist." + ) from exc + except ValueError as exc: raise CommandError( - f'Invalid organization ID input "{org_id}"') + f'Invalid organization ID input "{org_id}"' + ) from exc else: result[org.name] = get_organization_members(org) else: @@ -81,12 +94,11 @@ def handle(self, *args, **options): for project in Project.objects.filter(deleted_at__isnull=True): result[project.name] = get_project_users(project) - for org in OrganizationProfile.objects.filter( - user__is_active=True): + for org in OrganizationProfile.objects.filter(user__is_active=True): result[org.name] = get_organization_members(org) if output_file: - with open(output_file, 'w+') as outfile: + with open(output_file, "w+", encoding="utf-8") as outfile: json.dump(result, outfile) self.stdout.write( f'Outputted members/collaborators list to "{output_file}"' diff --git a/onadata/apps/api/management/commands/set_api_permissions.py b/onadata/apps/api/management/commands/set_api_permissions.py index a8d6b7bf2a..dae524217f 100644 --- a/onadata/apps/api/management/commands/set_api_permissions.py +++ b/onadata/apps/api/management/commands/set_api_permissions.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Set object permissions for all objects. +""" from django.core.management.base import BaseCommand from django.utils.translation import gettext as _ @@ -11,7 +15,9 @@ class Command(BaseCommand): - help = _(u"Set object permissions for all objects.") + """Set object permissions for all objects.""" + + help = _("Set object permissions for all objects.") def handle(self, *args, **options): # XForms @@ -29,18 +35,15 @@ def handle(self, *args, **options): # OrganizationProfile for profile in queryset_iterator(OrganizationProfile.objects.all()): OwnerRole.add(profile.user, profile) - OwnerRole.add( - profile.user, profile.userprofile_ptr) + OwnerRole.add(profile.user, profile.userprofile_ptr) if profile.created_by is not None: OwnerRole.add(profile.created_by, profile) - OwnerRole.add( - profile.created_by, profile.userprofile_ptr) + OwnerRole.add(profile.created_by, profile.userprofile_ptr) if profile.creator is not None: OwnerRole.add(profile.creator, profile) - OwnerRole.add( - profile.creator, profile.userprofile_ptr) + OwnerRole.add(profile.creator, profile.userprofile_ptr) # Project for project in queryset_iterator(Project.objects.all()): diff --git a/onadata/apps/api/models/odk_token.py b/onadata/apps/api/models/odk_token.py index 2d86886f7c..261daea331 100644 --- a/onadata/apps/api/models/odk_token.py +++ b/onadata/apps/api/models/odk_token.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ ODK token model module """ @@ -6,17 +7,17 @@ from datetime import timedelta from django.conf import settings +from django.contrib.auth import get_user_model from django.db import models from django.db.models.signals import post_save from django.utils.translation import gettext_lazy as _ from cryptography.fernet import Fernet -from django_digest.models import (_persist_partial_digests, - _prepare_partial_digests) +from django_digest.models import _persist_partial_digests, _prepare_partial_digests -AUTH_USER_MODEL = getattr(settings, 'AUTH_USER_MODEL', 'auth.User') -ODK_TOKEN_LENGTH = getattr(settings, 'ODK_TOKEN_LENGTH', 7) -ODK_TOKEN_FERNET_KEY = getattr(settings, 'ODK_TOKEN_FERNET_KEY', '') +AUTH_USER_MODEL = get_user_model() +ODK_TOKEN_LENGTH = getattr(settings, "ODK_TOKEN_LENGTH", 7) +ODK_TOKEN_FERNET_KEY = getattr(settings, "ODK_TOKEN_FERNET_KEY", "") ODK_TOKEN_LIFETIME = getattr(settings, "ODK_KEY_LIFETIME", 7) @@ -24,26 +25,21 @@ class ODKToken(models.Model): """ ODK Token class """ - ACTIVE = '1' - INACTIVE = '2' - STATUS_CHOICES = ( - (ACTIVE, _('Active')), - (INACTIVE, _('Inactive')) - ) + + ACTIVE = "1" + INACTIVE = "2" + STATUS_CHOICES = ((ACTIVE, _("Active")), (INACTIVE, _("Inactive"))) key = models.CharField(max_length=150, primary_key=True) - user = models.ForeignKey( - AUTH_USER_MODEL, on_delete=models.CASCADE) + user = models.ForeignKey(AUTH_USER_MODEL, on_delete=models.CASCADE) status = models.CharField( - 'Status', - choices=STATUS_CHOICES, - default=ACTIVE, - max_length=1) + "Status", choices=STATUS_CHOICES, default=ACTIVE, max_length=1 + ) created = models.DateTimeField(auto_now_add=True) expires = models.DateTimeField(blank=True, null=True) class Meta: - app_label = 'api' + app_label = "api" def _generate_partial_digest(self, raw_key): """ @@ -61,10 +57,14 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ if not self.key: self.key = self.generate_key() - return super(ODKToken, self).save(*args, **kwargs) + return super().save(*args, **kwargs) def generate_key(self): - key = binascii.hexlify(os.urandom(ODK_TOKEN_LENGTH)).decode('utf-8') + """ + Generates and returns ODK Token key encrypted with the Fernet cryptography + scheme. + """ + key = binascii.hexlify(os.urandom(ODK_TOKEN_LENGTH)).decode("utf-8") self._generate_partial_digest(key) return _encrypt_key(key) @@ -77,7 +77,7 @@ def raw_key(self): Decrypts the key and returns it in its Raw Form """ fernet = Fernet(ODK_TOKEN_FERNET_KEY) - return fernet.decrypt(self.key.encode('utf-8')) + return fernet.decrypt(self.key.encode("utf-8")) def _encrypt_key(raw_key): @@ -86,23 +86,23 @@ def _encrypt_key(raw_key): the fernet cryptography scheme """ fernet = Fernet(ODK_TOKEN_FERNET_KEY) - return fernet.encrypt(raw_key.encode('utf-8')).decode('utf-8') + return fernet.encrypt(raw_key.encode("utf-8")).decode("utf-8") +# pylint: disable=unused-argument def _post_save_persist_partial_digests(sender, instance=None, **kwargs): if instance: _persist_partial_digests(instance.user) +# pylint: disable=unused-argument def _post_save_set_expiry_date(sender, instance=None, **kwargs): if instance and not instance.expires: expiry_date = instance.created + timedelta(days=ODK_TOKEN_LIFETIME) - instance.expires = expiry_date.astimezone( - instance.created.tzinfo) + instance.expires = expiry_date.astimezone(instance.created.tzinfo) instance.save() -post_save.connect( - _post_save_persist_partial_digests, sender=ODKToken) +post_save.connect(_post_save_persist_partial_digests, sender=ODKToken) post_save.connect(_post_save_set_expiry_date, sender=ODKToken) diff --git a/onadata/apps/api/models/organization_profile.py b/onadata/apps/api/models/organization_profile.py index d0dd1a3dad..cd6625bebf 100644 --- a/onadata/apps/api/models/organization_profile.py +++ b/onadata/apps/api/models/organization_profile.py @@ -2,17 +2,22 @@ """ OrganizationProfile module. """ -from django.contrib.auth.models import Permission, User +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Permission from django.contrib.contenttypes.models import ContentType from django.db import models from django.db.models.signals import post_delete, post_save from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from guardian.shortcuts import assign_perm, get_perms_for_model +from multidb.pinning import use_master from onadata.apps.api.models.team import Team -from onadata.apps.main.models import UserProfile +from onadata.apps.main.models.user_profile import UserProfile from onadata.libs.utils.cache_tools import IS_ORG, safe_delete +from onadata.libs.utils.common_tags import MEMBERS + +User = get_user_model() # pylint: disable=invalid-name,unused-argument @@ -22,7 +27,7 @@ def org_profile_post_delete_callback(sender, instance, **kwargs): """ # delete the org_user too instance.user.delete() - safe_delete("{}{}".format(IS_ORG, instance.pk)) + safe_delete(f"{IS_ORG}{instance.pk}") def create_owner_team_and_assign_permissions(org): @@ -37,7 +42,7 @@ def create_owner_team_and_assign_permissions(org): # pylint: disable=unpacking-non-sequence permission, _ = Permission.objects.get_or_create( codename="is_org_owner", name="Organization Owner", content_type=content_type - ) # pylint: disable= + ) team.permissions.add(permission) org.creator.groups.add(team) @@ -63,6 +68,82 @@ def create_owner_team_and_assign_permissions(org): return team +# pylint: disable=invalid-name +def get_or_create_organization_owners_team(org): + """ + Get the owners team of an organization + :param org: organization + :return: Owners team of the organization + """ + team_name = f"{org.user.username}#{Team.OWNER_TEAM_NAME}" + try: + team = Team.objects.get(name=team_name, organization=org.user) + except Team.DoesNotExist: + with use_master: + queryset = Team.objects.filter(name=team_name, organization=org.user) + if queryset.count() > 0: + return queryset.first() # pylint: disable=no-member + return create_owner_team_and_assign_permissions(org) + return team + + +def add_user_to_team(team, user): + """ + Adds a user to a team and assigns them team permissions. + """ + user.groups.add(team) + + # give the user perms to view the team + assign_perm("view_team", user, team) + + # if team is owners team assign more perms + if team.name.find(Team.OWNER_TEAM_NAME) > 0: + _assign_organization_team_perms(team.organization, user) + + +def _assign_organization_team_perms(organization, user): + owners_team = get_or_create_organization_owners_team(organization.profile) + members_team = get_organization_members_team(organization.profile) + for perm in get_perms_for_model(Team): + assign_perm(perm.codename, user, owners_team) + assign_perm(perm.codename, user, members_team) + + +def create_organization_team(organization, name, permission_names=None): + """ + Creates an organization team with the given permissions as defined in + permission_names. + """ + organization = ( + organization.user + if isinstance(organization, OrganizationProfile) + else organization + ) + team = Team.objects.create(organization=organization, name=name) + content_type = ContentType.objects.get(app_label="api", model="organizationprofile") + if permission_names: + # get permission objects + perms = Permission.objects.filter( + codename__in=permission_names, content_type=content_type + ) + if perms: + team.permissions.add(*tuple(perms)) + return team + + +def get_organization_members_team(organization): + """Get organization members team + create members team if it does not exist and add organization owner + to the members team""" + try: + team = Team.objects.get(name=f"{organization.user.username}#{MEMBERS}") + except Team.DoesNotExist: + team = create_organization_team(organization, MEMBERS) + add_user_to_team(team, organization.user) + + return team + + def _post_save_create_owner_team(sender, instance, created, **kwargs): """ Signal handler that creates the Owner team and assigns group and user @@ -97,17 +178,17 @@ class Meta: creator = models.ForeignKey(User, on_delete=models.CASCADE) def __str__(self): - return "%s[%s]" % (self.name, self.user.username) + return f"{self.name}[{self.user.username}]" def save(self, *args, **kwargs): # pylint: disable=arguments-differ - super(OrganizationProfile, self).save(*args, **kwargs) + super().save(*args, **kwargs) def remove_user_from_organization(self, user): """Removes a user from all teams/groups in the organization. :param user: The user to remove from this organization. """ - for group in user.groups.filter("%s#" % self.user.username): + for group in user.groups.filter(name=f"{self.user.username}#"): user.groups.remove(group) def is_organization_owner(self, user): @@ -118,9 +199,9 @@ def is_organization_owner(self, user): :returns: Boolean whether user has organization level permissions. """ has_owner_group = user.groups.filter( - name="%s#%s" % (self.user.username, Team.OWNER_TEAM_NAME) + name=f"{self.user.username}#{Team.OWNER_TEAM_NAME}" ) - return True if has_owner_group else False + return has_owner_group.count() > 0 post_save.connect( diff --git a/onadata/apps/api/models/team.py b/onadata/apps/api/models/team.py index ad4a8e21ce..0d80c728d6 100644 --- a/onadata/apps/api/models/team.py +++ b/onadata/apps/api/models/team.py @@ -45,7 +45,7 @@ def __str__(self): @property def team_name(self): """Return the team name.""" - return self.__str__() + return str(self) def save(self, *args, **kwargs): # allow use of same name in different organizations/users diff --git a/onadata/apps/api/models/temp_token.py b/onadata/apps/api/models/temp_token.py index c9ded3c28e..231ce75317 100644 --- a/onadata/apps/api/models/temp_token.py +++ b/onadata/apps/api/models/temp_token.py @@ -29,7 +29,7 @@ def save(self, *args, **kwargs): self.key = self.generate_key() return super().save(*args, **kwargs) - def generate_key(self): # pylint: disable=no-self-use + def generate_key(self): """Generates a token key.""" return binascii.hexlify(os.urandom(20)).decode() diff --git a/onadata/apps/api/permissions.py b/onadata/apps/api/permissions.py index 7d359427b9..8f7a8d3672 100644 --- a/onadata/apps/api/permissions.py +++ b/onadata/apps/api/permissions.py @@ -1,31 +1,42 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ API permissions module. """ -from django.contrib.auth.models import User +from django.conf import settings +from django.contrib.auth import get_user_model from django.http import Http404 from django.shortcuts import get_object_or_404 -from django.conf import settings from rest_framework import exceptions from rest_framework.permissions import ( - BasePermission, DjangoModelPermissionsOrAnonReadOnly, - DjangoObjectPermissions, IsAuthenticated) - -from onadata.apps.api.tools import (check_inherit_permission_from_project, - get_instance_xform_or_none, - get_user_profile_or_none) + BasePermission, + DjangoModelPermissionsOrAnonReadOnly, + DjangoObjectPermissions, + IsAuthenticated, +) + +from onadata.apps.api.tools import ( + check_inherit_permission_from_project, + get_instance_xform_or_none, + get_user_profile_or_none, +) from onadata.apps.logger.models import DataView, Instance, Project, XForm from onadata.apps.main.models.user_profile import UserProfile -from onadata.libs.permissions import (CAN_ADD_XFORM_TO_PROFILE, - CAN_CHANGE_XFORM, CAN_DELETE_SUBMISSION, - ReadOnlyRoleNoDownload, - OwnerRole, ManagerRole) +from onadata.libs.permissions import ( + CAN_ADD_XFORM_TO_PROFILE, + CAN_CHANGE_XFORM, + CAN_DELETE_SUBMISSION, + ManagerRole, + OwnerRole, + ReadOnlyRoleNoDownload, +) -SAFE_METHODS = ('GET', 'HEAD', 'OPTIONS') +SAFE_METHODS = ("GET", "HEAD", "OPTIONS") +User = get_user_model() -class AlternateHasObjectPermissionMixin(object): # pylint: disable=R0903 + +class AlternateHasObjectPermissionMixin: # pylint: disable=too-few-public-methods """ AlternateHasObjectPermissionMixin - checks if user has read permissions. """ @@ -43,7 +54,7 @@ def _has_object_permission(self, request, model_cls, user, obj): # to make another lookup. raise Http404 - read_perms = self.get_required_object_permissions('GET', model_cls) + read_perms = self.get_required_object_permissions("GET", model_cls) if not user.has_perms(read_perms, obj): raise Http404 @@ -58,58 +69,61 @@ class ViewDjangoObjectPermissions(DjangoObjectPermissions): View DjangoObjectPermissions - applies view_ permissions for GET requests. """ + perms_map = { - 'GET': ['%(app_label)s.view_%(model_name)s'], - 'OPTIONS': [], - 'HEAD': [], - 'POST': ['%(app_label)s.add_%(model_name)s'], - 'PUT': ['%(app_label)s.change_%(model_name)s'], - 'PATCH': ['%(app_label)s.change_%(model_name)s'], - 'DELETE': ['%(app_label)s.delete_%(model_name)s'], + "GET": ["%(app_label)s.view_%(model_name)s"], + "OPTIONS": [], + "HEAD": [], + "POST": ["%(app_label)s.add_%(model_name)s"], + "PUT": ["%(app_label)s.change_%(model_name)s"], + "PATCH": ["%(app_label)s.change_%(model_name)s"], + "DELETE": ["%(app_label)s.delete_%(model_name)s"], } -class ExportDjangoObjectPermission(AlternateHasObjectPermissionMixin, - ViewDjangoObjectPermissions): +class ExportDjangoObjectPermission( + AlternateHasObjectPermissionMixin, ViewDjangoObjectPermissions +): """ Export DjangoObjectPermission - checks XForm permissions for export permissions. """ + authenticated_users_only = False perms_map = { - 'GET': ['logger.view_xform'], - 'OPTIONS': [], - 'HEAD': [], - 'POST': ['logger.add_xform'], - 'PUT': ['logger.change_xform'], - 'PATCH': ['logger.change_xform'], - 'DELETE': ['logger.delete_xform'], + "GET": ["logger.view_xform"], + "OPTIONS": [], + "HEAD": [], + "POST": ["logger.add_xform"], + "PUT": ["logger.change_xform"], + "PATCH": ["logger.change_xform"], + "DELETE": ["logger.delete_xform"], } def has_permission(self, request, view): - is_authenticated = (request and request.user and - request.user.is_authenticated) + is_authenticated = request and request.user and request.user.is_authenticated if not is_authenticated: - view._ignore_model_permissions = True # pylint: disable=W0212 + view._ignore_model_permissions = True # pylint: disable=protected-access - if view.action == 'destroy' and is_authenticated: - return request.user.has_perms(['logger.delete_xform']) + if view.action == "destroy" and is_authenticated: + return request.user.has_perms(["logger.delete_xform"]) - return super(ExportDjangoObjectPermission, self).has_permission( - request, view) + return super().has_permission(request, view) def has_object_permission(self, request, view, obj): model_cls = XForm user = request.user - return (obj.xform.shared_data or obj.xform.project.shared) or\ - self._has_object_permission(request, model_cls, user, obj.xform) + return ( + obj.xform.shared_data or obj.xform.project.shared + ) or self._has_object_permission(request, model_cls, user, obj.xform) class DjangoObjectPermissionsAllowAnon(DjangoObjectPermissions): """ DjangoObjectPermissionsAllowAnon - allow anonymous access permission. """ + authenticated_users_only = False @@ -117,57 +131,58 @@ class XFormPermissions(DjangoObjectPermissions): """ XFormPermissions - custom permissions check on XForm viewset. """ + authenticated_users_only = False def has_permission(self, request, view): - owner = view.kwargs.get('owner') + owner = view.kwargs.get("owner") is_authenticated = request and request.user.is_authenticated - if 'pk' in view.kwargs: - check_inherit_permission_from_project(view.kwargs['pk'], - request.user) + if "pk" in view.kwargs: + check_inherit_permission_from_project(view.kwargs["pk"], request.user) - if is_authenticated and view.action == 'create': + if is_authenticated and view.action == "create": owner = owner or request.user.username - return request.user.has_perm(CAN_ADD_XFORM_TO_PROFILE, - get_user_profile_or_none(owner)) + return request.user.has_perm( + CAN_ADD_XFORM_TO_PROFILE, get_user_profile_or_none(owner) + ) - return super(XFormPermissions, self).has_permission(request, view) + return super().has_permission(request, view) def has_object_permission(self, request, view, obj): - if hasattr(obj, 'shared') and obj.shared and view.action == 'clone': + if hasattr(obj, "shared") and obj.shared and view.action == "clone": return obj - if request.method == 'DELETE' and view.action == 'labels': + if request.method == "DELETE" and view.action == "labels": user = request.user return user.has_perm(CAN_CHANGE_XFORM, obj) - if request.method == 'DELETE' and view.action == 'destroy': + if request.method == "DELETE" and view.action == "destroy": return request.user.has_perm(CAN_DELETE_SUBMISSION, obj) - return super(XFormPermissions, self).has_object_permission( - request, view, obj) + return super().has_object_permission(request, view, obj) class SubmissionReviewPermissions(XFormPermissions): """ Custom Permission Checks for SubmissionReviews """ + perms_map = { - 'GET': [], - 'OPTIONS': [], - 'HEAD': [], - 'POST': ['logger.add_xform'], - 'PUT': ['logger.change_xform'], - 'PATCH': ['logger.change_xform'], - 'DELETE': ['logger.delete_xform'], + "GET": [], + "OPTIONS": [], + "HEAD": [], + "POST": ["logger.add_xform"], + "PUT": ["logger.change_xform"], + "PATCH": ["logger.change_xform"], + "DELETE": ["logger.delete_xform"], } - def _check_is_admin_or_manager( # pylint: disable=no-self-use - self, user: User, xform: XForm) -> bool: - return OwnerRole.user_has_role( - user, xform) or ManagerRole.user_has_role(user, xform) + def _check_is_admin_or_manager(self, user: User, xform: XForm) -> bool: + return OwnerRole.user_has_role(user, xform) or ManagerRole.user_has_role( + user, xform + ) def has_permission(self, request, view): """ @@ -175,40 +190,43 @@ def has_permission(self, request, view): """ is_authenticated = request and request.user.is_authenticated - if is_authenticated and view.action == 'create': + if is_authenticated and view.action == "create": # Handle bulk create # if doing a bulk create we will fail the entire process if the # user lacks permissions for even one instance if isinstance(request.data, list): - instance_ids = list(set([_['instance'] for _ in request.data])) - instances = Instance.objects.filter( - id__in=instance_ids).only('xform').order_by().distinct() + instance_ids = list(set(_["instance"] for _ in request.data)) + instances = ( + Instance.objects.filter(id__in=instance_ids) + .only("xform") + .order_by() + .distinct() + ) for instance in instances: if not self._check_is_admin_or_manager( - request.user, instance.xform): + request.user, instance.xform + ): return False return True # everything is okay # Handle single create like normal - instance_id = request.data.get('instance') + instance_id = request.data.get("instance") xform = get_instance_xform_or_none(instance_id) return self._check_is_admin_or_manager(request.user, xform) - return super(SubmissionReviewPermissions, self).has_permission( - request, view) + return super().has_permission(request, view) def has_object_permission(self, request, view, obj): """ Custom has_object_permission method """ - if (request.method == 'DELETE' and view.action == 'destroy') or ( - request.method == 'PATCH' and view.action == 'partial_update'): - return self._check_is_admin_or_manager( - request.user, obj.instance.xform) + if (request.method == "DELETE" and view.action == "destroy") or ( + request.method == "PATCH" and view.action == "partial_update" + ): + return self._check_is_admin_or_manager(request.user, obj.instance.xform) - return super(SubmissionReviewPermissions, self).has_object_permission( - request, view, obj) + return super().has_object_permission(request, view, obj) class UserProfilePermissions(DjangoObjectPermissions): @@ -220,22 +238,20 @@ class UserProfilePermissions(DjangoObjectPermissions): def has_permission(self, request, view): # allow anonymous users to create new profiles - if request.user.is_anonymous and view.action == 'create': + if request.user.is_anonymous and view.action == "create": return True - if view.action in ['send_verification_email', 'verify_email']: + if view.action in ["send_verification_email", "verify_email"]: enable_email_verification = getattr( - settings, 'ENABLE_EMAIL_VERIFICATION', False + settings, "ENABLE_EMAIL_VERIFICATION", False ) - if enable_email_verification is None or\ - not enable_email_verification: + if enable_email_verification is None or not enable_email_verification: return False - if view.action == 'send_verification_email': - return request.user.username == request.data.get('username') + if view.action == "send_verification_email": + return request.user.username == request.data.get("username") - return \ - super(UserProfilePermissions, self).has_permission(request, view) + return super().has_permission(request, view) class ProjectPermissions(DjangoObjectPermissions): @@ -247,27 +263,25 @@ class ProjectPermissions(DjangoObjectPermissions): def has_permission(self, request, view): # allow anonymous users to view public projects - if request.user.is_anonymous and view.action == 'list': + if request.user.is_anonymous and view.action == "list": return True - if not request.user.is_anonymous and view.action == 'star': + if not request.user.is_anonymous and view.action == "star": return True - return \ - super(ProjectPermissions, self).has_permission(request, view) + return super().has_permission(request, view) def has_object_permission(self, request, view, obj): - if view.action == 'share' and request.method == 'PUT': - remove = request.data.get('remove') - username = request.data.get('username', '') + if view.action == "share" and request.method == "PUT": + remove = request.data.get("remove") + username = request.data.get("username", "") if remove and request.user.username.lower() == username.lower(): return True - return super(ProjectPermissions, self).has_object_permission( - request, view, obj) + return super().has_object_permission(request, view, obj) -class AbstractHasPermissionMixin(object): # pylint: disable=R0903 +class AbstractHasPermissionMixin: # pylint: disable=too-few-public-methods """ Checks that the requesting user has permissions to access each of the models in the `model_classes` instance variable. @@ -280,24 +294,25 @@ def has_permission(self, request, view): # Workaround to ensure DjangoModelPermissions are not applied # to the root view when using DefaultRouter. - if getattr(view, '_ignore_model_permissions', False): + if getattr(view, "_ignore_model_permissions", False): return True perms = [] for model_class in self.model_classes: - perms.extend( - self.get_required_permissions(request.method, model_class)) + perms.extend(self.get_required_permissions(request.method, model_class)) - if (request.user and (request.user.is_authenticated - or not self.authenticated_users_only) - and request.user.has_perms(perms)): + if ( + request.user + and (request.user.is_authenticated or not self.authenticated_users_only) + and request.user.has_perms(perms) + ): return True return False -# pylint: disable=R0903 +# pylint: disable=too-few-public-methods class HasMetadataPermissionMixin(AbstractHasPermissionMixin): """ Use the Project, XForm, or both model classes to check permissions based @@ -312,13 +327,14 @@ def has_permission(self, request, view): else: self.model_classes = [Project, XForm] - return super(HasMetadataPermissionMixin, self).has_permission( - request, view) + return super().has_permission(request, view) -class MetaDataObjectPermissions(AlternateHasObjectPermissionMixin, - HasMetadataPermissionMixin, - DjangoObjectPermissions): +class MetaDataObjectPermissions( + AlternateHasObjectPermissionMixin, + HasMetadataPermissionMixin, + DjangoObjectPermissions, +): """ MetaData ObjectPermissions - apply Xform permision for given response. """ @@ -332,26 +348,25 @@ def has_object_permission(self, request, view, obj): model_cls = XForm xform_obj = obj.content_object.xform - return self._has_object_permission(request, model_cls, user, - xform_obj) + return self._has_object_permission(request, model_cls, user, xform_obj) - return self._has_object_permission(request, model_cls, user, - obj.content_object) + return self._has_object_permission(request, model_cls, user, obj.content_object) -class AttachmentObjectPermissions(AlternateHasObjectPermissionMixin, - DjangoObjectPermissions): +class AttachmentObjectPermissions( + AlternateHasObjectPermissionMixin, DjangoObjectPermissions +): """ Attachment ObjectPermissions - apply XForm model options. """ + authenticated_users_only = False def has_object_permission(self, request, view, obj): model_cls = XForm user = request.user - return self._has_object_permission(request, model_cls, user, - obj.instance.xform) + return self._has_object_permission(request, model_cls, user, obj.instance.xform) class ConnectViewsetPermissions(IsAuthenticated): @@ -360,11 +375,10 @@ class ConnectViewsetPermissions(IsAuthenticated): """ def has_permission(self, request, view): - if view.action == 'reset': + if view.action == "reset": return True - return super(ConnectViewsetPermissions, self)\ - .has_permission(request, view) + return super().has_permission(request, view) class UserViewSetPermissions(DjangoModelPermissionsOrAnonReadOnly): @@ -374,17 +388,19 @@ class UserViewSetPermissions(DjangoModelPermissionsOrAnonReadOnly): def has_permission(self, request, view): - if request.user.is_anonymous and view.action == 'list': - if request.GET.get('search'): + if request.user.is_anonymous and view.action == "list": + if request.GET.get("search"): raise exceptions.NotAuthenticated() - return \ - super(UserViewSetPermissions, self).has_permission(request, view) + return super().has_permission(request, view) class DataViewViewsetPermissions( - AlternateHasObjectPermissionMixin, ViewDjangoObjectPermissions, - AbstractHasPermissionMixin, DjangoObjectPermissions): + AlternateHasObjectPermissionMixin, + ViewDjangoObjectPermissions, + AbstractHasPermissionMixin, + DjangoObjectPermissions, +): """ DataView ViewSetPermissions - applies projet permissions to a filtered dataset. @@ -396,7 +412,7 @@ def has_permission(self, request, view): # To allow individual public dataviews to be visible on # `api/v1/dataviews/` but stop retreival of all dataviews when # the dataviews endpoint is queried `api/v1/dataviews` - return not (request.user.is_anonymous and view.action == 'list') + return not (request.user.is_anonymous and view.action == "list") def has_object_permission(self, request, view, obj): model_cls = Project @@ -404,13 +420,14 @@ def has_object_permission(self, request, view, obj): if obj.project.shared: return True - return self._has_object_permission(request, model_cls, user, - obj.project) + return self._has_object_permission(request, model_cls, user, obj.project) -class RestServiceObjectPermissions(AlternateHasObjectPermissionMixin, - HasMetadataPermissionMixin, - DjangoObjectPermissions): +class RestServiceObjectPermissions( + AlternateHasObjectPermissionMixin, + HasMetadataPermissionMixin, + DjangoObjectPermissions, +): """ RestService ObjectPermissions - apply XForm permisions for a RestService model. @@ -424,8 +441,11 @@ def has_object_permission(self, request, view, obj): class WidgetViewSetPermissions( - AlternateHasObjectPermissionMixin, ViewDjangoObjectPermissions, - AbstractHasPermissionMixin, DjangoObjectPermissions): + AlternateHasObjectPermissionMixin, + ViewDjangoObjectPermissions, + AbstractHasPermissionMixin, + DjangoObjectPermissions, +): """ Widget ViewSetPermissions - apply project permissions check. """ @@ -435,11 +455,10 @@ class WidgetViewSetPermissions( def has_permission(self, request, view): # User can access the widget with key - if 'key' in request.query_params or view.action == 'list': + if "key" in request.query_params or view.action == "list": return True - return super(WidgetViewSetPermissions, self).has_permission( - request, view) + return super().has_permission(request, view) def has_object_permission(self, request, view, obj): model_cls = Project @@ -448,16 +467,21 @@ def has_object_permission(self, request, view, obj): if not isinstance(obj.content_object, (XForm, DataView)): return False - xform = obj.content_object if isinstance(obj.content_object, XForm) \ + xform = ( + obj.content_object + if isinstance(obj.content_object, XForm) else obj.content_object.xform + ) - if view.action == 'partial_update' and \ - ReadOnlyRoleNoDownload.user_has_role(user, xform): + if view.action == "partial_update" and ReadOnlyRoleNoDownload.user_has_role( + user, xform + ): # allow readonlynodownload and above roles to edit widget return True - return self._has_object_permission(request, model_cls, user, - obj.content_object.project) + return self._has_object_permission( + request, model_cls, user, obj.content_object.project + ) __permissions__ = [DjangoObjectPermissions, IsAuthenticated] @@ -469,37 +493,36 @@ class OrganizationProfilePermissions(DjangoObjectPermissionsAllowAnon): """ def has_object_permission(self, request, view, obj): - is_authenticated = request and request.user.is_authenticated and \ - request.user.username == request.data.get( - 'username') - if is_authenticated and request.method == 'DELETE': + is_authenticated = ( + request + and request.user.is_authenticated + and request.user.username == request.data.get("username") + ) + if is_authenticated and request.method == "DELETE": return True - return super(OrganizationProfilePermissions, self)\ - .has_object_permission(request=request, view=view, obj=obj) + return super().has_object_permission(request=request, view=view, obj=obj) -class OpenDataViewSetPermissions(IsAuthenticated, - AlternateHasObjectPermissionMixin, - DjangoObjectPermissionsAllowAnon): +class OpenDataViewSetPermissions( + IsAuthenticated, AlternateHasObjectPermissionMixin, DjangoObjectPermissionsAllowAnon +): """ OpenDataViewSetPermissions - allow anonymous access to schema and data end-points of an open dataset. """ def has_permission(self, request, view): - if request.user.is_anonymous and view.action in ['schema', 'data']: + if request.user.is_anonymous and view.action in ["schema", "data"]: return True - return super(OpenDataViewSetPermissions, self).has_permission( - request, view) + return super().has_permission(request, view) def has_object_permission(self, request, view, obj): model_cls = XForm user = request.user - return self._has_object_permission(request, model_cls, user, - obj.content_object) + return self._has_object_permission(request, model_cls, user, obj.content_object) class IsAuthenticatedSubmission(BasePermission): @@ -509,9 +532,9 @@ class IsAuthenticatedSubmission(BasePermission): """ def has_permission(self, request, view): - username = view.kwargs.get('username') - form_pk = view.kwargs.get('xform_pk') - if request.method in ['HEAD', 'POST'] and request.user.is_anonymous: + username = view.kwargs.get("username") + form_pk = view.kwargs.get("xform_pk") + if request.method in ["HEAD", "POST"] and request.user.is_anonymous: if username: user = get_object_or_404(User, username__iexact=username) elif form_pk: diff --git a/onadata/apps/api/storage.py b/onadata/apps/api/storage.py index 5cf19e3d1a..f00a0c0e46 100644 --- a/onadata/apps/api/storage.py +++ b/onadata/apps/api/storage.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Storage module for the api app """ @@ -16,7 +17,7 @@ _l = logging.getLogger(__name__) _l.setLevel(logging.WARNING) -ODK_KEY_LIFETIME_IN_SEC = getattr(settings, 'ODK_KEY_LIFETIME', 7) * 86400 +ODK_KEY_LIFETIME_IN_SEC = getattr(settings, "ODK_KEY_LIFETIME", 7) * 86400 class ODKTokenAccountStorage(AccountStorage): @@ -27,6 +28,7 @@ class ODKTokenAccountStorage(AccountStorage): Digest Authentication set the DIGEST_ACCOUNT_BACKEND variable in your local_settings to 'onadata.apps.api.storage.ODKTokenAccountStorage' """ + GET_PARTIAL_DIGEST_QUERY = f""" SELECT django_digest_partialdigest.login, django_digest_partialdigest.partial_digest @@ -41,7 +43,7 @@ class ODKTokenAccountStorage(AccountStorage): AND api_odktoken.status='{ODKToken.ACTIVE}' """ - def get_partial_digest(self, login): + def get_partial_digest(self, username): """ Checks that the returned partial digest is associated with a Token that isn't past it's expire date. @@ -50,23 +52,23 @@ def get_partial_digest(self, login): its expiry date """ cursor = connection.cursor() - cursor.execute(self.GET_PARTIAL_DIGEST_QUERY, [login]) + cursor.execute(self.GET_PARTIAL_DIGEST_QUERY, [username]) # In MySQL, string comparison is case-insensitive by default. # Therefore a second round of filtering is required. - partial_digest = [(row[1]) for row in cursor.fetchall() - if row[0] == login] + partial_digest = [(row[1]) for row in cursor.fetchall() if row[0] == username] if not partial_digest: return None try: - token = ODKToken.objects.get(Q(user__username=login) - | Q(user__email=login), - status=ODKToken.ACTIVE) + token = ODKToken.objects.get( + Q(user__username=username) | Q(user__email=username), + status=ODKToken.ACTIVE, + ) except MultipleObjectsReturned: - _l.warn(f'User {login} has multiple ODK Tokens') + _l.error("User %s has multiple ODK Tokens", username) return None except ODKToken.DoesNotExist: - _l.warn(f'User {login} has no active ODK Token') + _l.error("User %s has no active ODK Token", username) return None else: if timezone.now() > token.expires: diff --git a/onadata/apps/api/tasks.py b/onadata/apps/api/tasks.py index 6f965e7943..d74cf87521 100644 --- a/onadata/apps/api/tasks.py +++ b/onadata/apps/api/tasks.py @@ -1,21 +1,28 @@ +# -*- coding: utf-8 -*- +""" +Celery api.tasks module. +""" import os import sys -from builtins import str from celery.result import AsyncResult from django.core.files.uploadedfile import TemporaryUploadedFile from django.core.files.storage import default_storage -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.utils.datastructures import MultiValueDict from onadata.apps.api import tools from onadata.libs.utils.email import send_generic_email from onadata.apps.logger.models.xform import XForm -from onadata.celery import app +from onadata.celeryapp import app + +User = get_user_model() def recreate_tmp_file(name, path, mime_type): + """Creates a TemporaryUploadedFile from a file path with given name""" tmp_file = TemporaryUploadedFile(name, mime_type, 0, None) + # pylint: disable=consider-using-with,unspecified-encoding tmp_file.file = open(path) tmp_file.size = os.fstat(tmp_file.fileno()).st_size return tmp_file @@ -23,6 +30,7 @@ def recreate_tmp_file(name, path, mime_type): @app.task(bind=True) def publish_xlsform_async(self, user_id, post_data, owner_id, file_data): + """Publishes an XLSForm""" try: files = MultiValueDict() files["xls_file"] = default_storage.open(file_data.get("path")) @@ -39,7 +47,7 @@ def publish_xlsform_async(self, user_id, post_data, owner_id, file_data): return {"pk": survey.pk} return survey - except Exception as exc: + except Exception as exc: # pylint: disable=broad-except if isinstance(exc, MemoryError): if self.request.retries < 3: self.retry(exc=exc, countdown=1) @@ -96,4 +104,5 @@ def send_verification_email(email, message_txt, subject): @app.task() def send_account_lockout_email(email, message_txt, subject): + """Sends account locked email.""" send_generic_email(email, message_txt, subject) diff --git a/onadata/apps/api/tests/mocked_data.py b/onadata/apps/api/tests/mocked_data.py index af76c1f064..fdc737a5ae 100644 --- a/onadata/apps/api/tests/mocked_data.py +++ b/onadata/apps/api/tests/mocked_data.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Contians mock functions used in some tests for example enketo urls and exports urls. diff --git a/onadata/apps/api/tests/models/test_team.py b/onadata/apps/api/tests/models/test_team.py index 771d642bf2..65408ffea8 100644 --- a/onadata/apps/api/tests/models/test_team.py +++ b/onadata/apps/api/tests/models/test_team.py @@ -1,46 +1,50 @@ from django.contrib.auth.models import Permission + from guardian.shortcuts import get_perms from onadata.apps.api import tools -from onadata.apps.logger.models.project import Project +from onadata.apps.api.models.organization_profile import create_organization_team from onadata.apps.api.models.team import Team -from onadata.apps.api.tests.models.test_abstract_models import ( - TestAbstractModels) +from onadata.apps.api.tests.models.test_abstract_models import TestAbstractModels +from onadata.apps.logger.models.project import Project from onadata.libs.permissions import ( - DataEntryRole, - CAN_VIEW_PROJECT, - CAN_ADD_XFORM, CAN_ADD_SUBMISSIONS_PROJECT, + CAN_ADD_XFORM, CAN_EXPORT_PROJECT, - CAN_VIEW_PROJECT_DATA, + CAN_VIEW_PROJECT, CAN_VIEW_PROJECT_ALL, - get_team_project_default_permissions) + CAN_VIEW_PROJECT_DATA, + DataEntryRole, + get_team_project_default_permissions, +) class TestTeam(TestAbstractModels): - def test_create_organization_team(self): profile = tools.create_organization_object("modilabs", self.user) profile.save() organization = profile.user - team_name = 'dev' - perms = ['is_org_owner', ] - tools.create_organization_team(organization, team_name, perms) + team_name = "dev" + perms = [ + "is_org_owner", + ] + create_organization_team(organization, team_name, perms) team_name = "modilabs#%s" % team_name dev_team = Team.objects.get(organization=organization, name=team_name) self.assertIsInstance(dev_team, Team) self.assertIsInstance( - dev_team.permissions.get(codename='is_org_owner'), Permission) + dev_team.permissions.get(codename="is_org_owner"), Permission + ) def test_assign_user_to_team(self): # create the organization organization = self._create_organization("modilabs", self.user) - user_deno = self._create_user('deno', 'deno') + user_deno = self._create_user("deno", "deno") # create another team - team_name = 'managers' - team = tools.create_organization_team(organization, team_name) + team_name = "managers" + team = create_organization_team(organization, team_name) tools.add_user_to_team(team, user_deno) self.assertIn(team.group_ptr, user_deno.groups.all()) @@ -50,7 +54,7 @@ def test_add_team_to_project(self): project_name = "demo" team_name = "enumerators" project = self._create_project(organization, project_name, self.user) - team = tools.create_organization_team(organization, team_name) + team = create_organization_team(organization, team_name) result = tools.add_team_to_project(team, project) self.assertTrue(result) @@ -59,32 +63,41 @@ def test_add_team_to_project(self): def test_add_project_perms_to_team(self): # create an org, user, team organization = self._create_organization("test org", self.user) - user_deno = self._create_user('deno', 'deno') + user_deno = self._create_user("deno", "deno") # add a member to the team - team = tools.create_organization_team(organization, "test team") + team = create_organization_team(organization, "test team") tools.add_user_to_team(team, user_deno) - project = Project.objects.create(name="Test Project", - organization=organization, - created_by=user_deno, - metadata='{}') + project = Project.objects.create( + name="Test Project", + organization=organization, + created_by=user_deno, + metadata="{}", + ) # confirm that the team has no permissions on project self.assertFalse(get_perms(team, project)) # set DataEntryRole role of project on team DataEntryRole.add(team, project) - self.assertEqual([CAN_EXPORT_PROJECT, CAN_ADD_SUBMISSIONS_PROJECT, - CAN_VIEW_PROJECT, CAN_VIEW_PROJECT_ALL, - CAN_VIEW_PROJECT_DATA], - sorted(get_perms(team, project))) - - self.assertEqual(get_team_project_default_permissions(team, project), - DataEntryRole.name) + self.assertEqual( + [ + CAN_EXPORT_PROJECT, + CAN_ADD_SUBMISSIONS_PROJECT, + CAN_VIEW_PROJECT, + CAN_VIEW_PROJECT_ALL, + CAN_VIEW_PROJECT_DATA, + ], + sorted(get_perms(team, project)), + ) + + self.assertEqual( + get_team_project_default_permissions(team, project), DataEntryRole.name + ) # Add a new user - user_sam = self._create_user('Sam', 'sammy_') + user_sam = self._create_user("Sam", "sammy_") self.assertFalse(user_sam.has_perm(CAN_VIEW_PROJECT, project)) self.assertFalse(user_sam.has_perm(CAN_ADD_XFORM, project)) diff --git a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py index b3092c9bb0..27c1fbd2d4 100644 --- a/onadata/apps/api/tests/viewsets/test_abstract_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_abstract_viewset.py @@ -5,15 +5,15 @@ import json import os import re +import warnings from tempfile import NamedTemporaryFile -import requests - from django.conf import settings from django.contrib.auth import authenticate, get_user_model from django.contrib.auth.models import Permission from django.test import TestCase +import requests from django_digest.test import Client as DigestClient from django_digest.test import DigestAuth from httmock import HTTMock @@ -41,10 +41,11 @@ from onadata.libs.utils.common_tools import merge_dicts from onadata.libs.utils.user_auth import get_user_default_project - # pylint: disable=invalid-name User = get_user_model() +warnings.simplefilter("ignore") + def _set_api_permissions(user): add_userprofile = Permission.objects.get( @@ -471,7 +472,8 @@ def _post_metadata(self, data, test=True): "/", data=data, **self.extra, - format='json' if 'extra_data' in data else None) + format="json" if "extra_data" in data else None, + ) response = view(request) @@ -494,11 +496,7 @@ def _add_form_metadata( test=True, extra_data=None, ): - data = { - "data_type": data_type, - "data_value": data_value, - "xform": xform.id - } + data = {"data_type": data_type, "data_value": data_value, "xform": xform.id} if extra_data: data.update({"extra_data": extra_data}) diff --git a/onadata/apps/api/tests/viewsets/test_connect_viewset.py b/onadata/apps/api/tests/viewsets/test_connect_viewset.py index ae3053e51d..ab4911f442 100644 --- a/onadata/apps/api/tests/viewsets/test_connect_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_connect_viewset.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Test /user API endpoint """ diff --git a/onadata/apps/api/tests/viewsets/test_floip_viewset.py b/onadata/apps/api/tests/viewsets/test_floip_viewset.py index 773dabc890..579d9feee4 100644 --- a/onadata/apps/api/tests/viewsets/test_floip_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_floip_viewset.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Test FloipViewset module. """ diff --git a/onadata/apps/api/tests/viewsets/test_media_viewset.py b/onadata/apps/api/tests/viewsets/test_media_viewset.py index edc94e5987..771d13b1c5 100644 --- a/onadata/apps/api/tests/viewsets/test_media_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_media_viewset.py @@ -2,57 +2,56 @@ import urllib from mock import MagicMock, patch -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.media_viewset import MediaViewSet from onadata.apps.logger.models import Attachment def attachment_url(attachment, suffix=None): - url = u'http://testserver/api/v1/files/{}?filename={}'.format( - attachment.pk, attachment.media_file.name) + url = "http://testserver/api/v1/files/{}?filename={}".format( + attachment.pk, attachment.media_file.name + ) if suffix: - url += u'?suffix={}'.format(suffix) + url += "?suffix={}".format(suffix) return url class TestMediaViewSet(TestAbstractViewSet): - def setUp(self): super(TestMediaViewSet, self).setUp() - self.retrieve_view = MediaViewSet.as_view({ - 'get': 'retrieve' - }) + self.retrieve_view = MediaViewSet.as_view({"get": "retrieve"}) self._publish_xls_form_to_project() self._submit_transport_instance_w_attachment() def test_retrieve_view(self): - request = self.factory.get('/', { - 'filename': self.attachment.media_file.name}, **self.extra) + request = self.factory.get( + "/", {"filename": self.attachment.media_file.name}, **self.extra + ) response = self.retrieve_view(request, self.attachment.pk) - self.assertEqual(response.status_code, 302) + self.assertEqual(response.status_code, 200) self.assertEqual(type(response.content), bytes) - @patch('onadata.libs.utils.image_tools.get_storage_class') - @patch('onadata.libs.utils.image_tools.boto3.client') - def test_retrieve_view_from_s3( - self, mock_presigned_urls, mock_get_storage_class): + @patch("onadata.libs.utils.image_tools.get_storage_class") + @patch("onadata.libs.utils.image_tools.boto3.client") + def test_retrieve_view_from_s3(self, mock_presigned_urls, mock_get_storage_class): expected_url = ( - 'https://testing.s3.amazonaws.com/doe/attachments/' - '4_Media_file/media.png?' - 'response-content-disposition=attachment%3Bfilename%3media.png&' - 'response-content-type=application%2Foctet-stream&' - 'AWSAccessKeyId=AKIAJ3XYHHBIJDL7GY7A' - '&Signature=aGhiK%2BLFVeWm%2Fmg3S5zc05g8%3D&Expires=1615554960') + "https://testing.s3.amazonaws.com/doe/attachments/" + "4_Media_file/media.png?" + "response-content-disposition=attachment%3Bfilename%3media.png&" + "response-content-type=application%2Foctet-stream&" + "AWSAccessKeyId=AKIAJ3XYHHBIJDL7GY7A" + "&Signature=aGhiK%2BLFVeWm%2Fmg3S5zc05g8%3D&Expires=1615554960" + ) mock_presigned_urls().generate_presigned_url = MagicMock( return_value=expected_url ) - mock_get_storage_class()().bucket.name = 'onadata' - request = self.factory.get('/', { - 'filename': self.attachment.media_file.name}, **self.extra) + mock_get_storage_class()().bucket.name = "onadata" + request = self.factory.get( + "/", {"filename": self.attachment.media_file.name}, **self.extra + ) response = self.retrieve_view(request, self.attachment.pk) self.assertEqual(response.status_code, 302, response.url) @@ -60,87 +59,104 @@ def test_retrieve_view_from_s3( self.assertTrue(mock_presigned_urls.called) filename = self.attachment.media_file.name.split("/")[-1] mock_presigned_urls().generate_presigned_url.assert_called_with( - 'get_object', + "get_object", Params={ - 'Bucket': 'onadata', - 'Key': self.attachment.media_file.name, - 'ResponseContentDisposition': urllib.parse.quote( - f'attachment; filename={filename}'), - 'ResponseContentType': 'application/octet-stream'}, - ExpiresIn=3600) + "Bucket": "onadata", + "Key": self.attachment.media_file.name, + "ResponseContentDisposition": urllib.parse.quote( + f"attachment; filename={filename}" + ), + "ResponseContentType": "application/octet-stream", + }, + ExpiresIn=3600, + ) def test_retrieve_view_with_suffix(self): - request = self.factory.get('/', { - 'filename': self.attachment.media_file.name, 'suffix': 'large'}, - **self.extra) + request = self.factory.get( + "/", + {"filename": self.attachment.media_file.name, "suffix": "large"}, + **self.extra, + ) response = self.retrieve_view(request, self.attachment.pk) self.assertEqual(response.status_code, 302) - self.assertTrue(response['Location'], attachment_url(self.attachment)) + self.assertTrue(response["Location"], attachment_url(self.attachment)) - @patch('onadata.apps.api.viewsets.media_viewset.image_url') + @patch("onadata.apps.api.viewsets.media_viewset.image_url") def test_handle_image_exception(self, mock_image_url): mock_image_url.side_effect = Exception() request = self.factory.get( - '/', - {'filename': self.attachment.media_file.name, 'suffix': 'large'}, - **self.extra + "/", + {"filename": self.attachment.media_file.name, "suffix": "large"}, + **self.extra, ) response = self.retrieve_view(request, self.attachment.pk) self.assertEqual(response.status_code, 400) def test_retrieve_view_small(self): request = self.factory.get( - '/', - {'filename': self.attachment.media_file.name, 'suffix': 'small'}, - **self.extra + "/", + {"filename": self.attachment.media_file.name, "suffix": "small"}, + **self.extra, ) response = self.retrieve_view(request, self.attachment.pk) self.assertEqual(response.status_code, 302) - self.assertTrue(response['Location'], - attachment_url(self.attachment, 'small')) + self.assertTrue(response["Location"], attachment_url(self.attachment, "small")) def test_retrieve_view_invalid_suffix(self): request = self.factory.get( - '/', - {'filename': self.attachment.media_file.name, 'suffix': 'TK'}, - **self.extra + "/", + {"filename": self.attachment.media_file.name, "suffix": "TK"}, + **self.extra, ) response = self.retrieve_view(request, self.attachment.pk) self.assertEqual(response.status_code, 404) def test_retrieve_view_invalid_pk(self): request = self.factory.get( - '/', - {'filename': self.attachment.media_file.name, 'suffix': 'small'}, - **self.extra + "/", + {"filename": self.attachment.media_file.name, "suffix": "small"}, + **self.extra, ) - response = self.retrieve_view(request, 'INVALID') + response = self.retrieve_view(request, "INVALID") self.assertEqual(response.status_code, 404) def test_retrieve_view_no_filename_param(self): - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.retrieve_view(request, self.attachment.pk) self.assertEqual(response.status_code, 404) def test_retrieve_small_png(self): """Test retrieve png images""" - s = 'transport_2011-07-25_19-05-49_1' + s = "transport_2011-07-25_19-05-49_1" media_file = "ona_png_image.png" - path = os.path.join(self.main_directory, 'fixtures', - 'transportation', 'instances', s, media_file) - with open(path, 'rb') as f: - self._make_submission(os.path.join( - self.main_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml'), media_file=f) + path = os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + media_file, + ) + with open(path, "rb") as f: + self._make_submission( + os.path.join( + self.main_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ), + media_file=f, + ) attachment = Attachment.objects.all().reverse()[0] self.attachment = attachment request = self.factory.get( - '/', - {'filename': self.attachment.media_file.name, 'suffix': 'small'}, - **self.extra + "/", + {"filename": self.attachment.media_file.name, "suffix": "small"}, + **self.extra, ) response = self.retrieve_view(request, self.attachment.pk) self.assertEqual(response.status_code, 302) - self.assertTrue(response['Location'], - attachment_url(self.attachment, 'small')) + self.assertTrue(response["Location"], attachment_url(self.attachment, "small")) diff --git a/onadata/apps/api/tests/viewsets/test_project_viewset.py b/onadata/apps/api/tests/viewsets/test_project_viewset.py index 77b0973846..4fd9546d2a 100644 --- a/onadata/apps/api/tests/viewsets/test_project_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_project_viewset.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Test ProjectViewSet module. """ diff --git a/onadata/apps/api/tools.py b/onadata/apps/api/tools.py index 7d27ad3938..18da1b990e 100644 --- a/onadata/apps/api/tools.py +++ b/onadata/apps/api/tools.py @@ -1,6 +1,6 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ -API util functions. +API utility functions. """ import os import tempfile @@ -9,8 +9,6 @@ from django import forms from django.conf import settings from django.contrib.auth import get_user_model -from django.contrib.auth.models import Permission -from django.contrib.contenttypes.models import ContentType from django.contrib.sites.models import Site from django.core.files.storage import get_storage_class from django.core.files.uploadedfile import InMemoryUploadedFile @@ -23,7 +21,7 @@ from django.utils.module_loading import import_string from django.utils.translation import gettext as _ -from guardian.shortcuts import assign_perm, get_perms, get_perms_for_model, remove_perm +from guardian.shortcuts import get_perms, get_perms_for_model, remove_perm from kombu.exceptions import OperationalError from multidb.pinning import use_master from registration.models import RegistrationProfile @@ -33,7 +31,9 @@ from onadata.apps.api.models.organization_profile import ( OrganizationProfile, - create_owner_team_and_assign_permissions, + add_user_to_team, + get_or_create_organization_owners_team, + get_organization_members_team, ) from onadata.apps.api.models.team import Team from onadata.apps.logger.models import DataView, Instance, Project, XForm @@ -55,8 +55,10 @@ get_role_in_org, is_organization, ) +from onadata.libs.serializers.project_serializer import ProjectSerializer from onadata.libs.utils.api_export_tools import ( - custom_response_handler, get_metadata_format + custom_response_handler, + get_metadata_format, ) from onadata.libs.utils.cache_tools import ( PROJ_BASE_FORMS_CACHE, @@ -181,60 +183,6 @@ def create_organization_object(org_name, creator, attrs=None): return profile -def create_organization_team(organization, name, permission_names=None): - """ - Creates an organization team with the given permissions as defined in - permission_names. - """ - organization = ( - organization.user - if isinstance(organization, OrganizationProfile) - else organization - ) - team = Team.objects.create(organization=organization, name=name) - content_type = ContentType.objects.get(app_label="api", model="organizationprofile") - if permission_names: - # get permission objects - perms = Permission.objects.filter( - codename__in=permission_names, content_type=content_type - ) - if perms: - team.permissions.add(*tuple(perms)) - return team - - -def get_organization_members_team(organization): - """Get organization members team - create members team if it does not exist and add organization owner - to the members team""" - try: - team = Team.objects.get(name=f"{organization.user.username}#{MEMBERS}") - except Team.DoesNotExist: - team = create_organization_team(organization, MEMBERS) - add_user_to_team(team, organization.user) - - return team - - -# pylint: disable=invalid-name -def get_or_create_organization_owners_team(org): - """ - Get the owners team of an organization - :param org: organization - :return: Owners team of the organization - """ - team_name = f"{org.user.username}#{Team.OWNER_TEAM_NAME}" - try: - team = Team.objects.get(name=team_name, organization=org.user) - except Team.DoesNotExist: - with use_master: - queryset = Team.objects.filter(name=team_name, organization=org.user) - if queryset.count() > 0: - return queryset.first() # pylint: disable=no-member - return create_owner_team_and_assign_permissions(org) - return team - - def remove_user_from_organization(organization, user): """Remove a user from an organization""" team = get_organization_members_team(organization) @@ -281,28 +229,6 @@ def add_user_to_organization(organization, user): add_user_to_team(team, user) -def add_user_to_team(team, user): - """ - Adds a user to a team and assigns them team permissions. - """ - user.groups.add(team) - - # give the user perms to view the team - assign_perm("view_team", user, team) - - # if team is owners team assign more perms - if team.name.find(Team.OWNER_TEAM_NAME) > 0: - _assign_organization_team_perms(team.organization, user) - - -def _assign_organization_team_perms(organization, user): - owners_team = get_or_create_organization_owners_team(organization.profile) - members_team = get_organization_members_team(organization.profile) - for perm in get_perms_for_model(Team): - assign_perm(perm.codename, user, owners_team) - assign_perm(perm.codename, user, members_team) - - def get_organization_members(organization): """Get members team user queryset""" team = get_organization_members_team(organization) @@ -491,7 +417,7 @@ def id_string_exists_in_account(): # Ensure the cached project is the updated version. # Django lazy loads related objects as such we need to # ensure the project retrieved is up to date. - reset_project_cache(xform.project, request) + reset_project_cache(xform.project, request, ProjectSerializer) return xform diff --git a/onadata/apps/api/urls/v1_urls.py b/onadata/apps/api/urls/v1_urls.py index 15d3a830a9..e4bcb249ad 100644 --- a/onadata/apps/api/urls/v1_urls.py +++ b/onadata/apps/api/urls/v1_urls.py @@ -1,9 +1,10 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Custom rest_framework Router - MultiLookupRouter. """ -from django.urls import re_path from django.contrib import admin +from django.urls import re_path + from rest_framework import routers from rest_framework.urlpatterns import format_suffix_patterns @@ -11,8 +12,7 @@ from onadata.apps.api.viewsets.briefcase_viewset import BriefcaseViewset from onadata.apps.api.viewsets.charts_viewset import ChartsViewSet from onadata.apps.api.viewsets.connect_viewset import ConnectViewSet -from onadata.apps.api.viewsets.data_viewset import (AuthenticatedDataViewSet, - DataViewSet) +from onadata.apps.api.viewsets.data_viewset import AuthenticatedDataViewSet, DataViewSet from onadata.apps.api.viewsets.dataview_viewset import DataViewViewSet from onadata.apps.api.viewsets.export_viewset import ExportViewSet from onadata.apps.api.viewsets.floip_viewset import FloipViewSet @@ -21,26 +21,23 @@ from onadata.apps.api.viewsets.metadata_viewset import MetaDataViewSet from onadata.apps.api.viewsets.note_viewset import NoteViewSet from onadata.apps.api.viewsets.open_data_viewset import OpenDataViewSet -from onadata.apps.api.viewsets.organization_profile_viewset import \ - OrganizationProfileViewSet +from onadata.apps.api.viewsets.organization_profile_viewset import ( + OrganizationProfileViewSet, +) from onadata.apps.api.viewsets.osm_viewset import OsmViewSet from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.stats_viewset import StatsViewSet -from onadata.apps.api.viewsets.submission_review_viewset import \ - SubmissionReviewViewSet -from onadata.apps.api.viewsets.submissionstats_viewset import \ - SubmissionStatsViewSet +from onadata.apps.api.viewsets.submission_review_viewset import SubmissionReviewViewSet +from onadata.apps.api.viewsets.submissionstats_viewset import SubmissionStatsViewSet from onadata.apps.api.viewsets.team_viewset import TeamViewSet from onadata.apps.api.viewsets.user_profile_viewset import UserProfileViewSet from onadata.apps.api.viewsets.user_viewset import UserViewSet from onadata.apps.api.viewsets.widget_viewset import WidgetViewSet from onadata.apps.api.viewsets.xform_list_viewset import XFormListViewSet -from onadata.apps.api.viewsets.xform_submission_viewset import \ - XFormSubmissionViewSet +from onadata.apps.api.viewsets.xform_submission_viewset import XFormSubmissionViewSet from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.apps.messaging.viewsets import MessagingViewSet -from onadata.apps.restservice.viewsets.restservices_viewset import \ - RestServicesViewSet +from onadata.apps.restservice.viewsets.restservices_viewset import RestServicesViewSet admin.autodiscover() @@ -49,21 +46,20 @@ class MultiLookupRouter(routers.DefaultRouter): """ Support multiple lookup keys e.g. /parent_pk/pk """ + multi = False - def get_lookup_regex(self, viewset, lookup_prefix=''): + def get_lookup_regex(self, viewset, lookup_prefix=""): """ Returns a lookup regex, this extends the default to allow for multiple lookup keys as defined by a viewset.lookup_fields property. """ - result = super(MultiLookupRouter, self).get_lookup_regex( - viewset, lookup_prefix) - lookup_fields = getattr(viewset, 'lookup_fields', None) + result = super().get_lookup_regex(viewset, lookup_prefix) + lookup_fields = getattr(viewset, "lookup_fields", None) if lookup_fields and not self.multi: - lookup_value = getattr(viewset, 'lookup_value_regex', '[^/.]+') + lookup_value = getattr(viewset, "lookup_value_regex", "[^/.]+") for lookup_field in lookup_fields[1:]: - result += '/(?P<{lookup_url_kwarg}>{lookup_value})'.format( - lookup_url_kwarg=lookup_field, lookup_value=lookup_value) + result += f"/(?P<{lookup_field}>{lookup_value})" return result @@ -72,39 +68,41 @@ def get_urls(self): Return a list of URL regexs, this extends the default by adding a {prefix}-list route that accepts a lookup url kwarg. """ - urls = super(MultiLookupRouter, self).get_urls() + urls = super().get_urls() extra_urls = [] for prefix, viewset, basename in self.registry: - lookup_fields = getattr(viewset, 'lookup_fields', None) + lookup_fields = getattr(viewset, "lookup_fields", None) if lookup_fields: route = routers.Route( - url=r'^{prefix}/{lookup}{trailing_slash}$', + url=r"^{prefix}/{lookup}{trailing_slash}$", mapping={ - 'delete': 'destroy', - 'get': 'list', - 'post': 'create', + "delete": "destroy", + "get": "list", + "post": "create", }, - name='{basename}-list', + name="{basename}-list", detail=False, - initkwargs={'suffix': 'List'}) + initkwargs={"suffix": "List"}, + ) self.multi = True lookup = self.get_lookup_regex(viewset) # reset self.multi = False regex = route.url.format( - prefix=prefix, - lookup=lookup, - trailing_slash=self.trailing_slash) + prefix=prefix, lookup=lookup, trailing_slash=self.trailing_slash + ) mapping = self.get_method_map(viewset, route.mapping) if not mapping: continue initkwargs = route.initkwargs.copy() - initkwargs.update({ - 'basename': basename, - 'detail': route.detail, - }) + initkwargs.update( + { + "basename": basename, + "detail": route.detail, + } + ) view = viewset.as_view(mapping, **initkwargs) name = route.name.format(basename=basename) extra_urls.append(re_path(regex, view, name=name)) @@ -115,39 +113,37 @@ def get_urls(self): return urls -router = MultiLookupRouter(trailing_slash=False) # pylint: disable=c0103 -router.register(r'briefcase', BriefcaseViewset, basename='briefcase') -router.register(r'charts', ChartsViewSet, basename='chart') -router.register(r'data', DataViewSet, basename='data') -router.register(r'dataviews', DataViewViewSet, basename='dataviews') -router.register(r'export', ExportViewSet, basename='export') -router.register(r'files', MediaViewSet, basename='files') -router.register( - r'flow-results/packages', FloipViewSet, basename='flow-results') -router.register(r'formlist', XFormListViewSet, basename='formlist') -router.register(r'forms', XFormViewSet) -router.register(r'media', AttachmentViewSet, basename='attachment') -router.register( - r'merged-datasets', MergedXFormViewSet, basename='merged-xform') -router.register(r'messaging', MessagingViewSet, basename="messaging") -router.register(r'metadata', MetaDataViewSet, basename='metadata') -router.register(r'notes', NoteViewSet) -router.register(r'open-data', OpenDataViewSet, basename='open-data') -router.register(r'orgs', OrganizationProfileViewSet) -router.register(r'osm', OsmViewSet, basename='osm') -router.register( - r'private-data', AuthenticatedDataViewSet, basename='private-data') -router.register(r'profiles', UserProfileViewSet) -router.register(r'projects', ProjectViewSet) -router.register(r'restservices', RestServicesViewSet, basename='restservices') -router.register(r'stats', StatsViewSet, basename='stats') -router.register( - r'submissionreview', SubmissionReviewViewSet, basename='submissionreview') +router = MultiLookupRouter(trailing_slash=False) # pylint: disable=invalid-name +router.register(r"briefcase", BriefcaseViewset, basename="briefcase") +router.register(r"charts", ChartsViewSet, basename="chart") +router.register(r"data", DataViewSet, basename="data") +router.register(r"dataviews", DataViewViewSet, basename="dataviews") +router.register(r"export", ExportViewSet, basename="export") +router.register(r"files", MediaViewSet, basename="files") +router.register(r"flow-results/packages", FloipViewSet, basename="flow-results") +router.register(r"formlist", XFormListViewSet, basename="formlist") +router.register(r"forms", XFormViewSet) +router.register(r"media", AttachmentViewSet, basename="attachment") +router.register(r"merged-datasets", MergedXFormViewSet, basename="merged-xform") +router.register(r"messaging", MessagingViewSet, basename="messaging") +router.register(r"metadata", MetaDataViewSet, basename="metadata") +router.register(r"notes", NoteViewSet) +router.register(r"open-data", OpenDataViewSet, basename="open-data") +router.register(r"orgs", OrganizationProfileViewSet) +router.register(r"osm", OsmViewSet, basename="osm") +router.register(r"private-data", AuthenticatedDataViewSet, basename="private-data") +router.register(r"profiles", UserProfileViewSet) +router.register(r"projects", ProjectViewSet) +router.register(r"restservices", RestServicesViewSet, basename="restservices") +router.register(r"stats", StatsViewSet, basename="stats") router.register( - r'stats/submissions', SubmissionStatsViewSet, basename='submissionstats') + r"submissionreview", SubmissionReviewViewSet, basename="submissionreview" +) router.register( - r'submissions', XFormSubmissionViewSet, basename='submissions') -router.register(r'teams', TeamViewSet) -router.register(r'user', ConnectViewSet) -router.register(r'users', UserViewSet, basename='user') -router.register(r'widgets', WidgetViewSet, basename='widgets') + r"stats/submissions", SubmissionStatsViewSet, basename="submissionstats" +) +router.register(r"submissions", XFormSubmissionViewSet, basename="submissions") +router.register(r"teams", TeamViewSet) +router.register(r"user", ConnectViewSet) +router.register(r"users", UserViewSet, basename="user") +router.register(r"widgets", WidgetViewSet, basename="widgets") diff --git a/onadata/apps/api/urls/v2_urls.py b/onadata/apps/api/urls/v2_urls.py index 34da69ad81..0751997480 100644 --- a/onadata/apps/api/urls/v2_urls.py +++ b/onadata/apps/api/urls/v2_urls.py @@ -1,9 +1,10 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Custom rest_framework Router V2 """ -from .v1_urls import MultiLookupRouter from onadata.apps.api.viewsets.v2.tableau_viewset import TableauViewSet +from .v1_urls import MultiLookupRouter + router = MultiLookupRouter(trailing_slash=False) -router.register(r'open-data', TableauViewSet, basename='open-data') +router.register(r"open-data", TableauViewSet, basename="open-data") diff --git a/onadata/apps/api/viewsets/attachment_viewset.py b/onadata/apps/api/viewsets/attachment_viewset.py index ac5af6a90f..2a92b1ecd8 100644 --- a/onadata/apps/api/viewsets/attachment_viewset.py +++ b/onadata/apps/api/viewsets/attachment_viewset.py @@ -1,11 +1,13 @@ -from builtins import str as text - +# -*- coding: utf-8 -*- +""" +The /api/v1/attachments API implementation. +""" +from django.conf import settings +from django.core.files.storage import default_storage from django.http import Http404 from django.utils.translation import gettext as _ -from django.core.files.storage import default_storage -from django.conf import settings -from rest_framework import renderers -from rest_framework import viewsets + +from rest_framework import renderers, viewsets from rest_framework.decorators import action from rest_framework.exceptions import ParseError from rest_framework.response import Response @@ -14,41 +16,47 @@ from onadata.apps.logger.models.attachment import Attachment from onadata.apps.logger.models.xform import XForm from onadata.libs import filters -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.pagination import StandardPageNumberPagination +from onadata.libs.renderers.renderers import ( + MediaFileContentNegotiation, + MediaFileRenderer, +) from onadata.libs.serializers.attachment_serializer import AttachmentSerializer -from onadata.libs.renderers.renderers import MediaFileContentNegotiation, \ - MediaFileRenderer from onadata.libs.utils.image_tools import image_url from onadata.libs.utils.viewer_tools import get_path def get_attachment_data(attachment, suffix): + """Returns attachment file contents.""" if suffix in list(settings.THUMB_CONF): image_url(attachment, suffix) - suffix = settings.THUMB_CONF.get(suffix).get('suffix') - f = default_storage.open( - get_path(attachment.media_file.name, suffix)) - data = f.read() - else: - data = attachment.media_file.read() + suffix = settings.THUMB_CONF.get(suffix).get("suffix") + f = default_storage.open(get_path(attachment.media_file.name, suffix)) + return f.read() - return data + return attachment.media_file.read() -class AttachmentViewSet(AuthenticateHeaderMixin, CacheControlMixin, ETagsMixin, - viewsets.ReadOnlyModelViewSet): +# pylint: disable=too-many-ancestors +class AttachmentViewSet( + AuthenticateHeaderMixin, + CacheControlMixin, + ETagsMixin, + viewsets.ReadOnlyModelViewSet, +): """ - List attachments of viewsets. + GET, List attachments implementation. """ + content_negotiation_class = MediaFileContentNegotiation filter_backends = (filters.AttachmentFilter, filters.AttachmentTypeFilter) - lookup_field = 'pk' + lookup_field = "pk" queryset = Attachment.objects.filter( - instance__deleted_at__isnull=True, deleted_at__isnull=True) + instance__deleted_at__isnull=True, deleted_at__isnull=True + ) permission_classes = (AttachmentObjectPermissions,) serializer_class = AttachmentSerializer pagination_class = StandardPageNumberPagination @@ -59,48 +67,51 @@ class AttachmentViewSet(AuthenticateHeaderMixin, CacheControlMixin, ETagsMixin, ) def retrieve(self, request, *args, **kwargs): + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - if isinstance(request.accepted_renderer, MediaFileRenderer) \ - and self.object.media_file is not None: - suffix = request.query_params.get('suffix') + if ( + isinstance(request.accepted_renderer, MediaFileRenderer) + and self.object.media_file is not None + ): + suffix = request.query_params.get("suffix") try: data = get_attachment_data(self.object, suffix) except IOError as e: - if text(e).startswith('File does not exist'): - raise Http404() + if str(e).startswith("File does not exist"): + raise Http404() from e - raise ParseError(e) + raise ParseError(e) from e else: return Response(data, content_type=self.object.mimetype) - filename = request.query_params.get('filename') + filename = request.query_params.get("filename") serializer = self.get_serializer(self.object) if filename: if filename == self.object.media_file.name: return Response(serializer.get_download_url(self.object)) - else: - raise Http404(_("Filename '%s' not found." % filename)) + + raise Http404(_(f"Filename '{filename}' not found.")) return Response(serializer.data) - @action(methods=['GET'], detail=False) + @action(methods=["GET"], detail=False) def count(self, request, *args, **kwargs): - data = { - "count": self.filter_queryset(self.get_queryset()).count() - } + """Returns the number of attachments the user has access to.""" + data = {"count": self.filter_queryset(self.get_queryset()).count()} return Response(data=data) def list(self, request, *args, **kwargs): if request.user.is_anonymous: - xform = request.query_params.get('xform') + xform = request.query_params.get("xform") if xform: xform = XForm.objects.get(id=xform) if not xform.shared_data: raise Http404(_("Not Found")) + # pylint: disable=attribute-defined-outside-init self.object_list = self.filter_queryset(self.get_queryset()) page = self.paginate_queryset(self.object_list) if page is not None: @@ -108,4 +119,4 @@ def list(self, request, *args, **kwargs): return Response(serializer.data) - return super(AttachmentViewSet, self).list(request, *args, **kwargs) + return super().list(request, *args, **kwargs) diff --git a/onadata/apps/api/viewsets/briefcase_viewset.py b/onadata/apps/api/viewsets/briefcase_viewset.py index b6fa26753c..7056d80163 100644 --- a/onadata/apps/api/viewsets/briefcase_viewset.py +++ b/onadata/apps/api/viewsets/briefcase_viewset.py @@ -45,7 +45,7 @@ def _extract_uuid(text): if isinstance(text, six.string_types): form_id_parts = text.split("/") - if form_id_parts.__len__() < 2: + if len(form_id_parts) < 2: raise ValidationError(_(f"Invalid formId {text}.")) text = form_id_parts[1] @@ -111,7 +111,9 @@ def get_object(self, queryset=None): # pylint: disable=too-many-branches def filter_queryset(self, queryset): - """Filters an XForm submission instances using ODK Aggregate query parameters.""" + """ + Filters an XForm submission instances using ODK Aggregate query parameters. + """ username = self.kwargs.get("username") if username is None and self.request.user.is_anonymous: # raises a permission denied exception, forces authentication diff --git a/onadata/apps/api/viewsets/charts_viewset.py b/onadata/apps/api/viewsets/charts_viewset.py index a6833ff546..0251f4f7f4 100644 --- a/onadata/apps/api/viewsets/charts_viewset.py +++ b/onadata/apps/api/viewsets/charts_viewset.py @@ -3,37 +3,42 @@ /charts api endpoint for chart data and chart widgets """ -from django.core.exceptions import ImproperlyConfigured -from django.core.cache import cache from django.conf import settings +from django.core.cache import cache +from django.core.exceptions import ImproperlyConfigured from rest_framework import viewsets from rest_framework.exceptions import ParseError -from rest_framework.renderers import (BrowsableAPIRenderer, JSONRenderer, - TemplateHTMLRenderer) +from rest_framework.renderers import ( + BrowsableAPIRenderer, + JSONRenderer, + TemplateHTMLRenderer, +) from rest_framework.response import Response from onadata.apps.api.permissions import XFormPermissions from onadata.apps.logger.models.xform import XForm from onadata.libs import filters -from onadata.libs.utils.common_tools import str_to_bool -from onadata.libs.mixins.anonymous_user_public_forms_mixin import \ - AnonymousUserPublicFormsMixin -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.anonymous_user_public_forms_mixin import ( + AnonymousUserPublicFormsMixin, +) +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.renderers.renderers import DecimalJSONRenderer -from onadata.libs.serializers.chart_serializer import (ChartSerializer, - FieldsChartSerializer) -from onadata.libs.utils.chart_tools import get_chart_data_for_field +from onadata.libs.serializers.chart_serializer import ( + ChartSerializer, + FieldsChartSerializer, +) from onadata.libs.utils.cache_tools import XFORM_CHARTS +from onadata.libs.utils.chart_tools import get_chart_data_for_field +from onadata.libs.utils.common_tools import str_to_bool def get_form_field_chart_url(url, field): """Append 'field_name' to a given url""" - return u'%s?field_name=%s' % (url, field) + return f"{url}?field_name={field}" class ChartBrowsableAPIRenderer(BrowsableAPIRenderer): @@ -47,60 +52,70 @@ def get_default_renderer(self, view): (Don't use another documenting renderer.) """ renderers = [ - renderer for renderer in view.renderer_classes + renderer + for renderer in view.renderer_classes if not issubclass(renderer, BrowsableAPIRenderer) ] if not renderers: return None return renderers[0]() - def get_content(self, renderer, data, accepted_media_type, - renderer_context): + def get_content(self, renderer, data, accepted_media_type, renderer_context): try: - content = super(ChartBrowsableAPIRenderer, self).get_content( - renderer, data, accepted_media_type, renderer_context) + content = super().get_content( + renderer, data, accepted_media_type, renderer_context + ) except ImproperlyConfigured: - content = super(ChartBrowsableAPIRenderer, self).get_content( - JSONRenderer(), data, accepted_media_type, renderer_context) + content = super().get_content( + JSONRenderer(), data, accepted_media_type, renderer_context + ) return content -# pylint: disable=R0901 -class ChartsViewSet(AnonymousUserPublicFormsMixin, AuthenticateHeaderMixin, - CacheControlMixin, ETagsMixin, - viewsets.ReadOnlyModelViewSet): +# pylint: disable=too-many-ancestors +class ChartsViewSet( + AnonymousUserPublicFormsMixin, + AuthenticateHeaderMixin, + CacheControlMixin, + ETagsMixin, + viewsets.ReadOnlyModelViewSet, +): """ ChartsViewSet: /charts api endpoint for chart data and chart widgets """ - filter_backends = (filters.AnonDjangoObjectPermissionFilter, ) + filter_backends = (filters.AnonDjangoObjectPermissionFilter,) queryset = XForm.objects.all() serializer_class = ChartSerializer - lookup_field = 'pk' - renderer_classes = (DecimalJSONRenderer, ChartBrowsableAPIRenderer, - TemplateHTMLRenderer, ) + lookup_field = "pk" + renderer_classes = ( + DecimalJSONRenderer, + ChartBrowsableAPIRenderer, + TemplateHTMLRenderer, + ) permission_classes = [ XFormPermissions, ] + # pylint: disable=too-many-locals def retrieve(self, request, *args, **kwargs): - field_name = request.query_params.get('field_name') - field_xpath = request.query_params.get('field_xpath') - fields = request.query_params.get('fields') - group_by = request.query_params.get('group_by') - fmt = kwargs.get('format') - refresh_cache = str_to_bool(request.query_params.get('refresh')) + field_name = request.query_params.get("field_name") + field_xpath = request.query_params.get("field_xpath") + fields = request.query_params.get("fields") + group_by = request.query_params.get("group_by") + fmt = kwargs.get("format") + refresh_cache = str_to_bool(request.query_params.get("refresh")) xform = self.get_object() serializer = self.get_serializer(xform) # Default format to JSON if format is not specified if fmt is None and request.accepted_media_type == "application/json": - fmt = 'json' + fmt = "json" if fields: - if fmt is not None and fmt != 'json': + if fmt is not None and fmt != "json": raise ParseError("Error: only JSON format supported.") xform = self.get_object() @@ -110,21 +125,22 @@ def retrieve(self, request, *args, **kwargs): return Response(serializer.data) if field_name or field_xpath: - cache_key = '{}{}{}{}{}{}'.format(XFORM_CHARTS, xform.pk, - field_xpath, field_name, - group_by, fmt) + cache_key = ( + f"{XFORM_CHARTS}{xform.pk}{field_xpath}{field_name}{group_by}{fmt}" + ) data = cache.get(cache_key) if not data or refresh_cache: - data = get_chart_data_for_field(field_name, xform, fmt, - group_by, field_xpath) + data = get_chart_data_for_field( + field_name, xform, fmt, group_by, field_xpath + ) cache.set(cache_key, data, settings.XFORM_CHARTS_CACHE_TIME) - return Response(data, template_name='chart_detail.html') + return Response(data, template_name="chart_detail.html") - if fmt != 'json' and field_name is None: + if fmt != "json" and field_name is None: raise ParseError("Not supported") data = serializer.data diff --git a/onadata/apps/api/viewsets/connect_viewset.py b/onadata/apps/api/viewsets/connect_viewset.py index a1c0f3817f..549a26fe55 100644 --- a/onadata/apps/api/viewsets/connect_viewset.py +++ b/onadata/apps/api/viewsets/connect_viewset.py @@ -1,113 +1,131 @@ +# -*- coding: utf-8 -*- +""" +The /api/v1/user API implementation + +User authentication API support to access API tokens. +""" from django.core.exceptions import MultipleObjectsReturned from django.utils import timezone from django.utils.decorators import classonlymethod from django.utils.translation import gettext as _ from django.views.decorators.cache import never_cache -from rest_framework import status, viewsets + +from multidb.pinning import use_master +from rest_framework import mixins, status, viewsets from rest_framework.authtoken.models import Token from rest_framework.decorators import action from rest_framework.exceptions import ParseError from rest_framework.response import Response -from rest_framework import mixins -from multidb.pinning import use_master from onadata.apps.api.models.odk_token import ODKToken from onadata.apps.api.models.temp_token import TempToken from onadata.apps.api.permissions import ConnectViewsetPermissions -from onadata.apps.api.viewsets.user_profile_viewset import \ - serializer_from_settings +from onadata.apps.api.viewsets.user_profile_viewset import serializer_from_settings from onadata.apps.main.models.user_profile import UserProfile -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.mixins.object_lookup_mixin import ObjectLookupMixin from onadata.libs.serializers.password_reset_serializer import ( - PasswordResetChangeSerializer, PasswordResetSerializer, get_user_from_uid) + PasswordResetChangeSerializer, + PasswordResetSerializer, + get_user_from_uid, +) from onadata.libs.serializers.project_serializer import ProjectSerializer -from onadata.libs.serializers.user_profile_serializer import \ - UserProfileWithTokenSerializer +from onadata.libs.serializers.user_profile_serializer import ( + UserProfileWithTokenSerializer, +) +from onadata.libs.utils.cache_tools import USER_PROFILE_PREFIX, cache from onadata.settings.common import DEFAULT_SESSION_EXPIRY_TIME -from onadata.libs.utils.cache_tools import (cache, - USER_PROFILE_PREFIX) -def user_profile_w_token_response(request, status): - """ Returns authenticated user profile""" +def user_profile_w_token_response(request, status_code): + """Returns authenticated user profile""" if request and not request.user.is_anonymous: session = getattr(request, "session") if not session.session_key: # login user to create session token - # TODO cannot call this without calling authenticate first or - # setting the backend, commented for now. - # login(request, request.user) session.set_expiry(DEFAULT_SESSION_EXPIRY_TIME) try: user_profile = request.user.profile except UserProfile.DoesNotExist: - user_profile = cache.get( - f'{USER_PROFILE_PREFIX}{request.user.username}') + user_profile = cache.get(f"{USER_PROFILE_PREFIX}{request.user.username}") if not user_profile: with use_master: - user_profile, _ = UserProfile.objects.get_or_create( - user=request.user) + user_profile, _ = UserProfile.objects.get_or_create(user=request.user) serializer = serializer_from_settings()( - user_profile, - context={'request': request}) + user_profile, context={"request": request} + ) cache.set( - f'{USER_PROFILE_PREFIX}{request.user.username}', - serializer.data) + f"{USER_PROFILE_PREFIX}{request.user.username}", serializer.data + ) serializer = UserProfileWithTokenSerializer( - instance=user_profile, context={"request": request}) + instance=user_profile, context={"request": request} + ) - return Response(serializer.data, status=status) + return Response(serializer.data, status=status_code) -class ConnectViewSet(mixins.CreateModelMixin, AuthenticateHeaderMixin, - CacheControlMixin, ETagsMixin, ObjectLookupMixin, - viewsets.GenericViewSet): +# pylint: disable=too-many-ancestors +class ConnectViewSet( + mixins.CreateModelMixin, + AuthenticateHeaderMixin, + CacheControlMixin, + ETagsMixin, + ObjectLookupMixin, + viewsets.GenericViewSet, +): """ This endpoint allows you retrieve the authenticated user's profile info. """ - lookup_field = 'user' + + lookup_field = "user" queryset = UserProfile.objects.all() - permission_classes = (ConnectViewsetPermissions, ) + permission_classes = (ConnectViewsetPermissions,) serializer_class = UserProfileWithTokenSerializer - # pylint: disable=R0201 def create(self, request, *args, **kwargs): return user_profile_w_token_response(request, status.HTTP_201_CREATED) def list(self, request, *args, **kwargs): + """ + Implements the List endpoint - returns authentication tokens for current user. + """ return user_profile_w_token_response(request, status.HTTP_200_OK) - @action(methods=['GET'], detail=True) + @action(methods=["GET"], detail=True) def starred(self, request, *args, **kwargs): """Return projects starred for this user.""" user_profile = self.get_object() user = user_profile.user projects = user.project_stars.all() serializer = ProjectSerializer( - projects, context={'request': request}, many=True) + projects, context={"request": request}, many=True + ) return Response(data=serializer.data) - @action(methods=['POST'], detail=False) + @action(methods=["POST"], detail=False) def reset(self, request, *args, **kwargs): - context = {'request': request} + """ + Implements the /reset endpoint + + Allows a user to reset and change their password. + """ + context = {"request": request} data = request.data if request.data is not None else {} - if 'token' in request.data: - serializer = PasswordResetChangeSerializer( - data=data, context=context) + if "token" in request.data: + serializer = PasswordResetChangeSerializer(data=data, context=context) if serializer.is_valid(): serializer.save() - user = get_user_from_uid(serializer.data['uid']) - return Response(data={'username': user.username}, - status=status.HTTP_200_OK) + user = get_user_from_uid(serializer.data["uid"]) + return Response( + data={"username": user.username}, status=status.HTTP_200_OK + ) else: serializer = PasswordResetSerializer(data=data, context=context) if serializer.is_valid(): @@ -116,63 +134,78 @@ def reset(self, request, *args, **kwargs): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - @action(methods=['DELETE'], detail=False) + @action(methods=["DELETE"], detail=False) def expire(self, request, *args, **kwargs): + """ + Implements the /expire endpoint + + Allows a user to expire a TempToken. + """ try: TempToken.objects.get(user=request.user).delete() - except TempToken.DoesNotExist: - raise ParseError(_(u"Temporary token not found!")) + except TempToken.DoesNotExist as exc: + raise ParseError(_("Temporary token not found!")) from exc return Response(status=status.HTTP_204_NO_CONTENT) - @action(methods=['GET'], detail=False) + @action(methods=["GET"], detail=False) def regenerate_auth_token(self, request, *args, **kwargs): + """ + Implements the /regenerate_auth_token endpoint + + Allows a user to expire and create a new API Token. + """ try: Token.objects.get(user=request.user).delete() - except Token.DoesNotExist: - raise ParseError(_(u" Token not found!")) + except Token.DoesNotExist as exc: + raise ParseError(_(" Token not found!")) from exc new_token = Token.objects.create(user=request.user) return Response(data=new_token.key, status=status.HTTP_201_CREATED) - @action(methods=['GET', 'POST'], detail=False) + @action(methods=["GET", "POST"], detail=False) def odk_token(self, request, *args, **kwargs): + """ + Implements the /odk_token endpoint + + Allows a user to get or create or expire an ODKToken for use with ODK Collect. + """ user = request.user + status_code = status.HTTP_200_OK - if request.method == 'GET': + if request.method == "GET": try: token, created = ODKToken.objects.get_or_create( - user=user, status=ODKToken.ACTIVE) + user=user, status=ODKToken.ACTIVE + ) if not created and timezone.now() > token.expires: token.status = ODKToken.INACTIVE token.save() token = ODKToken.objects.create(user=user) except MultipleObjectsReturned: - ODKToken.objects.filter( - user=user, status=ODKToken.ACTIVE).update( - status=ODKToken.INACTIVE) + ODKToken.objects.filter(user=user, status=ODKToken.ACTIVE).update( + status=ODKToken.INACTIVE + ) token = ODKToken.objects.create(user=user) - return Response(data={ - 'odk_token': token.raw_key, - 'expires': token.expires}, status=status.HTTP_200_OK) - elif request.method == 'POST': + if request.method == "POST": # Regenerates the ODK Token if one is already existant - ODKToken.objects.filter( - user=user, status=ODKToken.ACTIVE).update( - status=ODKToken.INACTIVE) + ODKToken.objects.filter(user=user, status=ODKToken.ACTIVE).update( + status=ODKToken.INACTIVE + ) token = ODKToken.objects.create(user=user) + status_code = status.HTTP_201_CREATED - return Response(data={ - 'odk_token': token.raw_key, - 'expires': token.expires - }, status=status.HTTP_201_CREATED) + return Response( + data={"odk_token": token.raw_key, "expires": token.expires}, + status=status_code, + ) @classonlymethod - def as_view(cls, actions=None, **initkwargs): + def as_view(cls, actions=None, **initkwargs): # noqa view = super(ConnectViewSet, cls).as_view(actions, **initkwargs) return never_cache(view) diff --git a/onadata/apps/api/viewsets/data_viewset.py b/onadata/apps/api/viewsets/data_viewset.py index fe7cda9e52..18cadc0a49 100644 --- a/onadata/apps/api/viewsets/data_viewset.py +++ b/onadata/apps/api/viewsets/data_viewset.py @@ -102,6 +102,7 @@ def delete_instance(instance, user): raise ParseError(str(e)) from e +# pylint: disable=http-response-with-content-type-json # pylint: disable=too-many-ancestors class DataViewSet( AnonymousUserPublicFormsMixin, @@ -401,7 +402,7 @@ def destroy(self, request, *args, **kwargs): message_verb=SUBMISSION_DELETED, ) else: - raise PermissionDenied(_("You do not have delete " "permissions.")) + raise PermissionDenied(_("You do not have delete permissions.")) return Response(status=status.HTTP_204_NO_CONTENT) @@ -757,7 +758,6 @@ def get_json_string(item): content_type="application/xml", ) else: - # pylint: disable=http-response-with-content-type-json response = StreamingHttpResponse( json_stream(self.object_list, get_json_string), content_type="application/json", diff --git a/onadata/apps/api/viewsets/export_viewset.py b/onadata/apps/api/viewsets/export_viewset.py index f154eb989f..cb6230c61f 100644 --- a/onadata/apps/api/viewsets/export_viewset.py +++ b/onadata/apps/api/viewsets/export_viewset.py @@ -1,21 +1,35 @@ +# -*- coding: utf-8 -*- +""" +The /api/v1/exports API implementation. + +List, Create, Update, Destroy Export model objects. +""" import os +from rest_framework.mixins import DestroyModelMixin from rest_framework.settings import api_settings from rest_framework.viewsets import ReadOnlyModelViewSet -from rest_framework.mixins import DestroyModelMixin -from onadata.apps.viewer.models.export import Export from onadata.apps.api.permissions import ExportDjangoObjectPermission +from onadata.apps.viewer.models.export import Export +from onadata.libs import filters +from onadata.libs.authentication import TempTokenURLParameterAuthentication from onadata.libs.renderers import renderers from onadata.libs.serializers.export_serializer import ExportSerializer -from onadata.libs.authentication import TempTokenURLParameterAuthentication from onadata.libs.utils.logger_tools import response_with_mimetype_and_name -from onadata.libs import filters +# pylint: disable=too-many-ancestors class ExportViewSet(DestroyModelMixin, ReadOnlyModelViewSet): + """ + The /api/v1/exports API implementation. + + List, Create, Update, Destroy Export model objects. + """ + authentication_classes = api_settings.DEFAULT_AUTHENTICATION_CLASSES + [ - TempTokenURLParameterAuthentication] + TempTokenURLParameterAuthentication + ] queryset = Export.objects.all() renderer_classes = api_settings.DEFAULT_RENDERER_CLASSES + [ renderers.CSVRenderer, @@ -25,7 +39,7 @@ class ExportViewSet(DestroyModelMixin, ReadOnlyModelViewSet): renderers.SAVZIPRenderer, renderers.XLSRenderer, renderers.XLSXRenderer, - renderers.ZipRenderer + renderers.ZipRenderer, ] serializer_class = ExportSerializer filter_backends = (filters.ExportFilter,) @@ -41,4 +55,5 @@ def retrieve(self, request, *args, **kwargs): filename, extension=extension, file_path=export.filepath, - show_date=False) + show_date=False, + ) diff --git a/onadata/apps/api/viewsets/floip_viewset.py b/onadata/apps/api/viewsets/floip_viewset.py index 6a989b3c15..b00948f010 100644 --- a/onadata/apps/api/viewsets/floip_viewset.py +++ b/onadata/apps/api/viewsets/floip_viewset.py @@ -6,6 +6,7 @@ from django.db.models import Q from django.shortcuts import get_object_or_404 + from rest_framework import mixins, status, viewsets from rest_framework.decorators import action from rest_framework.response import Response @@ -15,11 +16,14 @@ from rest_framework_json_api.renderers import JSONRenderer from onadata.apps.api.permissions import XFormPermissions -from onadata.apps.logger.models import XForm, Instance +from onadata.apps.logger.models import Instance, XForm from onadata.libs import filters from onadata.libs.renderers.renderers import floip_list from onadata.libs.serializers.floip_serializer import ( - FloipListSerializer, FloipSerializer, FlowResultsResponseSerializer) + FloipListSerializer, + FloipSerializer, + FlowResultsResponseSerializer, +) class FlowResultsJSONRenderer(JSONRenderer): @@ -30,39 +34,57 @@ class FlowResultsJSONRenderer(JSONRenderer): # pylint: disable=too-many-arguments @classmethod def build_json_resource_obj( - cls, fields, resource, resource_instance, - resource_name, serializer, force_type_resolution=False): + cls, + fields, + resource, + resource_instance, + resource_name, + serializer, + force_type_resolution=False, + ): """ Build a JSON resource object using the id as it appears in the resource. """ - obj = super(FlowResultsJSONRenderer, cls).build_json_resource_obj( - fields, resource, resource_instance, resource_name, - serializer, force_type_resolution) - obj['id'] = resource['id'] + obj = super().build_json_resource_obj( + fields, + resource, + resource_instance, + resource_name, + serializer, + force_type_resolution, + ) + obj["id"] = resource["id"] return obj # pylint: disable=too-many-ancestors -class FloipViewSet(mixins.CreateModelMixin, mixins.DestroyModelMixin, - mixins.ListModelMixin, mixins.RetrieveModelMixin, - mixins.UpdateModelMixin, viewsets.GenericViewSet): +class FloipViewSet( + mixins.CreateModelMixin, + mixins.DestroyModelMixin, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + mixins.UpdateModelMixin, + viewsets.GenericViewSet, +): """ FloipViewSet: create, list, retrieve, destroy """ - filter_backends = (filters.AnonDjangoObjectPermissionFilter, - filters.PublicDatasetsFilter) + filter_backends = ( + filters.AnonDjangoObjectPermissionFilter, + filters.PublicDatasetsFilter, + ) permission_classes = [XFormPermissions] queryset = XForm.objects.filter(deleted_at__isnull=True) serializer_class = FloipSerializer pagination_class = PageNumberPagination - parser_classes = (JSONParser, ) - renderer_classes = (FlowResultsJSONRenderer, ) + parser_classes = (JSONParser,) + renderer_classes = (FlowResultsJSONRenderer,) - lookup_field = 'uuid' + lookup_field = "uuid" def get_object(self): queryset = self.filter_queryset(self.get_queryset()) @@ -76,24 +98,25 @@ def get_object(self): return obj def get_serializer_class(self): - if self.action == 'list': + if self.action == "list": return FloipListSerializer - if self.action == 'responses': + if self.action == "responses": return FlowResultsResponseSerializer - return super(FloipViewSet, self).get_serializer_class() + return super().get_serializer_class() def get_success_headers(self, data): - headers = super(FloipViewSet, self).get_success_headers(data) - headers['Content-Type'] = 'application/vnd.api+json' - uuid = str(UUID(data['id'])) - headers['Location'] = self.request.build_absolute_uri( - reverse('flow-results-detail', kwargs={'uuid': uuid})) + headers = super().get_success_headers(data) + headers["Content-Type"] = "application/vnd.api+json" + uuid = str(UUID(data["id"])) + headers["Location"] = self.request.build_absolute_uri( + reverse("flow-results-detail", kwargs={"uuid": uuid}) + ) return headers - @action(methods=['GET', 'POST'], detail=True) + @action(methods=["GET", "POST"], detail=True) def responses(self, request, uuid=None): """ Flow Results Responses endpoint. @@ -101,23 +124,21 @@ def responses(self, request, uuid=None): status_code = status.HTTP_200_OK xform = self.get_object() uuid = str(UUID(uuid or xform.uuid, version=4)) - data = { - "id": uuid, - "type": "flow-results-data", - "attributes": {} - } + data = {"id": uuid, "type": "flow-results-data", "attributes": {}} headers = { - 'Content-Type': 'application/vnd.api+json', - 'Location': self.request.build_absolute_uri( - reverse('flow-results-responses', kwargs={'uuid': uuid})) + "Content-Type": "application/vnd.api+json", + "Location": self.request.build_absolute_uri( + reverse("flow-results-responses", kwargs={"uuid": uuid}) + ), } # yapf: disable - if request.method == 'POST': + if request.method == "POST": serializer = FlowResultsResponseSerializer( - data=request.data, context={'request': request}) + data=request.data, context={"request": request} + ) serializer.is_valid(raise_exception=True) serializer.save() - data['response'] = serializer.data['responses'] - if serializer.data['duplicates']: + data["response"] = serializer.data["responses"] + if serializer.data["duplicates"]: status_code = status.HTTP_202_ACCEPTED else: status_code = status.HTTP_201_CREATED @@ -125,22 +146,22 @@ def responses(self, request, uuid=None): if xform.is_merged_dataset: pks = xform.mergedxform.xforms.filter( deleted_at__isnull=True - ).values_list('pk', flat=True) + ).values_list("pk", flat=True) queryset = Instance.objects.filter( - xform_id__in=pks, - deleted_at__isnull=True).values_list('json', flat=True) + xform_id__in=pks, deleted_at__isnull=True + ).values_list("json", flat=True) else: - queryset = xform.instances.values_list('json', flat=True) + queryset = xform.instances.values_list("json", flat=True) paginate_queryset = self.paginate_queryset(queryset) if paginate_queryset: - data['attributes']['responses'] = floip_list(paginate_queryset) + data["attributes"]["responses"] = floip_list(paginate_queryset) response = self.get_paginated_response(data) for key, value in headers.items(): response[key] = value return response - data['attributes']['responses'] = floip_list(queryset) + data["attributes"]["responses"] = floip_list(queryset) return Response(data, headers=headers, status=status_code) diff --git a/onadata/apps/api/viewsets/media_viewset.py b/onadata/apps/api/viewsets/media_viewset.py index a902b37e04..422296c88f 100644 --- a/onadata/apps/api/viewsets/media_viewset.py +++ b/onadata/apps/api/viewsets/media_viewset.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +""" +The /api/v1/media API implementation. + +List, Create, Update, Delete MetaData objects. +""" from django.conf import settings from django.http import Http404 from django.http import HttpResponseRedirect @@ -9,23 +15,28 @@ from rest_framework.exceptions import ParseError from onadata.apps.logger.models import Attachment -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin -from onadata.libs.utils.image_tools import \ - image_url, generate_media_download_url +from onadata.libs.utils.image_tools import image_url, generate_media_download_url from onadata.apps.api.tools import get_baseviewset_class BaseViewset = get_baseviewset_class() -class MediaViewSet(AuthenticateHeaderMixin, - CacheControlMixin, ETagsMixin, BaseViewset, - viewsets.ViewSet): +# pylint: disable=too-many-ancestors +class MediaViewSet( + AuthenticateHeaderMixin, + CacheControlMixin, + ETagsMixin, + BaseViewset, + viewsets.ViewSet, +): """A view to redirect to actual attachments url""" - permission_classes = (AllowAny, ) + permission_classes = (AllowAny,) + + # pylint: disable=invalid-name def retrieve(self, request, pk=None): """ Redirect to final attachment url @@ -40,10 +51,10 @@ def retrieve(self, request, pk=None): """ try: int(pk) - except ValueError: - raise Http404() + except ValueError as exc: + raise Http404() from exc else: - filename = request.query_params.get('filename') + filename = request.query_params.get("filename") attachments = Attachment.objects.all() obj = get_object_or_404(attachments, pk=pk) @@ -52,15 +63,15 @@ def retrieve(self, request, pk=None): url = None - if obj.mimetype.startswith('image'): - suffix = request.query_params.get('suffix') + if obj.mimetype.startswith("image"): + suffix = request.query_params.get("suffix") if suffix: if suffix in list(settings.THUMB_CONF): try: url = image_url(obj, suffix) except Exception as e: - raise ParseError(e) + raise ParseError(e) from e else: raise Http404() @@ -75,7 +86,7 @@ def retrieve(self, request, pk=None): def list(self, request, *args, **kwargs): """ - Action NOT IMPLEMENTED, only needed because of the automatic url - routing in /api/v1/ + Action NOT IMPLEMENTED, only needed because of the automatic url + routing in /api/v1/ """ return Response(data=[]) diff --git a/onadata/apps/api/viewsets/metadata_viewset.py b/onadata/apps/api/viewsets/metadata_viewset.py index 0ce33444ec..7e1299ba3b 100644 --- a/onadata/apps/api/viewsets/metadata_viewset.py +++ b/onadata/apps/api/viewsets/metadata_viewset.py @@ -1,3 +1,9 @@ +# -*- coding: utf-8 -*- +""" +The /api/v1/metadata API implementation. + +List, Create, Update, Delete MetaData objects. +""" from rest_framework import renderers from rest_framework import viewsets from rest_framework.response import Response @@ -7,42 +13,49 @@ from onadata.apps.main.models.meta_data import MetaData from onadata.libs.serializers.metadata_serializer import MetaDataSerializer from onadata.libs import filters -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin -from onadata.libs.renderers.renderers import MediaFileContentNegotiation, \ - MediaFileRenderer +from onadata.libs.renderers.renderers import ( + MediaFileContentNegotiation, + MediaFileRenderer, +) from onadata.apps.api.tools import get_baseviewset_class BaseViewset = get_baseviewset_class() -class MetaDataViewSet(AuthenticateHeaderMixin, - CacheControlMixin, - ETagsMixin, - BaseViewset, - viewsets.ModelViewSet): +# pylint: disable=too-many-ancestors +class MetaDataViewSet( + AuthenticateHeaderMixin, + CacheControlMixin, + ETagsMixin, + BaseViewset, + viewsets.ModelViewSet, +): """ - This endpoint provides access to form metadata. + List, Create, Update, Delete MetaData objects. """ content_negotiation_class = MediaFileContentNegotiation filter_backends = (filters.MetaDataFilter,) - queryset = MetaData.objects.filter( - deleted_at__isnull=True).select_related() + queryset = MetaData.objects.filter(deleted_at__isnull=True).select_related() permission_classes = (MetaDataObjectPermissions,) renderer_classes = ( renderers.JSONRenderer, renderers.BrowsableAPIRenderer, - MediaFileRenderer) + MediaFileRenderer, + ) serializer_class = MetaDataSerializer def retrieve(self, request, *args, **kwargs): + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - if isinstance(request.accepted_renderer, MediaFileRenderer) \ - and self.object.data_file is not None: + if ( + isinstance(request.accepted_renderer, MediaFileRenderer) + and self.object.data_file is not None + ): return get_media_file_response(self.object, request) diff --git a/onadata/apps/api/viewsets/note_viewset.py b/onadata/apps/api/viewsets/note_viewset.py index cf5d210ec7..eaf1f26de8 100644 --- a/onadata/apps/api/viewsets/note_viewset.py +++ b/onadata/apps/api/viewsets/note_viewset.py @@ -1,39 +1,49 @@ +# -*- coding: utf-8 -*- +""" +The /api/v1/notes API implementation. + +List, Create, Update, Delete Note objects. +""" from rest_framework import status -from rest_framework_guardian.filters import ObjectPermissionsFilter from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet +from rest_framework_guardian.filters import ObjectPermissionsFilter from onadata.apps.api import permissions -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin - +from onadata.apps.api.tools import get_baseviewset_class +from onadata.apps.logger.models import Note from onadata.libs import filters +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.serializers.note_serializer import NoteSerializer -from onadata.apps.logger.models import Note -from onadata.apps.api.tools import get_baseviewset_class BaseViewset = get_baseviewset_class() -class NoteViewSet(AuthenticateHeaderMixin, - CacheControlMixin, - ETagsMixin, - BaseViewset, - ModelViewSet): +# pylint: disable=too-many-ancestors +class NoteViewSet( + AuthenticateHeaderMixin, CacheControlMixin, ETagsMixin, BaseViewset, ModelViewSet +): + """ + The /api/v1/notes API implementation. + + List, Create, Update, Delete Note objects.""" + queryset = Note.objects.all() filter_backends = (filters.NoteFilter, ObjectPermissionsFilter) serializer_class = NoteSerializer - permission_classes = [permissions.ViewDjangoObjectPermissions, - permissions.IsAuthenticated, ] + permission_classes = [ + permissions.ViewDjangoObjectPermissions, + permissions.IsAuthenticated, + ] def get_object(self): obj = [] queryset = self.filter_queryset(self.get_queryset()) if queryset: - obj = super(NoteViewSet, self).get_object() + obj = super().get_object() return obj @@ -43,8 +53,7 @@ def retrieve(self, request, *args, **kwargs): if instance: serializer = self.get_serializer(instance) return Response(serializer.data) - else: - return Response(data=[]) + return Response(data=[]) def destroy(self, request, *args, **kwargs): obj = self.get_object() diff --git a/onadata/apps/api/viewsets/open_data_viewset.py b/onadata/apps/api/viewsets/open_data_viewset.py index b634d50909..aec1a4a8c2 100644 --- a/onadata/apps/api/viewsets/open_data_viewset.py +++ b/onadata/apps/api/viewsets/open_data_viewset.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ The /api/v1/open-data implementation. """ @@ -67,7 +67,7 @@ def process_tableau_data(data, xform): # noqa C901 def get_xpath(key, nested_key, index): val = nested_key.split("/") - start_index = key.split("/").__len__() + start_index = len(key.split("/")) nested_key_diff = val[start_index:] xpaths = key + f"[{index}]/" + "/".join(nested_key_diff) return xpaths @@ -176,7 +176,6 @@ class OpenDataViewSet(ETagsMixin, CacheControlMixin, BaseViewset, ModelViewSet): MAX_INSTANCES_PER_REQUEST = 1000 pagination_class = StandardPageNumberPagination - # pylint: disable=no-self-use def get_tableau_type(self, xform_type): """ Returns a tableau-supported type based on a xform type. @@ -290,7 +289,6 @@ def data(self, request, **kwargs): return Response(data) - # pylint: disable=no-self-use def get_streaming_response(self, data): """Get a StreamingHttpResponse response object""" @@ -340,7 +338,6 @@ def schema(self, request, **kwargs): return Response(status=status.HTTP_404_NOT_FOUND) - # pylint: disable=no-self-use @action(methods=["GET"], detail=False) def uuid(self, request, *args, **kwargs): """Respond with the OpenData uuid.""" diff --git a/onadata/apps/api/viewsets/organization_profile_viewset.py b/onadata/apps/api/viewsets/organization_profile_viewset.py index 129d674bbd..28edf3d094 100644 --- a/onadata/apps/api/viewsets/organization_profile_viewset.py +++ b/onadata/apps/api/viewsets/organization_profile_viewset.py @@ -1,127 +1,134 @@ +# -*- coding: utf-8 -*- +""" +The /api/v1/orgs API implementation + +List, Retrieve, Update, Create/Register Organizations. +""" import json + from django.conf import settings -from django.utils.module_loading import import_string from django.core.cache import cache +from django.utils.module_loading import import_string from rest_framework import status -from rest_framework.viewsets import ModelViewSet from rest_framework.decorators import action from rest_framework.response import Response - -from onadata.apps.api.models.organization_profile import OrganizationProfile +from rest_framework.viewsets import ModelViewSet from onadata.apps.api import permissions +from onadata.apps.api.models.organization_profile import OrganizationProfile from onadata.apps.api.tools import get_baseviewset_class -from onadata.libs.utils.common_tools import merge_dicts -from onadata.libs.filters import (OrganizationPermissionFilter, - OrganizationsSharedWithUserFilter) -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.filters import ( + OrganizationPermissionFilter, + OrganizationsSharedWithUserFilter, +) +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.mixins.object_lookup_mixin import ObjectLookupMixin -from onadata.libs.serializers.organization_member_serializer import \ - OrganizationMemberSerializer -from onadata.libs.serializers.organization_serializer import ( - OrganizationSerializer) -from onadata.libs.utils.cache_tools import ( - safe_delete, - ORG_PROFILE_CACHE) - +from onadata.libs.serializers.organization_member_serializer import ( + OrganizationMemberSerializer, +) +from onadata.libs.serializers.organization_serializer import OrganizationSerializer +from onadata.libs.utils.cache_tools import ORG_PROFILE_CACHE, safe_delete +from onadata.libs.utils.common_tools import merge_dicts BaseViewset = get_baseviewset_class() def serializer_from_settings(): + """Return the OrganizationSerializer either from settings or the default.""" if settings.ORG_PROFILE_SERIALIZER: return import_string(settings.ORG_PROFILE_SERIALIZER) return OrganizationSerializer -class OrganizationProfileViewSet(AuthenticateHeaderMixin, - CacheControlMixin, - ETagsMixin, - ObjectLookupMixin, - BaseViewset, - ModelViewSet): +# pylint: disable=too-many-ancestors +class OrganizationProfileViewSet( + AuthenticateHeaderMixin, + CacheControlMixin, + ETagsMixin, + ObjectLookupMixin, + BaseViewset, + ModelViewSet, +): """ List, Retrieve, Update, Create/Register Organizations. """ + queryset = OrganizationProfile.objects.filter(user__is_active=True) serializer_class = serializer_from_settings() - lookup_field = 'user' + lookup_field = "user" permission_classes = [permissions.OrganizationProfilePermissions] - filter_backends = (OrganizationPermissionFilter, - OrganizationsSharedWithUserFilter) + filter_backends = (OrganizationPermissionFilter, OrganizationsSharedWithUserFilter) def retrieve(self, request, *args, **kwargs): - """ Get organization from cache or db """ - username = kwargs.get('user') - cached_org = cache.get(f'{ORG_PROFILE_CACHE}{username}') + """Get organization from cache or db""" + username = kwargs.get("user") + cached_org = cache.get(f"{ORG_PROFILE_CACHE}{username}") if cached_org: return Response(cached_org) - response = super(OrganizationProfileViewSet, self)\ - .retrieve(request, *args, **kwargs) - cache.set(f'{ORG_PROFILE_CACHE}{username}', response.data) + response = super().retrieve(request, *args, **kwargs) + cache.set(f"{ORG_PROFILE_CACHE}{username}", response.data) return response def create(self, request, *args, **kwargs): - """ Create and cache organization """ - response = super(OrganizationProfileViewSet, self)\ - .create(request, *args, **kwargs) + """Create and cache organization""" + response = super().create(request, *args, **kwargs) organization = response.data - username = organization.get('org') - cache.set(f'{ORG_PROFILE_CACHE}{username}', organization) + username = organization.get("org") + cache.set(f"{ORG_PROFILE_CACHE}{username}", organization) return response def destroy(self, request, *args, **kwargs): - """ Clear cache and destroy organization """ - username = kwargs.get('user') - safe_delete(f'{ORG_PROFILE_CACHE}{username}') - return super(OrganizationProfileViewSet, self)\ - .destroy(request, *args, **kwargs) + """Clear cache and destroy organization""" + username = kwargs.get("user") + safe_delete(f"{ORG_PROFILE_CACHE}{username}") + return super().destroy(request, *args, **kwargs) def update(self, request, *args, **kwargs): - """ Update org in cache and db""" - username = kwargs.get('user') - response = super(OrganizationProfileViewSet, self)\ - .update(request, *args, **kwargs) - cache.set(f'{ORG_PROFILE_CACHE}{username}', response.data) + """Update org in cache and db""" + username = kwargs.get("user") + response = super().update(request, *args, **kwargs) + cache.set(f"{ORG_PROFILE_CACHE}{username}", response.data) return response - @action(methods=['DELETE', 'GET', 'POST', 'PUT'], detail=True) + @action(methods=["DELETE", "GET", "POST", "PUT"], detail=True) def members(self, request, *args, **kwargs): + """Return organization members.""" organization = self.get_object() - data = merge_dicts(request.data, - request.query_params.dict(), - {'organization': organization.pk}) + data = merge_dicts( + request.data, request.query_params.dict(), {"organization": organization.pk} + ) - if request.method == 'POST' and 'username' not in data: - data['username'] = None + if request.method == "POST" and "username" not in data: + data["username"] = None - if request.method == 'DELETE': - data['remove'] = True + if request.method == "DELETE": + data["remove"] = True - if request.method == 'PUT' and 'role' not in data: - data['role'] = None + if request.method == "PUT" and "role" not in data: + data["role"] = None serializer = OrganizationMemberSerializer(data=data) - username = kwargs.get('user') + username = kwargs.get("user") if serializer.is_valid(): serializer.save() - organization = serializer.validated_data.get('organization') - data = OrganizationSerializer(organization, - context={'request': request}).data - cache.set(f'{ORG_PROFILE_CACHE}{username}', data) + organization = serializer.validated_data.get("organization") + data = OrganizationSerializer( + organization, context={"request": request} + ).data + cache.set(f"{ORG_PROFILE_CACHE}{username}", data) else: - return Response(data=serializer.errors, - status=status.HTTP_400_BAD_REQUEST) + return Response(data=serializer.errors, status=status.HTTP_400_BAD_REQUEST) + # pylint: disable=attribute-defined-outside-init self.etag_data = json.dumps(data) - resp_status = status.HTTP_201_CREATED if request.method == 'POST' \ - else status.HTTP_200_OK + resp_status = ( + status.HTTP_201_CREATED if request.method == "POST" else status.HTTP_200_OK + ) - return Response(status=resp_status, data=serializer.data()) + return Response(status=resp_status, data=serializer.data) diff --git a/onadata/apps/api/viewsets/project_viewset.py b/onadata/apps/api/viewsets/project_viewset.py index 3c0da0d1c0..872fea5579 100644 --- a/onadata/apps/api/viewsets/project_viewset.py +++ b/onadata/apps/api/viewsets/project_viewset.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ The /projects API endpoint implementation. """ diff --git a/onadata/apps/api/viewsets/stats_viewset.py b/onadata/apps/api/viewsets/stats_viewset.py index aad56aaf1d..bf203ffa4a 100644 --- a/onadata/apps/api/viewsets/stats_viewset.py +++ b/onadata/apps/api/viewsets/stats_viewset.py @@ -1,41 +1,51 @@ +# -*- coding: utf-8 -*- +""" +The /api/v1/stats API endpoint implementaion. +""" from rest_framework import viewsets from onadata.apps.api.permissions import XFormPermissions +from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models.xform import XForm - from onadata.libs import filters from onadata.libs.mixins.anonymous_user_public_forms_mixin import ( - AnonymousUserPublicFormsMixin) -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin + AnonymousUserPublicFormsMixin, +) +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.serializers.stats_serializer import ( - StatsSerializer, StatsInstanceSerializer) -from onadata.apps.api.tools import get_baseviewset_class + StatsInstanceSerializer, + StatsSerializer, +) BaseViewset = get_baseviewset_class() -class StatsViewSet(AuthenticateHeaderMixin, - CacheControlMixin, - ETagsMixin, - AnonymousUserPublicFormsMixin, - BaseViewset, - viewsets.ReadOnlyModelViewSet): - - lookup_field = 'pk' +# pylint: disable=too-many-ancestors +class StatsViewSet( + AuthenticateHeaderMixin, + CacheControlMixin, + ETagsMixin, + AnonymousUserPublicFormsMixin, + BaseViewset, + viewsets.ReadOnlyModelViewSet, +): + """ + The /api/v1/stats API endpoint implementaion. + """ + + lookup_field = "pk" queryset = XForm.objects.all() - filter_backends = (filters.AnonDjangoObjectPermissionFilter, ) - permission_classes = [XFormPermissions, ] + filter_backends = (filters.AnonDjangoObjectPermissionFilter,) + permission_classes = [ + XFormPermissions, + ] serializer_class = StatsSerializer def get_serializer_class(self): lookup = self.kwargs.get(self.lookup_field) if lookup is not None: - serializer_class = StatsInstanceSerializer - else: - serializer_class = \ - super(StatsViewSet, self).get_serializer_class() + return StatsInstanceSerializer - return serializer_class + return super().get_serializer_class() diff --git a/onadata/apps/api/viewsets/submissionstats_viewset.py b/onadata/apps/api/viewsets/submissionstats_viewset.py index 2b80e28fc9..6959a3d139 100644 --- a/onadata/apps/api/viewsets/submissionstats_viewset.py +++ b/onadata/apps/api/viewsets/submissionstats_viewset.py @@ -1,72 +1,83 @@ +# -*- coding: utf-8 -*- +""" +The /api/v1/stats/submissions API endpoint implementation. +""" from rest_framework import viewsets from onadata.apps.api.permissions import XFormPermissions from onadata.apps.logger.models.xform import XForm from onadata.libs import filters from onadata.libs.mixins.anonymous_user_public_forms_mixin import ( - AnonymousUserPublicFormsMixin) -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin + AnonymousUserPublicFormsMixin, +) +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.serializers.stats_serializer import ( - SubmissionStatsSerializer, SubmissionStatsInstanceSerializer) + SubmissionStatsSerializer, + SubmissionStatsInstanceSerializer, +) from onadata.apps.api.tools import get_baseviewset_class BaseViewset = get_baseviewset_class() -class SubmissionStatsViewSet(AnonymousUserPublicFormsMixin, - AuthenticateHeaderMixin, - CacheControlMixin, - ETagsMixin, - BaseViewset, - viewsets.ReadOnlyModelViewSet): +# pylint: disable=too-many-ancestors +class SubmissionStatsViewSet( + AnonymousUserPublicFormsMixin, + AuthenticateHeaderMixin, + CacheControlMixin, + ETagsMixin, + BaseViewset, + viewsets.ReadOnlyModelViewSet, +): """ -Provides submissions counts grouped by a specified field. -It accepts query parameters `group` and `name`. Default result -is grouped by `_submission_time`, hence you get submission counts per day. -If a date field is used as the group, the result will be grouped by day. + Provides submissions counts grouped by a specified field. + It accepts query parameters `group` and `name`. Default result + is grouped by `_submission_time`, hence you get submission counts per day. + If a date field is used as the group, the result will be grouped by day. -* *group* - field to group submission counts by -* *name* - name to be applied to the group on results + * *group* - field to group submission counts by + * *name* - name to be applied to the group on results -Example: + Example: - GET /api/v1/stats/submissions/1? - group=_submission_time&name=day_of_submission + GET /api/v1/stats/submissions/1? + group=_submission_time&name=day_of_submission -Response:: + Response:: - [ - { - "count": 8, - "day_of_submission": "2013-11-15", - }, - { - "count": 99, - "day_of_submission": "2013-11-16", - }, - { - "count": 133, - "day_of_submission": "2013-11-17", - }, - { - "count": 162, - "day_of_submission": "2013-11-18", - }, - { - "count": 102, - "day_of_submission": "2013-11-19", - } - ] + [ + { + "count": 8, + "day_of_submission": "2013-11-15", + }, + { + "count": 99, + "day_of_submission": "2013-11-16", + }, + { + "count": 133, + "day_of_submission": "2013-11-17", + }, + { + "count": 162, + "day_of_submission": "2013-11-18", + }, + { + "count": 102, + "day_of_submission": "2013-11-19", + } + ] + """ -""" - lookup_field = 'pk' + lookup_field = "pk" queryset = XForm.objects.all() - filter_backends = (filters.AnonDjangoObjectPermissionFilter, ) - permission_classes = [XFormPermissions, ] + filter_backends = (filters.AnonDjangoObjectPermissionFilter,) + permission_classes = [ + XFormPermissions, + ] serializer_class = SubmissionStatsSerializer def get_serializer_class(self): @@ -74,7 +85,6 @@ def get_serializer_class(self): if lookup is not None: serializer_class = SubmissionStatsInstanceSerializer else: - serializer_class = \ - super(SubmissionStatsViewSet, self).get_serializer_class() + serializer_class = super().get_serializer_class() return serializer_class diff --git a/onadata/apps/api/viewsets/team_viewset.py b/onadata/apps/api/viewsets/team_viewset.py index 2127746cb6..1b59309b31 100644 --- a/onadata/apps/api/viewsets/team_viewset.py +++ b/onadata/apps/api/viewsets/team_viewset.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ The /teams API endpoint implementation. """ diff --git a/onadata/apps/api/viewsets/user_profile_viewset.py b/onadata/apps/api/viewsets/user_profile_viewset.py index f02ffa851a..846b810a81 100644 --- a/onadata/apps/api/viewsets/user_profile_viewset.py +++ b/onadata/apps/api/viewsets/user_profile_viewset.py @@ -15,8 +15,6 @@ from django.utils.module_loading import import_string from django.utils.translation import gettext as _ -from six.moves.urllib.parse import urlencode - from multidb.pinning import use_master from registration.models import RegistrationProfile from rest_framework import serializers, status @@ -26,6 +24,7 @@ from rest_framework.generics import get_object_or_404 from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet +from six.moves.urllib.parse import urlencode from onadata.apps.api.permissions import UserProfilePermissions from onadata.apps.api.tasks import send_verification_email @@ -328,7 +327,6 @@ def monthly_submissions(self, request, *args, **kwargs): serializer = MonthlySubmissionsSerializer(instance_count, many=True) return Response(serializer.data[0]) - # pylint: disable=no-self-use @action(detail=False) def verify_email(self, request, *args, **kwargs): """Accpet's email verification token and marks the profile as verified.""" @@ -378,7 +376,6 @@ def verify_email(self, request, *args, **kwargs): return HttpResponseBadRequest(response_message) - # pylint: disable=no-self-use @action(methods=["POST"], detail=False) def send_verification_email(self, request, *args, **kwargs): """Sends verification email on user profile registration.""" @@ -398,12 +395,13 @@ def send_verification_email(self, request, *args, **kwargs): except RegistrationProfile.DoesNotExist: pass else: - set_is_email_verified(registration_profile.user.profile, False) + user = registration_profile.user + set_is_email_verified(user.profile, False) verification_key = registration_profile.activation_key if verification_key == verified_key_text: verification_key = ( - registration_profile.user.registrationprofile.create_new_activation_key() + user.registrationprofile.create_new_activation_key() ) verification_url = get_verification_url( @@ -411,8 +409,8 @@ def send_verification_email(self, request, *args, **kwargs): ) email_data = get_verification_email_data( - registration_profile.user.email, - registration_profile.user.username, + user.email, + user.username, verification_url, request, ) diff --git a/onadata/apps/api/viewsets/user_viewset.py b/onadata/apps/api/viewsets/user_viewset.py index e7e28b1a85..33add40dae 100644 --- a/onadata/apps/api/viewsets/user_viewset.py +++ b/onadata/apps/api/viewsets/user_viewset.py @@ -1,16 +1,15 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Users /users API endpoint. """ from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from rest_framework.generics import get_object_or_404 from rest_framework.viewsets import ReadOnlyModelViewSet from rest_framework import filters from onadata.libs.filters import UserNoOrganizationsFilter -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.serializers.user_serializer import UserSerializer @@ -18,21 +17,32 @@ from onadata.apps.api.tools import get_baseviewset_class BaseViewset = get_baseviewset_class() # pylint: disable=invalid-name +User = get_user_model() # pylint: disable=too-many-ancestors -class UserViewSet(AuthenticateHeaderMixin, BaseViewset, CacheControlMixin, - ETagsMixin, ReadOnlyModelViewSet): +class UserViewSet( + AuthenticateHeaderMixin, + BaseViewset, + CacheControlMixin, + ETagsMixin, + ReadOnlyModelViewSet, +): """ This endpoint allows you to list and retrieve user's first and last names. """ + queryset = User.objects.filter(is_active=True).exclude( - username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME, ) + username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME, + ) serializer_class = UserSerializer - lookup_field = 'username' + lookup_field = "username" permission_classes = [permissions.UserViewSetPermissions] - filter_backends = (filters.SearchFilter, UserNoOrganizationsFilter, ) - search_fields = ('=email', ) + filter_backends = ( + filters.SearchFilter, + UserNoOrganizationsFilter, + ) + search_fields = ("=email",) def get_object(self): """Lookup a username by pk else use lookup_field""" @@ -44,9 +54,9 @@ def get_object(self): try: user_id = int(username) except ValueError: - filter_kwargs = {'%s__iexact' % self.lookup_field: username} + filter_kwargs = {f"{self.lookup_field}__iexact": username} else: - filter_kwargs = {'pk': user_id} + filter_kwargs = {"pk": user_id} user = get_object_or_404(queryset, **filter_kwargs) diff --git a/onadata/apps/api/viewsets/v2/tableau_viewset.py b/onadata/apps/api/viewsets/v2/tableau_viewset.py index 158d0ce055..80de83a8c5 100644 --- a/onadata/apps/api/viewsets/v2/tableau_viewset.py +++ b/onadata/apps/api/viewsets/v2/tableau_viewset.py @@ -3,33 +3,33 @@ Implements the /api/v2/tableau endpoint """ import re +from collections import defaultdict from typing import List from rest_framework import status -from collections import defaultdict from rest_framework.decorators import action from rest_framework.response import Response -from onadata.libs.data import parse_int -from onadata.libs.renderers.renderers import pairing -from onadata.apps.logger.models import Instance -from onadata.apps.logger.models.xform import XForm from onadata.apps.api.tools import replace_attachment_name_with_url from onadata.apps.api.viewsets.open_data_viewset import OpenDataViewSet +from onadata.apps.logger.models import Instance +from onadata.apps.logger.models.xform import XForm +from onadata.libs.data import parse_int +from onadata.libs.renderers.renderers import pairing from onadata.libs.serializers.data_serializer import TableauDataSerializer from onadata.libs.utils.common_tags import ( ID, MULTIPLE_SELECT_TYPE, - REPEAT_SELECT_TYPE, - PARENT_TABLE, PARENT_ID, + PARENT_TABLE, + REPEAT_SELECT_TYPE, ) - DEFAULT_TABLE_NAME = "data" GPS_PARTS = ["latitude", "longitude", "altitude", "precision"] +# pylint: disable=too-many-locals def process_tableau_data( data, xform, @@ -37,7 +37,9 @@ def process_tableau_data( parent_id: int = None, current_table: str = DEFAULT_TABLE_NAME, ): + """Returns data formatted for Tableau.""" result = [] + # pylint: disable=too-many-nested-blocks if data: for idx, row in enumerate(data, start=1): flat_dict = defaultdict(list) @@ -96,6 +98,7 @@ def process_tableau_data( def unpack_select_multiple_data(picked_choices, list_name, choice_names, prefix): + """Unpacks select multiple data and returns a dictionary of selected choices.""" unpacked_data = {} for choice in choice_names: qstn_name = f"{list_name}_{choice}" @@ -111,6 +114,7 @@ def unpack_select_multiple_data(picked_choices, list_name, choice_names, prefix) def unpack_repeat_data(repeat_data, flat_dict): + """Prepares repeat data.""" # Pop any list within the returned repeat data. # Lists represent a repeat group which should be in a # separate field. @@ -128,6 +132,7 @@ def unpack_repeat_data(repeat_data, flat_dict): def unpack_gps_data(value, qstn_name, prefix): + """Prepares GPS data.""" value_parts = value.split(" ") gps_xpath_parts = [] for part in GPS_PARTS: @@ -138,9 +143,11 @@ def unpack_gps_data(value, qstn_name, prefix): if len(value_parts) == 4: gps_parts = dict(zip(dict(gps_xpath_parts), value_parts)) return gps_parts + return {} def clean_xform_headers(headers: list) -> list: + """Prepare valid headers for Tableau.""" ret = [] for header in headers: if re.search(r"\[+\d+\]", header): @@ -154,9 +161,15 @@ def clean_xform_headers(headers: list) -> list: return ret +# pylint: disable=too-many-ancestors class TableauViewSet(OpenDataViewSet): + """ + TableauViewSet - the /api/v2/tableau API endpoin implementation. + """ + @action(methods=["GET"], detail=True) def data(self, request, **kwargs): + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() # get greater than value and cast it to an int gt_id = request.query_params.get("gt_id") @@ -167,7 +180,7 @@ def data(self, request, **kwargs): self.paginator.page_size_query_param, ] query_param_keys = request.query_params - should_paginate = any([k in query_param_keys for k in pagination_keys]) + should_paginate = any(k in query_param_keys for k in pagination_keys) data = [] if isinstance(self.object.content_object, XForm): @@ -207,7 +220,7 @@ def data(self, request, **kwargs): return Response(data) - # pylint: disable=arguments-differ + # pylint: disable=arguments-differ,too-many-locals def flatten_xform_columns( self, json_of_columns_fields, table: str = None, field_prefix: str = None ): @@ -230,8 +243,8 @@ def flatten_xform_columns( columns = self.flatten_xform_columns( field.get("children"), table=table_name, field_prefix=prefix ) - for key in columns.keys(): - ret[key].extend(columns[key]) + for key, val in columns.items(): + ret[key].extend(val) elif field_type == MULTIPLE_SELECT_TYPE: for option in field.get("children"): list_name = field.get("list_name") @@ -288,6 +301,7 @@ def get_tableau_column_headers(self): return tableau_column_headers def get_tableau_table_schemas(self) -> List[dict]: + """Return a list of Tableau table schemas.""" ret = [] project = self.xform.project_id id_str = self.xform.id_string @@ -304,6 +318,7 @@ def get_tableau_table_schemas(self) -> List[dict]: @action(methods=["GET"], detail=True) def schema(self, request, **kwargs): + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() if isinstance(self.object.content_object, XForm): self.xform = self.object.content_object diff --git a/onadata/apps/api/viewsets/widget_viewset.py b/onadata/apps/api/viewsets/widget_viewset.py index 8c28a3bc70..1a73b427fb 100644 --- a/onadata/apps/api/viewsets/widget_viewset.py +++ b/onadata/apps/api/viewsets/widget_viewset.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Expose and persist charts and corresponding data. +""" from django.shortcuts import get_object_or_404 from django.utils.translation import gettext as _ from django.contrib.contenttypes.models import ContentType @@ -7,8 +11,7 @@ from rest_framework.exceptions import ParseError from onadata.libs import filters -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.cache_control_mixin import CacheControlMixin from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.apps.logger.models.widget import Widget @@ -20,37 +23,44 @@ BaseViewset = get_baseviewset_class() -class WidgetViewSet(AuthenticateHeaderMixin, - CacheControlMixin, ETagsMixin, BaseViewset, ModelViewSet): +# pylint: disable=too-many-ancestors +class WidgetViewSet( + AuthenticateHeaderMixin, CacheControlMixin, ETagsMixin, BaseViewset, ModelViewSet +): + """ + Expose and persist charts and corresponding data. + """ + queryset = Widget.objects.all() serializer_class = WidgetSerializer permission_classes = [WidgetViewSetPermissions] - lookup_field = 'pk' + lookup_field = "pk" filter_backends = (filters.WidgetFilter,) def filter_queryset(self, queryset): - dataviewid = self.request.query_params.get('dataview') + dataviewid = self.request.query_params.get("dataview") if dataviewid: try: int(dataviewid) - except ValueError: - raise ParseError( - u"Invalid value for dataview %s." % dataviewid) + except ValueError as exc: + raise ParseError(f"Invalid value for dataview {dataviewid}.") from exc dataview = get_object_or_404(DataView, pk=dataviewid) dataview_ct = ContentType.objects.get_for_model(dataview) - dataview_qs = Widget.objects.filter(object_id=dataview.pk, - content_type=dataview_ct) + dataview_qs = Widget.objects.filter( + object_id=dataview.pk, content_type=dataview_ct + ) return dataview_qs - return super(WidgetViewSet, self).filter_queryset(queryset) + return super().filter_queryset(queryset) + # pylint: disable=unused-argument def get_object(self, queryset=None): - pk = self.kwargs.get('pk') + widget_pk = self.kwargs.get("pk") - if pk is not None: - obj = get_object_or_404(Widget, pk=pk) + if widget_pk is not None: + obj = get_object_or_404(Widget, pk=widget_pk) self.check_object_permissions(self.request, obj) else: raise ParseError(_("'pk' required for this action")) @@ -58,12 +68,12 @@ def get_object(self, queryset=None): return obj def list(self, request, *args, **kwargs): - if 'key' in request.GET: - key = request.GET['key'] + if "key" in request.GET: + key = request.GET["key"] obj = get_object_or_404(Widget, key=key) serializer = self.get_serializer(obj) return Response(serializer.data) - return super(WidgetViewSet, self).list(request, *args, **kwargs) + return super().list(request, *args, **kwargs) diff --git a/onadata/apps/api/viewsets/xform_list_viewset.py b/onadata/apps/api/viewsets/xform_list_viewset.py index 3f9a406709..3fe92295de 100644 --- a/onadata/apps/api/viewsets/xform_list_viewset.py +++ b/onadata/apps/api/viewsets/xform_list_viewset.py @@ -1,58 +1,70 @@ +# -*- coding: utf-8 -*- +""" +OpenRosa Form List API - https://docs.getodk.org/openrosa-form-list/ +""" from django.conf import settings from django.http import Http404 from django.shortcuts import get_object_or_404 from django.views.decorators.cache import never_cache -from django_filters import rest_framework as django_filter_filters -from rest_framework import viewsets -from rest_framework import permissions +from django_filters import rest_framework as django_filter_filters +from rest_framework import permissions, viewsets from rest_framework.decorators import action from rest_framework.response import Response -from onadata.apps.api.tools import get_media_file_response +from onadata.apps.api.tools import get_baseviewset_class, get_media_file_response from onadata.apps.logger.models.xform import XForm, get_forms_shared_with_user from onadata.apps.main.models.meta_data import MetaData from onadata.apps.main.models.user_profile import UserProfile from onadata.libs import filters -from onadata.libs.authentication import DigestAuthentication -from onadata.libs.authentication import EnketoTokenAuthentication +from onadata.libs.authentication import DigestAuthentication, EnketoTokenAuthentication from onadata.libs.mixins.etags_mixin import ETagsMixin from onadata.libs.mixins.openrosa_headers_mixin import get_openrosa_headers -from onadata.libs.renderers.renderers import MediaFileContentNegotiation -from onadata.libs.renderers.renderers import XFormListRenderer -from onadata.libs.renderers.renderers import XFormManifestRenderer -from onadata.libs.serializers.xform_serializer import XFormListSerializer -from onadata.libs.serializers.xform_serializer import XFormManifestSerializer -from onadata.apps.api.tools import get_baseviewset_class -from onadata.libs.utils.export_tools import ExportBuilder -from onadata.libs.utils.common_tags import (GROUP_DELIMETER_TAG, - REPEAT_INDEX_TAGS) - +from onadata.libs.renderers.renderers import ( + MediaFileContentNegotiation, + XFormListRenderer, + XFormManifestRenderer, +) +from onadata.libs.serializers.xform_serializer import ( + XFormListSerializer, + XFormManifestSerializer, +) +from onadata.libs.utils.common_tags import GROUP_DELIMETER_TAG, REPEAT_INDEX_TAGS +from onadata.libs.utils.export_builder import ExportBuilder BaseViewset = get_baseviewset_class() # 10,000,000 bytes -DEFAULT_CONTENT_LENGTH = getattr(settings, 'DEFAULT_CONTENT_LENGTH', 10000000) +DEFAULT_CONTENT_LENGTH = getattr(settings, "DEFAULT_CONTENT_LENGTH", 10000000) -class XFormListViewSet(ETagsMixin, BaseViewset, - viewsets.ReadOnlyModelViewSet): - authentication_classes = (DigestAuthentication, - EnketoTokenAuthentication,) +# pylint: disable=too-many-ancestors +class XFormListViewSet(ETagsMixin, BaseViewset, viewsets.ReadOnlyModelViewSet): + """ + OpenRosa Form List API - https://docs.getodk.org/openrosa-form-list/ + """ + + authentication_classes = ( + DigestAuthentication, + EnketoTokenAuthentication, + ) content_negotiation_class = MediaFileContentNegotiation filter_class = filters.FormIDFilter - filter_backends = (filters.XFormListObjectPermissionFilter, - filters.XFormListXFormPKFilter, - django_filter_filters.DjangoFilterBackend) + filter_backends = ( + filters.XFormListObjectPermissionFilter, + filters.XFormListXFormPKFilter, + django_filter_filters.DjangoFilterBackend, + ) queryset = XForm.objects.filter( - downloadable=True, deleted_at=None, - is_merged_dataset=False).only('id_string', 'title', 'version', 'uuid', - 'description', 'user__username', 'hash') + downloadable=True, deleted_at=None, is_merged_dataset=False + ).only( + "id_string", "title", "version", "uuid", "description", "user__username", "hash" + ) permission_classes = (permissions.AllowAny,) renderer_classes = (XFormListRenderer,) serializer_class = XFormListSerializer - template_name = 'api/xformsList.xml' + template_name = "api/xformsList.xml" def get_object(self): queryset = self.filter_queryset(self.get_queryset()) @@ -66,24 +78,24 @@ def get_object(self): return obj def get_renderers(self): - if self.action and self.action == 'manifest': + if self.action and self.action == "manifest": return [XFormManifestRenderer()] - return super(XFormListViewSet, self).get_renderers() + return super().get_renderers() def filter_queryset(self, queryset): - username = self.kwargs.get('username') - form_pk = self.kwargs.get('xform_pk') - project_pk = self.kwargs.get('project_pk') - if (not username and not form_pk and not project_pk) and \ - self.request.user.is_anonymous: + username = self.kwargs.get("username") + form_pk = self.kwargs.get("xform_pk") + project_pk = self.kwargs.get("project_pk") + if ( + not username and not form_pk and not project_pk + ) and self.request.user.is_anonymous: # raises a permission denied exception, forces authentication self.permission_denied(self.request) profile = None if username: - profile = get_object_or_404( - UserProfile, user__username__iexact=username) + profile = get_object_or_404(UserProfile, user__username__iexact=username) elif form_pk: queryset = queryset.filter(pk=form_pk) if queryset.first(): @@ -97,78 +109,96 @@ def filter_queryset(self, queryset): # raises a permission denied exception, forces authentication self.permission_denied(self.request) else: - queryset = queryset.filter( - user=profile.user, downloadable=True) + queryset = queryset.filter(user=profile.user, downloadable=True) - queryset = super(XFormListViewSet, self).filter_queryset(queryset) + queryset = super().filter_queryset(queryset) if not self.request.user.is_anonymous: - xform_pk = self.kwargs.get('xform_pk') - if self.action == 'list' and profile and xform_pk is None\ - and project_pk is None: - forms_shared_with_user = get_forms_shared_with_user( - profile.user) - id_string = self.request.query_params.get('formID') - forms_shared_with_user = forms_shared_with_user.filter( - id_string=id_string) if id_string else \ - forms_shared_with_user + xform_pk = self.kwargs.get("xform_pk") + if ( + self.action == "list" + and profile + and xform_pk is None + and project_pk is None + ): + forms_shared_with_user = get_forms_shared_with_user(profile.user) + id_string = self.request.query_params.get("formID") + forms_shared_with_user = ( + forms_shared_with_user.filter(id_string=id_string) + if id_string + else forms_shared_with_user + ) queryset = queryset | forms_shared_with_user if self.request.user != profile.user: public_forms = profile.user.xforms.filter( - downloadable=True, shared=True) + downloadable=True, shared=True + ) queryset = queryset | public_forms return queryset @never_cache def list(self, request, *args, **kwargs): + # pylint: disable=attribute-defined-outside-init self.object_list = self.filter_queryset(self.get_queryset()) headers = get_openrosa_headers(request, location=False) serializer = self.get_serializer(self.object_list, many=True) - if request.method in ['HEAD']: - return Response('', headers=headers, status=204) + if request.method in ["HEAD"]: + return Response("", headers=headers, status=204) return Response(serializer.data, headers=headers) def retrieve(self, request, *args, **kwargs): + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - return Response(self.object.xml, - headers=get_openrosa_headers(request, location=False)) + return Response( + self.object.xml, headers=get_openrosa_headers(request, location=False) + ) - @action(methods=['GET', 'HEAD'], detail=True) + @action(methods=["GET", "HEAD"], detail=True) def manifest(self, request, *args, **kwargs): + """A manifest defining additional supporting objects.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - object_list = MetaData.objects.filter(data_type='media', - object_id=self.object.pk) + object_list = MetaData.objects.filter( + data_type="media", object_id=self.object.pk + ) context = self.get_serializer_context() context[GROUP_DELIMETER_TAG] = ExportBuilder.GROUP_DELIMITER_DOT - context[REPEAT_INDEX_TAGS] = '_,_' - serializer = XFormManifestSerializer(object_list, many=True, - context=context) + context[REPEAT_INDEX_TAGS] = "_,_" + serializer = XFormManifestSerializer(object_list, many=True, context=context) - return Response(serializer.data, - headers=get_openrosa_headers(request, location=False)) + return Response( + serializer.data, headers=get_openrosa_headers(request, location=False) + ) - @action(methods=['GET', 'HEAD'], detail=True) + @action(methods=["GET", "HEAD"], detail=True) def media(self, request, *args, **kwargs): + """Returns the media file contents.""" + # pylint: disable=attribute-defined-outside-init self.object = self.get_object() - pk = kwargs.get('metadata') + media_pk = kwargs.get("metadata") - if not pk: + if not media_pk: raise Http404() meta_obj = get_object_or_404( - MetaData, data_type='media', object_id=self.object.pk, pk=pk) + MetaData, data_type="media", object_id=self.object.pk, pk=media_pk + ) response = get_media_file_response(meta_obj, request) if response.status_code == 403 and request.user.is_anonymous: # raises a permission denied exception, forces authentication self.permission_denied(request) - else: - return response + + return response class PreviewXFormListViewSet(XFormListViewSet): + """ + OpenRosa Form List API - for preview purposes only + """ + filter_backends = (filters.AnonDjangoObjectPermissionFilter,) permission_classes = (permissions.AllowAny,) diff --git a/onadata/apps/api/viewsets/xform_submission_viewset.py b/onadata/apps/api/viewsets/xform_submission_viewset.py index c05d3df4f8..df1629ebb7 100644 --- a/onadata/apps/api/viewsets/xform_submission_viewset.py +++ b/onadata/apps/api/viewsets/xform_submission_viewset.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ XFormSubmissionViewSet module """ @@ -7,8 +7,7 @@ from django.utils.translation import gettext as _ from rest_framework import mixins, permissions, status, viewsets -from rest_framework.authentication import (BasicAuthentication, - TokenAuthentication) +from rest_framework.authentication import BasicAuthentication, TokenAuthentication from rest_framework.parsers import FormParser, JSONParser, MultiPartParser from rest_framework.renderers import BrowsableAPIRenderer, JSONRenderer from rest_framework.response import Response @@ -17,51 +16,67 @@ from onadata.apps.api.tools import get_baseviewset_class from onadata.apps.logger.models import Instance from onadata.libs import filters -from onadata.libs.authentication import (DigestAuthentication, - EnketoTokenAuthentication) -from onadata.libs.mixins.authenticate_header_mixin import \ - AuthenticateHeaderMixin +from onadata.libs.authentication import DigestAuthentication, EnketoTokenAuthentication +from onadata.libs.mixins.authenticate_header_mixin import AuthenticateHeaderMixin from onadata.libs.mixins.openrosa_headers_mixin import OpenRosaHeadersMixin from onadata.libs.renderers.renderers import FLOIPRenderer, TemplateXMLRenderer from onadata.libs.serializers.data_serializer import ( - FLOIPSubmissionSerializer, JSONSubmissionSerializer, - RapidProSubmissionSerializer, SubmissionSerializer, - RapidProJSONSubmissionSerializer) -from onadata.libs.utils.logger_tools import (OpenRosaResponseBadRequest, - OpenRosaNotAuthenticated) - -BaseViewset = get_baseviewset_class() # pylint: disable=C0103 + FLOIPSubmissionSerializer, + JSONSubmissionSerializer, + RapidProSubmissionSerializer, + SubmissionSerializer, + RapidProJSONSubmissionSerializer, +) +from onadata.libs.utils.logger_tools import ( + OpenRosaResponseBadRequest, + OpenRosaNotAuthenticated, +) + +BaseViewset = get_baseviewset_class() # pylint: disable=invalid-name # 10,000,000 bytes -DEFAULT_CONTENT_LENGTH = getattr(settings, 'DEFAULT_CONTENT_LENGTH', 10000000) -FLOIP_RESULTS_CONTENT_TYPE = 'application/vnd.org.flowinterop.results+json' +DEFAULT_CONTENT_LENGTH = getattr(settings, "DEFAULT_CONTENT_LENGTH", 10000000) +FLOIP_RESULTS_CONTENT_TYPE = "application/vnd.org.flowinterop.results+json" class FLOIPParser(JSONParser): # pylint: disable=too-few-public-methods """ Flow Results JSON parser. """ + media_type = FLOIP_RESULTS_CONTENT_TYPE renderer_classes = FLOIPRenderer -class XFormSubmissionViewSet(AuthenticateHeaderMixin, # pylint: disable=R0901 - OpenRosaHeadersMixin, - mixins.CreateModelMixin, - BaseViewset, - viewsets.GenericViewSet): +# pylint: disable=too-many-ancestors +class XFormSubmissionViewSet( + AuthenticateHeaderMixin, # pylint: disable=too-many-ancestors + OpenRosaHeadersMixin, + mixins.CreateModelMixin, + BaseViewset, + viewsets.GenericViewSet, +): """ XFormSubmissionViewSet class """ - authentication_classes = (DigestAuthentication, BasicAuthentication, - TokenAuthentication, EnketoTokenAuthentication) - filter_backends = (filters.AnonDjangoObjectPermissionFilter, ) + + authentication_classes = ( + DigestAuthentication, + BasicAuthentication, + TokenAuthentication, + EnketoTokenAuthentication, + ) + filter_backends = (filters.AnonDjangoObjectPermissionFilter,) model = Instance permission_classes = (permissions.AllowAny, IsAuthenticatedSubmission) - renderer_classes = (TemplateXMLRenderer, JSONRenderer, - BrowsableAPIRenderer, FLOIPRenderer) + renderer_classes = ( + TemplateXMLRenderer, + JSONRenderer, + BrowsableAPIRenderer, + FLOIPRenderer, + ) serializer_class = SubmissionSerializer - template_name = 'submission.xml' + template_name = "submission.xml" parser_classes = (FLOIPParser, JSONParser, FormParser, MultiPartParser) def get_serializer(self, *args, **kwargs): @@ -71,12 +86,10 @@ def get_serializer(self, *args, **kwargs): data = kwargs.get("data") content_type = self.request.content_type.lower() - if (isinstance(data, list) - and FLOIP_RESULTS_CONTENT_TYPE in content_type): + if isinstance(data, list) and FLOIP_RESULTS_CONTENT_TYPE in content_type: kwargs["many"] = True - return super(XFormSubmissionViewSet, self).get_serializer( - *args, **kwargs) + return super().get_serializer(*args, **kwargs) def get_serializer_class(self): """ @@ -84,16 +97,15 @@ def get_serializer_class(self): """ content_type = self.request.content_type.lower() - if 'application/json' in content_type: - if 'RapidProMailroom' in self.request.headers.get( - 'User-Agent', ''): + if "application/json" in content_type: + if "RapidProMailroom" in self.request.headers.get("User-Agent", ""): return RapidProJSONSubmissionSerializer self.request.accepted_renderer = JSONRenderer() - self.request.accepted_media_type = 'application/json' + self.request.accepted_media_type = "application/json" return JSONSubmissionSerializer - if 'application/x-www-form-urlencoded' in content_type: + if "application/x-www-form-urlencoded" in content_type: return RapidProSubmissionSerializer if FLOIP_RESULTS_CONTENT_TYPE in content_type: @@ -104,32 +116,32 @@ def get_serializer_class(self): return SubmissionSerializer def create(self, request, *args, **kwargs): - if request.method.upper() == 'HEAD': + if request.method.upper() == "HEAD": return Response( - status=status.HTTP_204_NO_CONTENT, - template_name=self.template_name) + status=status.HTTP_204_NO_CONTENT, template_name=self.template_name + ) - return super(XFormSubmissionViewSet, self).create( - request, *args, **kwargs) + return super().create(request, *args, **kwargs) def handle_exception(self, exc): """ Handles exceptions thrown by handler method and returns appropriate error response. """ - if hasattr(exc, 'response'): + if hasattr(exc, "response"): return exc.response if isinstance(exc, UnreadablePostError): return OpenRosaResponseBadRequest( - _(u"Unable to read submitted file, please try re-submitting.")) + _("Unable to read submitted file, please try re-submitting.") + ) try: if exc.status_code == 401: auth_header = self.get_authenticate_header(self.request) response = OpenRosaNotAuthenticated( data=exc.detail, - headers={'WWW-Authenticate': auth_header}, + headers={"WWW-Authenticate": auth_header}, ) response.exception = True return response @@ -137,4 +149,4 @@ def handle_exception(self, exc): # 'Http404' object has no attribute 'status_code' pass - return super(XFormSubmissionViewSet, self).handle_exception(exc) + return super().handle_exception(exc) diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index a55135c8e2..2735b4cc8d 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -450,7 +450,6 @@ def form(self, request, **kwargs): return response - # pylint: disable=no-self-use # pylint: disable=unused-argument @action(methods=["GET"], detail=False) def login(self, request, **kwargs): diff --git a/onadata/apps/logger/factory.py b/onadata/apps/logger/factory.py index 7ce6529203..9236a67ca8 100644 --- a/onadata/apps/logger/factory.py +++ b/onadata/apps/logger/factory.py @@ -95,7 +95,6 @@ def create_registration_xform(self): return xform - # pylint: disable=no-self-use def get_registration_xform(self): """ Gets a registration xform. (currently loaded in from fixture) diff --git a/onadata/apps/logger/import_tools.py b/onadata/apps/logger/import_tools.py index f25d5b8a71..099ba76810 100644 --- a/onadata/apps/logger/import_tools.py +++ b/onadata/apps/logger/import_tools.py @@ -12,7 +12,7 @@ from django.http.response import Http404 from onadata.apps.logger.xform_fs import XFormInstanceFS -from onadata.celery import app +from onadata.celeryapp import app from onadata.libs.utils.logger_tools import create_instance # odk @@ -110,6 +110,7 @@ def iterate_through_instances( success_count = 0 errors = [] + # pylint: disable=too-many-nested-blocks for directory, _subdirs, subfiles in os.walk(dirpath): for filename in subfiles: filepath = os.path.join(directory, filename) diff --git a/onadata/apps/logger/management/commands/add_id.py b/onadata/apps/logger/management/commands/add_id.py index fe8651a7d8..f05434176f 100644 --- a/onadata/apps/logger/management/commands/add_id.py +++ b/onadata/apps/logger/management/commands/add_id.py @@ -1,4 +1,8 @@ -from django.contrib.auth.models import User +# -*- coding: utf-8 -*- +""" +Sync account with '_id' +""" +from django.contrib.auth import get_user_model from django.utils.translation import gettext_lazy from django.core.management.base import BaseCommand from django.conf import settings @@ -7,7 +11,12 @@ from onadata.libs.utils.model_tools import queryset_iterator +User = get_user_model() + + class Command(BaseCommand): + """Sync account with '_id'""" + args = "" help = gettext_lazy("Sync account with '_id'") @@ -18,7 +27,7 @@ def handle(self, *args, **kwargs): users = User.objects.filter(username__contains=args[0]) else: # All the accounts - self.stdout.write("Fetching all the account {}", ending="\n") + self.stdout.write("Fetching all the accounts.", ending="\n") users = User.objects.exclude( username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME ) @@ -27,7 +36,8 @@ def handle(self, *args, **kwargs): self.add_id(user) def add_id(self, user): - self.stdout.write("Syncing for account {}".format(user.username), ending="\n") + """Append _id in submissions for the specifing ``user``.""" + self.stdout.write(f"Syncing for account {user.username}", ending="\n") xforms = XForm.objects.filter(user=user) count = 0 @@ -40,14 +50,13 @@ def add_id(self, user): try: instance.save() count += 1 + # pylint: disable=broad-except except Exception as e: failed += 1 self.stdout.write(str(e), ending="\n") - pass self.stdout.write( - "Syncing for account {}. Done. Success {}, Fail {}".format( - user.username, count, failed - ), + f"Syncing for account {user.username}. Done. " + f"Success {count}, Fail {failed}", ending="\n", ) diff --git a/onadata/apps/logger/management/commands/change_s3_media_permissions.py b/onadata/apps/logger/management/commands/change_s3_media_permissions.py index 7fa04adc28..1956f03447 100644 --- a/onadata/apps/logger/management/commands/change_s3_media_permissions.py +++ b/onadata/apps/logger/management/commands/change_s3_media_permissions.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ change_s3_media_permissions - makes all s3 files private. """ diff --git a/onadata/apps/logger/management/commands/create_image_thumbnails.py b/onadata/apps/logger/management/commands/create_image_thumbnails.py index 5c77b21d8b..eeb6b80575 100644 --- a/onadata/apps/logger/management/commands/create_image_thumbnails.py +++ b/onadata/apps/logger/management/commands/create_image_thumbnails.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ create_image_thumbnails - creates thumbnails for all form images and stores them. @@ -18,6 +17,8 @@ THUMB_CONF = settings.THUMB_CONF +User = get_user_model() + class Command(BaseCommand): """Creates thumbnails for all form images and stores them""" @@ -43,8 +44,6 @@ def handle(self, *args, **options): attachments_qs = Attachment.objects.select_related( "instance", "instance__xform" ) - # pylint: disable=invalid-name - User = get_user_model() if options.get("username"): username = options.get("username") try: @@ -61,19 +60,23 @@ def handle(self, *args, **options): f"Error: Form with id_string {id_string} does not exist" ) from e attachments_qs = attachments_qs.filter(instance__xform=xform) - fs = get_storage_class("django.core.files.storage.FileSystemStorage")() + file_storage = get_storage_class( + "django.core.files.storage.FileSystemStorage" + )() for att in queryset_iterator(attachments_qs): filename = att.media_file.name default_storage = get_storage_class()() full_path = get_path(filename, settings.THUMB_CONF["small"]["suffix"]) if options.get("force") is not None: - for s in ["small", "medium", "large"]: - fp = get_path(filename, settings.THUMB_CONF[s]["suffix"]) - if default_storage.exists(fp): - default_storage.delete(fp) + for suffix in ["small", "medium", "large"]: + file_path = get_path( + filename, settings.THUMB_CONF[suffix]["suffix"] + ) + if default_storage.exists(file_path): + default_storage.delete(file_path) if not default_storage.exists(full_path): try: - if default_storage.__class__ != fs.__class__: + if default_storage.__class__ != file_storage.__class__: resize(filename, att.extension) else: resize_local_env(filename, att.extension) diff --git a/onadata/apps/logger/management/commands/export_gps_points.py b/onadata/apps/logger/management/commands/export_gps_points.py index fec5259d0a..a1a51666f5 100644 --- a/onadata/apps/logger/management/commands/export_gps_points.py +++ b/onadata/apps/logger/management/commands/export_gps_points.py @@ -1,4 +1,7 @@ -#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Export all gps points with their timestamps +""" import csv from django.core.management.base import BaseCommand @@ -9,10 +12,12 @@ class Command(BaseCommand): + """Export all gps points with their timestamps""" + help = gettext_lazy("Export all gps points with their timestamps") def handle(self, *args, **kwargs): - with open("gps_points_export.csv", "w") as csvfile: + with open("gps_points_export.csv", "w", encoding="utf-8") as csvfile: fieldnames = ["longitude", "latitude", "date_created"] writer = csv.writer(csvfile) writer.writerow(fieldnames) diff --git a/onadata/apps/logger/management/commands/export_xforms_and_instances.py b/onadata/apps/logger/management/commands/export_xforms_and_instances.py index 28c7088473..3a36285c59 100644 --- a/onadata/apps/logger/management/commands/export_xforms_and_instances.py +++ b/onadata/apps/logger/management/commands/export_xforms_and_instances.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ export_xformx_and_instances - exports XForms and submission instances into JSON files. """ diff --git a/onadata/apps/logger/management/commands/fix_attachments_counts.py b/onadata/apps/logger/management/commands/fix_attachments_counts.py index edc2b203a6..747cdc7418 100644 --- a/onadata/apps/logger/management/commands/fix_attachments_counts.py +++ b/onadata/apps/logger/management/commands/fix_attachments_counts.py @@ -1,10 +1,10 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Fix submission media count command. """ import os -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError from django.utils.translation import gettext as _ from django.utils.translation import gettext_lazy @@ -16,6 +16,9 @@ from onadata.libs.utils.model_tools import queryset_iterator +User = get_user_model() + + def update_attachments(instance): """ Takes an Instance object and updates attachment tracking fields @@ -42,15 +45,15 @@ def add_arguments(self, parser): def handle(self, *args, **options): try: username = options["username"] - except KeyError: + except KeyError as exc: raise CommandError( _("You must provide the username to publish the form to.") - ) + ) from exc # make sure user exists try: user = User.objects.get(username=username) - except User.DoesNotExist: - raise CommandError(_("The user '%s' does not exist.") % username) + except User.DoesNotExist as exc: + raise CommandError(_(f"The user '{username}' does not exist.")) from exc self.process_attachments(user) @@ -70,6 +73,6 @@ def process_attachments(self, user): update_attachments(submission) not_processed = xform.instances.filter(media_all_received=False).count() self.stdout.write( - "%s to process %s - %s = %s processed" - % (xform, to_process, not_processed, (to_process - not_processed)) + f"{xform} to process {to_process} - {not_processed} " + f"= {to_process - not_processed} processed" ) diff --git a/onadata/apps/logger/management/commands/fix_duplicate_instances.py b/onadata/apps/logger/management/commands/fix_duplicate_instances.py index 2dfab73a58..d309b2d7b5 100644 --- a/onadata/apps/logger/management/commands/fix_duplicate_instances.py +++ b/onadata/apps/logger/management/commands/fix_duplicate_instances.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 - +""" +Fix duplicate instances by merging the attachments. +""" from django.core.management.base import BaseCommand from django.db import connection from django.utils.translation import gettext_lazy @@ -9,14 +11,18 @@ class Command(BaseCommand): + """Fix duplicate instances by merging the attachments.""" + help = gettext_lazy("Fix duplicate instances by merging the attachments.") def query_data(self, sql): + """Return results of given ``sql`` query.""" cursor = connection.cursor() cursor.execute(sql) for row in cursor.fetchall(): yield row + # pylint: disable=too-many-locals def handle(self, *args, **kwargs): sql = ( "select xform_id, uuid, COUNT(xform_id || uuid) " @@ -42,8 +48,8 @@ def handle(self, *args, **kwargs): # is a match, let's maintain the submission with the # correct xform.uuid. if is_mspray_form and not all_matches: - first = instances.filter(xml__contains=i.xform.uuid).first() - to_delete = instances.exclude(xml__contains=i.xform.uuid) + first = instances.filter(xml__contains=xform.uuid).first() + to_delete = instances.exclude(xml__contains=xform.uuid) else: to_delete = instances.exclude(pk=first.pk) @@ -58,14 +64,14 @@ def handle(self, *args, **kwargs): if delete_count >= dupes_count: raise AssertionError( - "# of records to delete %d should be less than total # of " - "duplicates %d." % (delete_count, dupes_count) + f"# of records to delete {delete_count} should be less than" + f" total # of duplicates {dupes_count}." ) to_delete.delete() total_count += dupes_count total_deleted += delete_count self.stdout.write( - "deleted %d: %s (%d of %d)." % (xform, uuid, delete_count, dupes_count) + f"deleted {xform}: {uuid} ({delete_count} of {dupes_count})." ) - self.stdout.write("done: deleted %d of %d" % (total_deleted, total_count)) + self.stdout.write(f"done: deleted {total_deleted} of {total_count}") diff --git a/onadata/apps/logger/management/commands/fix_submission_count.py b/onadata/apps/logger/management/commands/fix_submission_count.py index 5cc99a0c17..93faa9c593 100644 --- a/onadata/apps/logger/management/commands/fix_submission_count.py +++ b/onadata/apps/logger/management/commands/fix_submission_count.py @@ -1,5 +1,8 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 +""" +Fix num of submissions +""" from django.core.management.base import BaseCommand from django.db import transaction @@ -11,6 +14,8 @@ class Command(BaseCommand): + """Fix num of submissions""" + help = gettext_lazy("Fix num of submissions") def handle(self, *args, **kwargs): @@ -23,9 +28,7 @@ def handle(self, *args, **kwargs): xform.save(update_fields=["num_of_submissions"]) i += 1 self.stdout.write( - "Processing {} of {}: {} ({})".format( - i, xform_count, xform.id_string, instance_count - ) + f"Processing {i} of {xform_count}: {xform.id_string} ({instance_count})" ) i = 0 @@ -39,7 +42,6 @@ def handle(self, *args, **kwargs): profile.save(update_fields=["num_of_submissions"]) i += 1 self.stdout.write( - "Processing {} of {}: {} ({})".format( - i, profile_count, profile.user.username, instance_count - ) + f"Processing {i} of {profile_count}: {profile.user.username} " + f"({instance_count})" ) diff --git a/onadata/apps/logger/management/commands/generate_platform_stats.py b/onadata/apps/logger/management/commands/generate_platform_stats.py index a00dab92c0..e6b23afec0 100644 --- a/onadata/apps/logger/management/commands/generate_platform_stats.py +++ b/onadata/apps/logger/management/commands/generate_platform_stats.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Management command used to generate platform statistics containing information about the number of organizations, users, projects @@ -5,11 +6,13 @@ """ import calendar import csv -from datetime import date, datetime +import os.path +from datetime import datetime -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand from django.db.models import Q +from django.utils import timezone from django.utils.translation import gettext as _ from multidb.pinning import use_master @@ -18,54 +21,62 @@ from onadata.apps.logger.models import Instance, XForm from onadata.libs.permissions import is_organization +User = get_user_model() -def _write_stats_to_file(month: int, year: int, include_extra: bool): - out_file = open(f"/tmp/platform_statistics_{month}_{year}.csv", "w") # nosec - writer = csv.writer(out_file) - headers = ["Username", "Project Name", "Form Title", "No. of submissions"] - form_fields = ["id", "project__name", "project__organization__username", "title"] - if include_extra: - headers += ["Is Organization", "Organization Created By", "User last login"] - form_fields += ["project__organization__last_login"] - writer.writerow(headers) - _, last_day = calendar.monthrange(year, month) - date_obj = date(year, month, last_day) +# pylint: disable=too-many-locals +def _write_stats_to_file(month: int, year: int, include_extra: bool, filename: str): + with open(filename, "w", encoding="utf-8") as out_file: + writer = csv.writer(out_file) + headers = ["Username", "Project Name", "Form Title", "No. of submissions"] + form_fields = [ + "id", + "project__name", + "project__organization__username", + "title", + ] + if include_extra: + headers += ["Is Organization", "Organization Created By", "User last login"] + form_fields += ["project__organization__last_login"] - forms = XForm.objects.filter( - Q(deleted_at__isnull=True) | Q(deleted_at__gt=date_obj), - date_created__lte=date_obj, - ).values(*form_fields) - with use_master: - for form in forms: - instance_count = Instance.objects.filter( - Q(deleted_at__isnull=True) | Q(deleted_at__gt=date_obj), - xform_id=form.get("id"), - date_created__lte=date_obj, - ).count() - row = [ - form.get("project__organization__username"), - form.get("project__name"), - form.get("title"), - instance_count, - ] - if include_extra: - user = User.objects.get( - username=form.get("project__organization__username") - ) - is_org = is_organization(user.profile) - if is_org: - created_by = OrganizationProfile.objects.get( - user=user - ).creator.username - else: - created_by = "N/A" - row += [ - is_org, - created_by, - form.get("project__organization__last_login"), + writer.writerow(headers) + _, last_day = calendar.monthrange(year, month) + date_obj = timezone.make_aware(datetime(year, month, last_day), timezone.utc) + + forms = XForm.objects.filter( + Q(deleted_at__isnull=True) | Q(deleted_at__gt=date_obj), + date_created__lte=date_obj, + ).values(*form_fields) + with use_master: + for form in forms: + instance_count = Instance.objects.filter( + Q(deleted_at__isnull=True) | Q(deleted_at__gt=date_obj), + xform_id=form.get("id"), + date_created__lte=date_obj, + ).count() + row = [ + form.get("project__organization__username"), + form.get("project__name"), + form.get("title"), + instance_count, ] - writer.writerow(row) + if include_extra: + user = User.objects.get( + username=form.get("project__organization__username") + ) + is_org = is_organization(user.profile) + if is_org: + created_by = OrganizationProfile.objects.get( + user=user + ).creator.username + else: + created_by = "N/A" + row += [ + is_org, + created_by, + form.get("project__organization__last_login"), + ] + writer.writerow(row) class Command(BaseCommand): @@ -83,7 +94,7 @@ def add_arguments(self, parser): "-m", dest="month", help=( - "Month to calculate system statistics for." "Defaults to current month." + "Month to calculate system statistics for. Defaults to current month." ), default=None, ) @@ -91,9 +102,7 @@ def add_arguments(self, parser): "--year", "-y", dest="year", - help=( - "Year to calculate system statistics for." " Defaults to current year" - ), + help=("Year to calculate system statistics for. Defaults to current year"), default=None, ) parser.add_argument( @@ -102,11 +111,17 @@ def add_arguments(self, parser): action="store_true", dest="extra_info", default=False, - help="Include extra information; When an Organization was created and user last login", + help=( + "Include extra information; When an Organization was created and " + "user last login" + ), ) def handle(self, *args, **options): - month = int(options.get("month", datetime.now().month)) - year = int(options.get("year", datetime.now().year)) - include_extra = bool(options.get("extra_info", False)) - _write_stats_to_file(month, year) + month = int(options.get("month") or datetime.now().month) + year = int(options.get("year") or datetime.now().year) + include_extra = bool(options.get("extra_info")) + filename = f"platform_statistics_{month}_{year}.csv" + _write_stats_to_file(month, year, include_extra, filename) + if os.path.exists(filename): + self.stdout.write(f"File '{filename}' successfully created.") diff --git a/onadata/apps/logger/management/commands/import_tools.py b/onadata/apps/logger/management/commands/import_tools.py index fbb8eb7d35..49e46d37a6 100644 --- a/onadata/apps/logger/management/commands/import_tools.py +++ b/onadata/apps/logger/management/commands/import_tools.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ import_tools - import ODK formms and instances. """ @@ -26,7 +26,7 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): """Import ODK forms and instances.""" - if args.__len__() < 2: + if len(args) < 2: raise CommandError(_("path(xform instances) username")) path = args[0] username = args[1] diff --git a/onadata/apps/logger/management/commands/publish_xls.py b/onadata/apps/logger/management/commands/publish_xls.py index bd5f53851d..62da71e2d8 100644 --- a/onadata/apps/logger/management/commands/publish_xls.py +++ b/onadata/apps/logger/management/commands/publish_xls.py @@ -23,7 +23,7 @@ class Command(BaseCommand): args = "xls_file username project" help = gettext_lazy( - "Publish an XLS file with the option of replacing an" "existing one" + "Publish an XLSForm file with the option of replacing an existing one" ) def add_arguments(self, parser): diff --git a/onadata/apps/logger/management/commands/pull_from_aggregate.py b/onadata/apps/logger/management/commands/pull_from_aggregate.py index 07b7f75384..b94d9c92f9 100644 --- a/onadata/apps/logger/management/commands/pull_from_aggregate.py +++ b/onadata/apps/logger/management/commands/pull_from_aggregate.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ pull_from_aggregate command diff --git a/onadata/apps/logger/management/commands/remove_columns_from_briefcase_data.py b/onadata/apps/logger/management/commands/remove_columns_from_briefcase_data.py index a9a8fc3702..f9141f0e71 100644 --- a/onadata/apps/logger/management/commands/remove_columns_from_briefcase_data.py +++ b/onadata/apps/logger/management/commands/remove_columns_from_briefcase_data.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Delete specific columns from submission XMLs pulled by ODK Briefcase. +""" import os from typing import List @@ -8,8 +12,8 @@ def _traverse_child_nodes_and_delete_column(xml_obj, column: str) -> None: - childNodes = xml_obj.childNodes - for elem in childNodes: + child_nodes = xml_obj.childNodes + for elem in child_nodes: if elem.nodeName in column: xml_obj.removeChild(elem) if hasattr(elem, "childNodes"): @@ -17,6 +21,7 @@ def _traverse_child_nodes_and_delete_column(xml_obj, column: str) -> None: def remove_columns_from_xml(xml: str, columns: List[str]) -> str: + """Returns the ``xml`` with columns/tags removed.""" xml_obj = clean_and_parse_xml(xml).documentElement for column in columns: _traverse_child_nodes_and_delete_column(xml_obj, column) @@ -24,7 +29,11 @@ def remove_columns_from_xml(xml: str, columns: List[str]) -> str: class Command(BaseCommand): - help = _("Delete specific columns from submission " "XMLs pulled by ODK Briefcase.") + """ + Delete specific columns from submission XMLs pulled by ODK Briefcase. + """ + + help = _("Delete specific columns from submission XMLs pulled by ODK Briefcase.") def add_arguments(self, parser): parser.add_argument( @@ -76,7 +85,9 @@ def handle(self, *args, **options): ) data = None - with open(f"{in_dir}/{submission_folder}/submission.xml", "r") as in_file: + with open( + f"{in_dir}/{submission_folder}/submission.xml", "r", encoding="utf-8" + ) as in_file: data = in_file.read().replace("\n", "") data = remove_columns_from_xml(data, columns) in_file.close() @@ -87,13 +98,17 @@ def handle(self, *args, **options): os.makedirs(f"{out_dir}/{submission_folder}") with open( - f"{out_dir}/{submission_folder}/submission.xml", "w" + f"{out_dir}/{submission_folder}/submission.xml", + "w", + encoding="utf-8", ) as out_file: out_file.write(data) out_file.close() else: with open( - f"{in_dir}/{submission_folder}/submission.xml", "r+" + f"{in_dir}/{submission_folder}/submission.xml", + "r+", + encoding="utf-8", ) as out_file: out_file.truncate(0) out_file.write(data) diff --git a/onadata/apps/logger/management/commands/replace_form_id_root_node.py b/onadata/apps/logger/management/commands/replace_form_id_root_node.py index d4f5f47562..2b9498c392 100644 --- a/onadata/apps/logger/management/commands/replace_form_id_root_node.py +++ b/onadata/apps/logger/management/commands/replace_form_id_root_node.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Management command used to replace the root node of an Instance when the root node is the XForm ID @@ -15,9 +16,11 @@ from onadata.apps.logger.models.instance import InstanceHistory +# pylint: disable=invalid-name def replace_form_id_with_correct_root_node( inst_id: int, root: str = None, commit: bool = False ) -> str: + """Returns the submission XML with updated root node tag name.""" inst: Instance = Instance.objects.get(id=inst_id, deleted_at__isnull=True) initial_xml = inst.xml form_id = re.escape(inst.xform.id_string) @@ -41,11 +44,13 @@ def replace_form_id_with_correct_root_node( inst.xml = edited_xml inst.save() return f"Modified Instance ID {inst.id} - History object {history.id}" - else: - return edited_xml + + return edited_xml class Command(BaseCommand): + """Replaces form ID String with 'data' for an instances root node""" + help = _("Replaces form ID String with 'data' for an instances root node") def add_arguments(self, parser): diff --git a/onadata/apps/logger/management/commands/set_xform_surveys_with_geopoints.py b/onadata/apps/logger/management/commands/set_xform_surveys_with_geopoints.py index b40780779a..daf2a9c749 100644 --- a/onadata/apps/logger/management/commands/set_xform_surveys_with_geopoints.py +++ b/onadata/apps/logger/management/commands/set_xform_surveys_with_geopoints.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 - +""" +Import a folder of XForms for ODK. +""" from django.core.management.base import BaseCommand from django.utils.translation import gettext_lazy @@ -9,6 +11,8 @@ class Command(BaseCommand): + """Import a folder of XForms for ODK.""" + help = gettext_lazy("Import a folder of XForms for ODK.") def handle(self, *args, **kwargs): @@ -20,8 +24,9 @@ def handle(self, *args, **kwargs): try: xform.instances_with_geopoints = has_geo xform.save() + # pylint: disable=broad-except except Exception as e: self.stderr.write(e) else: count += 1 - self.stdout.write("%d of %d forms processed." % (count, total)) + self.stdout.write(f"{count} of {total} forms processed.") diff --git a/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py b/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py index df351dddae..d4236808b6 100644 --- a/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py +++ b/onadata/apps/logger/management/commands/set_xform_surveys_with_osm.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 - +""" +Set xform.instances_with_osm +""" from django.core.management.base import BaseCommand from django.utils.translation import gettext_lazy @@ -10,6 +12,8 @@ class Command(BaseCommand): + """Set xform.instances_with_osm""" + help = gettext_lazy("Set xform.instances_with_osm") def handle(self, *args, **kwargs): @@ -28,9 +32,10 @@ def handle(self, *args, **kwargs): try: xform.instances_with_osm = True xform.save() + # pylint: disable=broad-except except Exception as e: self.stderr.write(e) else: count += 1 - self.stdout.write("%d of %d forms processed." % (count, total)) + self.stdout.write(f"{count} of {total} forms processed.") diff --git a/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py b/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py index f80f9682be..5a52f4800e 100644 --- a/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py +++ b/onadata/apps/logger/management/commands/sync_deleted_instances_fix.py @@ -1,5 +1,8 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 +""" +Fixes deleted instances by syncing deleted items from mongo. +""" import json from django.conf import settings @@ -12,9 +15,11 @@ class Command(BaseCommand): - help = gettext_lazy( - "Fixes deleted instances by syncing " "deleted items from mongo." - ) + """ + Fixes deleted instances by syncing deleted items from mongo. + """ + + help = gettext_lazy("Fixes deleted instances by syncing deleted items from mongo.") def handle(self, *args, **kwargs): # Reset all sql deletes to None diff --git a/onadata/apps/logger/management/commands/transferproject.py b/onadata/apps/logger/management/commands/transferproject.py index 1f8548237a..5dda3ba081 100644 --- a/onadata/apps/logger/management/commands/transferproject.py +++ b/onadata/apps/logger/management/commands/transferproject.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """Functionality to transfer a project from one owner to another.""" from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand @@ -52,13 +53,14 @@ def add_arguments(self, parser): " transferred. If not, do not include the argument", ) - def get_user(self, username): # pylint: disable=C0111 + def get_user(self, username): + """Return user object with the given username.""" user_model = get_user_model() user = None try: user = user_model.objects.get(username=username) except user_model.DoesNotExist: - self.errors.append("User {0} does not exist".format(username)) + self.errors.append(f"User {username} does not exist") return user def update_xform_with_new_user(self, project, user): diff --git a/onadata/apps/logger/management/commands/update_moved_forms.py b/onadata/apps/logger/management/commands/update_moved_forms.py index 666c76c251..c2289a8a4b 100644 --- a/onadata/apps/logger/management/commands/update_moved_forms.py +++ b/onadata/apps/logger/management/commands/update_moved_forms.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Ensures all the forms are owned by the project owner +""" from django.core.management import BaseCommand from django.utils.translation import gettext_lazy @@ -6,7 +10,11 @@ class Command(BaseCommand): - help = gettext_lazy("Ensures all the forms are owned by the project" " owner") + """ + Ensures all the forms are owned by the project owner + """ + + help = gettext_lazy("Ensures all the forms are owned by the project owner") def handle(self, *args, **kwargs): self.stdout.write("Updating forms owner", ending="\n") @@ -16,16 +24,12 @@ def handle(self, *args, **kwargs): try: if xform.user != project.organization: self.stdout.write( - "Processing: {} - {}".format( - xform.id_string, xform.user.username - ) + f"Processing: {xform.id_string} - {xform.user.username}" ) xform.user = project.organization xform.save() + # pylint: disable=broad-except except Exception: self.stdout.write( - "Error processing: {} - {}".format( - xform.id_string, xform.user.username - ) + f"Error processing: {xform.id_string} - {xform.user.username}" ) - pass diff --git a/onadata/apps/logger/management/commands/update_xform_uuids.py b/onadata/apps/logger/management/commands/update_xform_uuids.py index ecc0ed9c8c..929f55f1c7 100644 --- a/onadata/apps/logger/management/commands/update_xform_uuids.py +++ b/onadata/apps/logger/management/commands/update_xform_uuids.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ update_xform_uuids command - Set uuid from a CSV file """ diff --git a/onadata/apps/logger/models/__init__.py b/onadata/apps/logger/models/__init__.py index 7d85c6d6be..8ddb3ffa20 100644 --- a/onadata/apps/logger/models/__init__.py +++ b/onadata/apps/logger/models/__init__.py @@ -10,9 +10,9 @@ from onadata.apps.logger.models.open_data import OpenData # noqa from onadata.apps.logger.models.osmdata import OsmData # noqa from onadata.apps.logger.models.project import Project # noqa +from onadata.apps.logger.models.submission_review import SubmissionReview # noqa from onadata.apps.logger.models.survey_type import SurveyType # noqa from onadata.apps.logger.models.widget import Widget # noqa from onadata.apps.logger.models.xform import XForm # noqa -from onadata.apps.logger.models.submission_review import SubmissionReview # noqa -from onadata.apps.logger.xform_instance_parser import InstanceParseError # noqa from onadata.apps.logger.models.xform_version import XFormVersion # noqa +from onadata.apps.logger.xform_instance_parser import InstanceParseError # noqa diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index dc29cb6566..a8ab841780 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -29,7 +29,7 @@ clean_and_parse_xml, get_uuid_from_xml, ) -from onadata.celery import app +from onadata.celeryapp import app from onadata.libs.data.query import get_numeric_fields from onadata.libs.utils.cache_tools import ( DATAVIEW_COUNT, @@ -447,7 +447,8 @@ def get_full_dict(self, load_existing=True): if review.get_note_text(): doc[REVIEW_COMMENT] = review.get_note_text() - # pylint: disable=attribute-defined-outside-init,access-member-before-definition + # pylint: disable=attribute-defined-outside-init + # pylint: disable=access-member-before-definition if not self.date_created: self.date_created = submission_time() @@ -487,7 +488,8 @@ def _set_survey_type(self): ) def _set_uuid(self): - # pylint: disable=no-member,attribute-defined-outside-init,access-member-before-definition + # pylint: disable=no-member,attribute-defined-outside-init + # pylint: disable=access-member-before-definition if self.xml and not self.uuid: # pylint: disable=no-member uuid = get_uuid_from_xml(self.xml) diff --git a/onadata/apps/logger/models/open_data.py b/onadata/apps/logger/models/open_data.py index 8c57f717c1..d24a99e9d6 100644 --- a/onadata/apps/logger/models/open_data.py +++ b/onadata/apps/logger/models/open_data.py @@ -28,7 +28,7 @@ class OpenData(models.Model): date_modified = models.DateTimeField(auto_now=True) def __str__(self): - return getattr(self, "name", "") + return str(getattr(self, "name", "")) class Meta: app_label = "logger" diff --git a/onadata/apps/logger/models/project.py b/onadata/apps/logger/models/project.py index 3dc7703c64..7ec858086b 100644 --- a/onadata/apps/logger/models/project.py +++ b/onadata/apps/logger/models/project.py @@ -2,6 +2,7 @@ """ Project model class """ +from django.apps import apps from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError @@ -28,9 +29,9 @@ class PrefetchManager(models.Manager): def get_queryset(self): """Return a queryset with the XForm, Team, tags, and other related relations prefetched.""" - # pylint: disable=import-outside-toplevel - from onadata.apps.api.models.team import Team - from onadata.apps.logger.models.xform import XForm + # pylint: disable=invalid-name + Team = apps.get_model("api", "Team") # noqa N806 + XForm = apps.get_model("logger", "XForm") # noqa N806 # pylint: disable=no-member return ( @@ -137,7 +138,9 @@ def __str__(self): return f"{self.organization}|{self.name}" def clean(self): - """Raises a validation error if a project with same name and organization exists.""" + """ + Raises a validation error if a project with same name and organization exists. + """ query_set = Project.objects.exclude(pk=self.pk).filter( name__iexact=self.name, organization=self.organization ) diff --git a/onadata/apps/logger/models/submission_review.py b/onadata/apps/logger/models/submission_review.py index 7a8205130e..2c40b1ee85 100644 --- a/onadata/apps/logger/models/submission_review.py +++ b/onadata/apps/logger/models/submission_review.py @@ -11,6 +11,7 @@ from django.utils.translation import gettext_lazy as _ +# pylint: disable=unused-argument def update_instance_json_on_save(sender, instance, **kwargs): """ Signal handler to update Instance Json with the submission review on save diff --git a/onadata/apps/logger/models/survey_type.py b/onadata/apps/logger/models/survey_type.py index 5a25fe0547..2c663e4aa4 100644 --- a/onadata/apps/logger/models/survey_type.py +++ b/onadata/apps/logger/models/survey_type.py @@ -16,4 +16,4 @@ class Meta: app_label = "logger" def __str__(self): - return "SurveyType: %s" % self.slug + return f"SurveyType: {self.slug}" diff --git a/onadata/apps/logger/models/widget.py b/onadata/apps/logger/models/widget.py index 44e878afac..eb5da5d4e1 100644 --- a/onadata/apps/logger/models/widget.py +++ b/onadata/apps/logger/models/widget.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Widget class module. +""" from builtins import str as text from django.db.models import JSONField @@ -24,6 +28,10 @@ class Widget(OrderedModel): + """ + Widget class - used for storing chart visual information. + """ + CHARTS = "charts" # Other widgets types to be added later @@ -57,10 +65,12 @@ def save(self, *args, **kwargs): if not self.key: self.key = get_uuid() - super(Widget, self).save(*args, **kwargs) + super().save(*args, **kwargs) + # pylint: disable=too-many-locals,too-many-branches @classmethod def query_data(cls, widget): + """Queries and returns chart information with the data for the chart.""" # get the columns needed column = widget.column group_by = widget.group_by if widget.group_by else None @@ -84,24 +94,24 @@ def query_data(cls, widget): field_label = get_field_label(field) columns = [ - SimpleField(field="json->>'%s'" % text(column), alias="{}".format(column)), - CountField(field="json->>'%s'" % text(column), alias="count"), + SimpleField(field=f"json->>'{text(column)}'", alias=f"{column}"), + CountField(field=f"json->>'{text(column)}'", alias="count"), ] if group_by: if field_type in NUMERIC_LIST: column_field = SimpleField( - field="json->>'%s'" % text(column), cast="float", alias=column + field=f"json->>'{text(column)}'", cast="float", alias=column ) else: column_field = SimpleField( - field="json->>'%s'" % text(column), alias=column + field=f"json->>'{text(column)}'", alias=column ) # build inner query inner_query_columns = [ column_field, - SimpleField(field="json->>'%s'" % text(group_by), alias=group_by), + SimpleField(field=f"json->>'{text(group_by)}'", alias=group_by), SimpleField(field="xform_id"), SimpleField(field="deleted_at"), ] @@ -110,14 +120,14 @@ def query_data(cls, widget): # build group-by query if field_type in NUMERIC_LIST: columns = [ - SimpleField(field=group_by, alias="%s" % group_by), + SimpleField(field=group_by, alias=f"{group_by}"), SumField(field=column, alias="sum"), AvgField(field=column, alias="mean"), ] elif field_type == SELECT_ONE: columns = [ - SimpleField(field=column, alias="%s" % column), - SimpleField(field=group_by, alias="%s" % group_by), + SimpleField(field=column, alias=f"{column}"), + SimpleField(field=group_by, alias=f"{group_by}"), CountField(field="*", alias="count"), ] @@ -138,7 +148,7 @@ def query_data(cls, widget): .from_table(Instance, columns) .where(xform_id=xform.pk, deleted_at=None) ) - query.group_by("json->>'%s'" % text(column)) + query.group_by(f"json->>'{text(column)}'") # run query records = query.select() diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index a605382ef5..3193cf9311 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -436,7 +436,6 @@ def flatten(elem, items=None): return flatten(element) - # pylint: disable=no-self-use def get_choice_label(self, field, choice_value, lang="English"): """Returns a choice's label for the given ``field`` and ``choice_value``.""" choices = [choice for choice in field.children if choice.name == choice_value] @@ -938,14 +937,14 @@ def _set_title(self): self.set_hash() if contains_xml_invalid_char(title_xml): raise XLSFormError( - _("Title shouldn't have any invalid xml " "characters ('>' '&' '<')") + _("Title shouldn't have any invalid xml characters ('>' '&' '<')") ) # Capture urls within form title if re.search( - r"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$", + r"^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~:/?#[\]@!\$&'\(\)\*\+,;=.]+$", # noqa self.title, - ): # noqa + ): raise XLSFormError(_("Invalid title value; value shouldn't match a URL")) self.title = title_xml @@ -1049,8 +1048,8 @@ def save(self, *args, **kwargs): # noqa: MC0001 ): raise XLSFormError( _( - f"The XForm id_string provided exceeds {self.MAX_ID_LENGTH} characters." - f' Please change the "id_string" or "form_id" values' + f"The XForm id_string provided exceeds {self.MAX_ID_LENGTH}" + f' characters. Please change the "id_string" or "form_id" values' f"in settings sheet or reduce the file name if you do" f" not have a settings sheets." ) @@ -1059,7 +1058,7 @@ def save(self, *args, **kwargs): # noqa: MC0001 is_version_available = self.version is not None if is_version_available and contains_xml_invalid_char(self.version): raise XLSFormError( - _("Version shouldn't have any invalid " "characters ('>' '&' '<')") + _("Version shouldn't have any invalid characters ('>' '&' '<')") ) self.description = conditional_escape(self.description) diff --git a/onadata/apps/logger/models/xform_version.py b/onadata/apps/logger/models/xform_version.py index 277a9bd0c7..d85e230354 100644 --- a/onadata/apps/logger/models/xform_version.py +++ b/onadata/apps/logger/models/xform_version.py @@ -1,7 +1,12 @@ +# -*- coding: utf-8 -*- """ Module containing the XForm Version model """ from django.db import models +from django.contrib.auth import get_user_model + + +User = get_user_model() class XFormVersion(models.Model): @@ -21,7 +26,7 @@ class XFormVersion(models.Model): version = models.CharField(max_length=100) date_created = models.DateTimeField(auto_now_add=True) date_modified = models.DateTimeField(auto_now=True) - created_by = models.ForeignKey("auth.User", on_delete=models.SET_NULL, null=True) + created_by = models.ForeignKey(User, on_delete=models.SET_NULL, null=True) xml = models.TextField() json = models.TextField() diff --git a/onadata/apps/logger/tests/test_encrypted_submissions.py b/onadata/apps/logger/tests/test_encrypted_submissions.py index 80d9f7809a..262eb7ed58 100644 --- a/onadata/apps/logger/tests/test_encrypted_submissions.py +++ b/onadata/apps/logger/tests/test_encrypted_submissions.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Test encrypted form submissions. """ diff --git a/onadata/apps/logger/views.py b/onadata/apps/logger/views.py index 64f0672cf6..5a3e6aa844 100644 --- a/onadata/apps/logger/views.py +++ b/onadata/apps/logger/views.py @@ -460,7 +460,7 @@ def download_xlsform(request, username, id_string): messages.add_message( request, messages.WARNING, - _("No XLS file for your form " "%(id)s") % {"id": id_string}, + _("No XLS file for your form %(id)s") % {"id": id_string}, ) return HttpResponseRedirect(f"/{username}") @@ -723,7 +723,7 @@ def view_download_submission(request, username): last_index = form_id.find("[") id_string = form_id[0:last_index] form_id_parts = form_id.split("/") - if form_id_parts.__len__() < 2: + if len(form_id_parts) < 2: return HttpResponseBadRequest() uuid = _extract_uuid(form_id_parts[1]) diff --git a/onadata/apps/logger/xform_instance_parser.py b/onadata/apps/logger/xform_instance_parser.py index bf2e160049..b782f4e257 100644 --- a/onadata/apps/logger/xform_instance_parser.py +++ b/onadata/apps/logger/xform_instance_parser.py @@ -392,7 +392,7 @@ def _set_attributes(self): if key in self._attributes: logger = logging.getLogger("console_logger") logger.debug( - "Skipping duplicate attribute: %s" " with value %s", key, value + "Skipping duplicate attribute: %s with value %s", key, value ) logger.debug(str(all_attributes)) else: diff --git a/onadata/apps/main/backends.py b/onadata/apps/main/backends.py index 3a11b237af..bdd607cf83 100644 --- a/onadata/apps/main/backends.py +++ b/onadata/apps/main/backends.py @@ -1,15 +1,29 @@ -from django.contrib.auth.models import User +# -*- coding: utf-8 -*- +""" +A custom ModelBackend class module. +""" +from django.contrib.auth import get_user_model from django.contrib.auth.backends import ModelBackend as DjangoModelBackend from django.db.models import Q +User = get_user_model() + class ModelBackend(DjangoModelBackend): - def authenticate(self, request=None, username=None, password=None): + """ + A custom ModelBackend class + """ + + def authenticate(self, request, username=None, password=None, **kwargs): """ Username is case insensitive. Supports using email in place of username """ + if username is None or password is None: + return None + user = User.objects.filter( - Q(username__iexact=username) | Q(email__iexact=username)).first() + Q(username__iexact=username) | Q(email__iexact=username) + ).first() if user and user.check_password(password): return user diff --git a/onadata/apps/main/context_processors.py b/onadata/apps/main/context_processors.py index 9250a629bd..96e115c8c2 100644 --- a/onadata/apps/main/context_processors.py +++ b/onadata/apps/main/context_processors.py @@ -1,24 +1,30 @@ +# -*- coding: utf-8 -*- +""" +google_analytics and site_name context processor functions. +""" from django.conf import settings from django.contrib.sites.models import Site def google_analytics(request): - ga_pid = getattr(settings, 'GOOGLE_ANALYTICS_PROPERTY_ID', False) - ga_domain = getattr(settings, 'GOOGLE_ANALYTICS_DOMAIN', False) - ga_site_verification = getattr(settings, 'GOOGLE_SITE_VERIFICATION', False) + """Returns Google Analytics property id, domain and site verification settings.""" + ga_pid = getattr(settings, "GOOGLE_ANALYTICS_PROPERTY_ID", False) + ga_domain = getattr(settings, "GOOGLE_ANALYTICS_DOMAIN", False) + ga_site_verification = getattr(settings, "GOOGLE_SITE_VERIFICATION", False) return { - 'GOOGLE_ANALYTICS_PROPERTY_ID': ga_pid, - 'GOOGLE_ANALYTICS_DOMAIN': ga_domain, - 'GOOGLE_SITE_VERIFICATION': ga_site_verification + "GOOGLE_ANALYTICS_PROPERTY_ID": ga_pid, + "GOOGLE_ANALYTICS_DOMAIN": ga_domain, + "GOOGLE_SITE_VERIFICATION": ga_site_verification, } def site_name(request): - site_id = getattr(settings, 'SITE_ID', None) + """Returns the SITE_NAME/""" + site_id = getattr(settings, "SITE_ID", None) try: site = Site.objects.get(pk=site_id) except Site.DoesNotExist: - site_name = 'example.org' + name = "example.org" else: - site_name = site.name - return {'SITE_NAME': site_name} + name = site.name + return {"SITE_NAME": name} diff --git a/onadata/apps/main/forms.py b/onadata/apps/main/forms.py index ae0a82286e..3b7b7f9b58 100644 --- a/onadata/apps/main/forms.py +++ b/onadata/apps/main/forms.py @@ -1,13 +1,11 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ forms module. """ import os import random import re -from six.moves.urllib.parse import urlparse -import requests from django import forms from django.conf import settings from django.contrib.auth import get_user_model @@ -17,7 +15,10 @@ from django.forms import ModelForm from django.utils.translation import gettext as _ from django.utils.translation import gettext_lazy + +import requests from registration.forms import RegistrationFormUniqueEmail +from six.moves.urllib.parse import urlparse # pylint: disable=ungrouped-imports from onadata.apps.logger.models import Project @@ -475,7 +476,7 @@ def clean_sms_id_string(self): if not re.match(r"^[a-z0-9\_\-]+$", sms_id_string): raise forms.ValidationError( - "id_string can only contain alphanum" " characters" + "id_string can only contain alphanum characters" ) return sms_id_string diff --git a/onadata/apps/main/management/commands/create_enketo_express_urls.py b/onadata/apps/main/management/commands/create_enketo_express_urls.py index c9658fbd0d..f0d677e4ce 100644 --- a/onadata/apps/main/management/commands/create_enketo_express_urls.py +++ b/onadata/apps/main/management/commands/create_enketo_express_urls.py @@ -1,96 +1,105 @@ +# -*- coding: utf-8 -*- +""" +Creates enketo url including preview +""" from django.core.management.base import BaseCommand, CommandError from django.http import HttpRequest from django.utils.translation import gettext_lazy from onadata.apps.logger.models import XForm from onadata.libs.utils.model_tools import queryset_iterator -from onadata.libs.utils.viewer_tools import ( - get_enketo_urls, get_form_url) +from onadata.libs.utils.viewer_tools import get_enketo_urls, get_form_url class Command(BaseCommand): + """Create enketo url including preview""" + help = gettext_lazy("Create enketo url including preview") def add_arguments(self, parser): parser.add_argument( - "-n", "--server_name", dest="server_name", default="api.ona.io") - parser.add_argument( - "-p", "--server_port", dest="server_port", default="80") - parser.add_argument( - "-r", "--protocol", dest="protocol", default="https") + "-n", "--server_name", dest="server_name", default="api.ona.io" + ) + parser.add_argument("-p", "--server_port", dest="server_port", default="80") + parser.add_argument("-r", "--protocol", dest="protocol", default="https") parser.add_argument("-u", "--username", dest="username", default=None) + parser.add_argument("-x", "--id_string", dest="id_string", default=None) parser.add_argument( - "-x", "--id_string", dest="id_string", default=None) - parser.add_argument( - "-c", "--generate_consistent_urls", - dest="generate_consistent_urls", default=True) + "-c", + "--generate_consistent_urls", + dest="generate_consistent_urls", + default=True, + ) + # pylint: disable=too-many-locals,too-many-statements def handle(self, *args, **options): request = HttpRequest() - server_name = options.get('server_name') - server_port = options.get('server_port') - protocol = options.get('protocol') - username = options.get('username') - id_string = options.get('id_string') - generate_consistent_urls = options.get('generate_consistent_urls') + server_name = options.get("server_name") + server_port = options.get("server_port") + protocol = options.get("protocol") + username = options.get("username") + id_string = options.get("id_string") + generate_consistent_urls = options.get("generate_consistent_urls") if not server_name or not server_port or not protocol: raise CommandError( - 'please provide a server_name, a server_port and a protocol') + "please provide a server_name, a server_port and a protocol" + ) - if protocol not in ['http', 'https']: - raise CommandError('protocol provided is not valid') + if protocol not in ["http", "https"]: + raise CommandError("protocol provided is not valid") # required for generation of enketo url - request.META['HTTP_HOST'] = '%s:%s' % (server_name, server_port)\ - if server_port != '80' else server_name + request.META["HTTP_HOST"] = ( + f"{server_name}:{server_port}" if server_port != "80" else server_name + ) # required for generation of enketo preview url - request.META['SERVER_NAME'] = server_name - request.META['SERVER_PORT'] = server_port + request.META["SERVER_NAME"] = server_name + request.META["SERVER_PORT"] = server_port if username and id_string: try: - xform = XForm.objects.get( - user__username=username, id_string=id_string) + xform = XForm.objects.get(user__username=username, id_string=id_string) form_url = get_form_url( - request, - username, - protocol=protocol, - xform_pk=xform.pk, - generate_consistent_urls=generate_consistent_urls) + request, + username, + protocol=protocol, + xform_pk=xform.pk, + generate_consistent_urls=generate_consistent_urls, + ) id_string = xform.id_string enketo_urls = get_enketo_urls(form_url, id_string) - _url = (enketo_urls.get('offline_url') or - enketo_urls.get('url')) - _preview_url = enketo_urls.get('preview_url') + _url = enketo_urls.get("offline_url") or enketo_urls.get("url") + _preview_url = enketo_urls.get("preview_url") - self.stdout.write('enketo url: %s | preview url: %s' % - (_url, _preview_url)) + self.stdout.write(f"enketo url: {_url} | preview url: {_preview_url}") self.stdout.write("enketo urls generation completed!!") except XForm.DoesNotExist: self.stdout.write( - "No xform matching the provided username and id_string") + "No xform matching the provided username and id_string" + ) elif username and id_string is None: xforms = XForm.objects.filter(user__username=username) num_of_xforms = xforms.count() if xforms: for xform in queryset_iterator(xforms): form_url = get_form_url( - request, - username, - protocol=protocol, - xform_pk=xform.pk, - generate_consistent_urls=generate_consistent_urls) + request, + username, + protocol=protocol, + xform_pk=xform.pk, + generate_consistent_urls=generate_consistent_urls, + ) id_string = xform.id_string enketo_urls = get_enketo_urls(form_url, id_string) - _url = (enketo_urls.get('offline_url') or - enketo_urls.get('url')) - _preview_url = enketo_urls.get('preview_url') + _url = enketo_urls.get("offline_url") or enketo_urls.get("url") + _preview_url = enketo_urls.get("preview_url") num_of_xforms -= 1 self.stdout.write( - 'enketo url: %s | preview url: %s | remaining: %s' % - (_url, _preview_url, num_of_xforms)) + f"enketo url: {_url} | preview url: {_preview_url}" + f" | remaining: {num_of_xforms}" + ) self.stdout.write("enketo urls generation completed!!") else: self.stdout.write("Username doesn't own any form") @@ -101,17 +110,18 @@ def handle(self, *args, **options): username = xform.user.username id_string = xform.id_string form_url = get_form_url( - request, - username, - protocol=protocol, - xform_pk=xform.pk, - generate_consistent_urls=generate_consistent_urls) + request, + username, + protocol=protocol, + xform_pk=xform.pk, + generate_consistent_urls=generate_consistent_urls, + ) enketo_urls = get_enketo_urls(form_url, id_string) - _url = (enketo_urls.get('offline_url') or - enketo_urls.get('url')) - _preview_url = enketo_urls.get('preview_url') + _url = enketo_urls.get("offline_url") or enketo_urls.get("url") + _preview_url = enketo_urls.get("preview_url") num_of_xforms -= 1 self.stdout.write( - 'enketo url: %s | preview url: %s | remaining: %s' % - (_url, _preview_url, num_of_xforms)) + f"enketo url: {_url} | preview url: {_preview_url}" + f" | remaining: {num_of_xforms}" + ) self.stdout.write("enketo urls generation completed!!") diff --git a/onadata/apps/main/management/commands/create_metadata_for_kpi_deployed_forms.py b/onadata/apps/main/management/commands/create_metadata_for_kpi_deployed_forms.py index c5e8105a29..fd5c60f7f4 100644 --- a/onadata/apps/main/management/commands/create_metadata_for_kpi_deployed_forms.py +++ b/onadata/apps/main/management/commands/create_metadata_for_kpi_deployed_forms.py @@ -1,5 +1,8 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 +""" +Create metadata for kpi forms that are not editable +""" from django.core.management.base import BaseCommand from django.utils.translation import gettext_lazy @@ -9,18 +12,17 @@ class Command(BaseCommand): + """Create metadata for kpi forms that are not editable""" + help = gettext_lazy("Create metadata for kpi forms that are not editable") def handle(self, *args, **kwargs): cursor = connection.cursor() - cursor.execute( - 'SELECT uid FROM kpi_asset WHERE asset_type=%s', ['survey']) - rs = cursor.cursor.fetchall() - uids = [a[0] for a in rs] + cursor.execute("SELECT uid FROM kpi_asset WHERE asset_type=%s", ["survey"]) + results = cursor.cursor.fetchall() + uids = [a[0] for a in results] xforms = XForm.objects.filter(id_string__in=uids) - for x in xforms: - MetaData.published_by_formbuilder(x, 'True') + for xform in xforms: + MetaData.published_by_formbuilder(xform, "True") - self.stdout.write( - "Done creating published_by_formbuilder metadata!!!" - ) + self.stdout.write("Done creating published_by_formbuilder metadata!!!") diff --git a/onadata/apps/main/management/commands/export_user_emails.py b/onadata/apps/main/management/commands/export_user_emails.py index 42289e9067..546357b2fa 100644 --- a/onadata/apps/main/management/commands/export_user_emails.py +++ b/onadata/apps/main/management/commands/export_user_emails.py @@ -1,5 +1,8 @@ #!/usr/bin/env python # vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 +""" +export_user_emails command - prints a CSV of usernames and emails. +""" from django.core.management.base import BaseCommand from django.utils.translation import gettext_lazy @@ -8,16 +11,17 @@ class Command(BaseCommand): + """Export users and emails""" + help = gettext_lazy("Export users and emails") def handle(self, *args, **kwargs): self.stdout.write( '"username","email","first_name","last_name","name","organization"' ) - for p in queryset_iterator(UserProfile.objects.all()): + for profile in queryset_iterator(UserProfile.objects.all()): self.stdout.write( - u'"{}","{}","{}","{}","{}","{}"'.format( - p.user.username, p.user.email, p.user.first_name, - p.user.last_name, p.name, p.organization - ) + f'"{profile.user.username}","{profile.user.email}",' + f'"{profile.user.first_name}","{profile.user.last_name}",' + f'"{profile.name}","{profile.organization}"' ) diff --git a/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py b/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py index b13b8cdd2e..7a932f1e63 100644 --- a/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py +++ b/onadata/apps/main/management/commands/get_accounts_with_duplicate_id_strings.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ get_accounts_with_duplicate_id_strings - Retrieves accounts with duplicate id_strings """ diff --git a/onadata/apps/main/management/commands/mailer.py b/onadata/apps/main/management/commands/mailer.py index 014c07d1e3..af3237abd8 100644 --- a/onadata/apps/main/management/commands/mailer.py +++ b/onadata/apps/main/management/commands/mailer.py @@ -1,23 +1,32 @@ +# -*- coding: utf-8 -*- +""" +mailer command - sends emails to all users. +""" +from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError -from django.contrib.auth.models import User from django.template.loader import get_template -from django.utils.translation import gettext as _, gettext_lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy from templated_email import send_templated_mail +User = get_user_model() + class Command(BaseCommand): + """Send an email to all onadata users""" + help = gettext_lazy("Send an email to all onadata users") def add_arguments(self, parser): parser.add_argument("-m", "--message", dest="message", default=False) def handle(self, *args, **options): - message = options.get('message') - verbosity = options.get('verbosity') - get_template('templated_email/notice.email') + message = options.get("message") + verbosity = options.get("verbosity") + get_template("templated_email/notice.email") if not message: - raise CommandError(_('message must be included in options')) + raise CommandError(_("message must be included in options")) # get all users users = User.objects.all() for user in users: @@ -25,16 +34,18 @@ def handle(self, *args, **options): if not name or len(name) == 0: name = user.email if verbosity: - self.stdout.write(_( - 'Emailing name: %(name)s, email: %(email)s') - % {'name': name, 'email': user.email}) + self.stdout.write( + _("Emailing name: %(name)s, email: %(email)s") + % {"name": name, "email": user.email} + ) # send each email separately so users cannot see eachother send_templated_mail( - template_name='notice', - from_email='noreply@ona.io', + template_name="notice", + from_email="noreply@ona.io", recipient_list=[user.email], context={ - 'username': user.username, - 'full_name': name, - 'message': message - }, ) + "username": user.username, + "full_name": name, + "message": message, + }, + ) diff --git a/onadata/apps/main/registration_views.py b/onadata/apps/main/registration_views.py index a18f3987b8..df81ffd30b 100644 --- a/onadata/apps/main/registration_views.py +++ b/onadata/apps/main/registration_views.py @@ -1,9 +1,15 @@ +# -*- coding: utf-8 -*- +""" +FHRegistrationView class module. +""" from registration.backends.default.views import RegistrationView class FHRegistrationView(RegistrationView): + """A custom RegistrationView.""" + def register(self, form): - new_user = super(FHRegistrationView, self).register(form) + new_user = super().register(form) form.save_user_profile(new_user) return new_user diff --git a/onadata/apps/main/signals.py b/onadata/apps/main/signals.py index 9e9b88883b..824d03b7d6 100644 --- a/onadata/apps/main/signals.py +++ b/onadata/apps/main/signals.py @@ -1,60 +1,64 @@ +# -*- coding: utf-8 -*- +""" +signal module. +""" from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.template.loader import render_to_string from onadata.libs.utils.email import send_generic_email +User = get_user_model() + + +# pylint: disable=unused-argument def set_api_permissions(sender, instance=None, created=False, **kwargs): + """Sets API permissions for a user.""" + # pylint: disable=import-outside-toplevel from onadata.libs.utils.user_auth import set_api_permissions_for_user + if created: set_api_permissions_for_user(instance) -def send_inactive_user_email( - sender, instance=None, created=False, **kwargs): +def send_inactive_user_email(sender, instance=None, created=False, **kwargs): + """Sends email to inactive user upon account creation.""" if (created and not instance.is_active) and getattr( - settings, "ENABLE_ACCOUNT_ACTIVATION_EMAILS", False): - deployment_name = getattr(settings, 'DEPLOYMENT_NAME', 'Ona') - context = { - 'username': instance.username, - 'deployment_name': deployment_name - } - email = render_to_string( - 'registration/inactive_account_email.txt', context) + settings, "ENABLE_ACCOUNT_ACTIVATION_EMAILS", False + ): + deployment_name = getattr(settings, "DEPLOYMENT_NAME", "Ona") + context = {"username": instance.username, "deployment_name": deployment_name} + email = render_to_string("registration/inactive_account_email.txt", context) if instance.email: send_generic_email( instance.email, email, - f'{deployment_name} account created - Pending activation') + f"{deployment_name} account created - Pending activation", + ) -def send_activation_email( - sender, instance=None, **kwargs -): +def send_activation_email(sender, instance=None, **kwargs): + """Sends activation email to user.""" instance_id = instance.id - if instance_id and getattr( - settings, "ENABLE_ACCOUNT_ACTIVATION_EMAILS", False): + if instance_id and getattr(settings, "ENABLE_ACCOUNT_ACTIVATION_EMAILS", False): try: - user = User.objects.using('default').get( - id=instance_id) + user = User.objects.using("default").get(id=instance_id) except User.DoesNotExist: pass else: if not user.is_active and instance.is_active: - deployment_name = getattr(settings, 'DEPLOYMENT_NAME', 'Ona') + deployment_name = getattr(settings, "DEPLOYMENT_NAME", "Ona") context = { - 'username': instance.username, - 'deployment_name': deployment_name + "username": instance.username, + "deployment_name": deployment_name, } email = render_to_string( - 'registration/activated_account_email.txt', context + "registration/activated_account_email.txt", context ) if instance.email: send_generic_email( - instance.email, - email, - f'{deployment_name} account activated' + instance.email, email, f"{deployment_name} account activated" ) diff --git a/onadata/apps/main/tests/test_base.py b/onadata/apps/main/tests/test_base.py index deee090673..34f6015b87 100644 --- a/onadata/apps/main/tests/test_base.py +++ b/onadata/apps/main/tests/test_base.py @@ -9,6 +9,7 @@ import os import re import socket +import warnings from io import StringIO from tempfile import NamedTemporaryFile @@ -19,12 +20,11 @@ from django.test.client import Client from django.utils import timezone -from six.moves.urllib.error import URLError -from six.moves.urllib.request import urlopen - from django_digest.test import Client as DigestClient from django_digest.test import DigestAuth from rest_framework.test import APIRequestFactory +from six.moves.urllib.error import URLError +from six.moves.urllib.request import urlopen from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.apps.logger.models import Attachment, Instance, XForm @@ -41,6 +41,8 @@ # pylint: disable=invalid-name User = get_user_model() +warnings.simplefilter("ignore") + # pylint: disable=too-many-instance-attributes class TestBase(PyxformMarkdown, TransactionTestCase): diff --git a/onadata/apps/main/tests/test_style.py b/onadata/apps/main/tests/test_style.py deleted file mode 100644 index 3d23a74ceb..0000000000 --- a/onadata/apps/main/tests/test_style.py +++ /dev/null @@ -1,12 +0,0 @@ -from subprocess import call - -from django.test import TestCase - - -class TestStyle(TestCase): - - def test_flake8(self): - result = call( - ['flake8', '--exclude=migrations,src,settings', 'onadata'] - ) - self.assertEqual(result, 0, "Code is not flake8.") diff --git a/onadata/apps/main/urls.py b/onadata/apps/main/urls.py index d26fa5c6c8..79c351c75e 100644 --- a/onadata/apps/main/urls.py +++ b/onadata/apps/main/urls.py @@ -171,7 +171,7 @@ ), re_path( # pylint: disable=line-too-long - r"^(?P[^/]+)/forms/(?P[^/]+)/formid-media/(?P\d+)", + r"^(?P[^/]+)/forms/(?P[^/]+)/formid-media/(?P\d+)", # noqa main_views.download_media_data, # noqa name="download-media-data", ), diff --git a/onadata/apps/main/views.py b/onadata/apps/main/views.py index 86735bf757..3c6812177b 100644 --- a/onadata/apps/main/views.py +++ b/onadata/apps/main/views.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- # pylint: disable=too-many-lines """ Main views. @@ -326,7 +326,7 @@ def set_form(): try: resp = render(request, "profile.html", data) except XLSFormError as e: - resp = HttpResponseBadRequest(e.__str__()) + resp = HttpResponseBadRequest(str(e)) return resp @@ -514,7 +514,7 @@ def show(request, username=None, id_string=None, uuid=None): data["mapbox_layer"] = MetaData.mapbox_layer_upload(xform) data["external_export"] = MetaData.external_export(xform) except XLSFormError as e: - return HttpResponseBadRequest(e.__str__()) + return HttpResponseBadRequest(str(e)) if is_xform_owner: set_xform_owner_data(data, xform, request, username, id_string) @@ -611,7 +611,7 @@ def api(request, username=None, id_string=None): # noqa C901 cursor = query_data(**args) except (ValueError, TypeError) as e: - return HttpResponseBadRequest(conditional_escape(e.__str__())) + return HttpResponseBadRequest(conditional_escape(str(e))) if "callback" in request.GET and request.GET.get("callback") != "": callback = request.GET.get("callback") @@ -1456,7 +1456,7 @@ def stringify_unknowns(obj): query_args["count"] = int(request.GET.get("count")) > 0 cursor = AuditLog.query_data(**query_args) except ValueError as e: - return HttpResponseBadRequest(e.__str__()) + return HttpResponseBadRequest(str(e)) records = list(record for record in cursor) if "callback" in request.GET and request.GET.get("callback") != "": diff --git a/onadata/apps/messaging/apps.py b/onadata/apps/messaging/apps.py index 1ee410eb62..3016851258 100644 --- a/onadata/apps/messaging/apps.py +++ b/onadata/apps/messaging/apps.py @@ -11,17 +11,18 @@ class MessagingConfig(AppConfig): """ Messaging AppsConfig class. """ - name = 'onadata.apps.messaging' - verbose_name = 'Messaging' + + name = "onadata.apps.messaging" + verbose_name = "Messaging" def ready(self): - from onadata.apps.messaging import signals # noqa pylint: disable=W0612 + # pylint: disable=import-outside-toplevel,unused-import + from onadata.apps.messaging import signals # noqa # this needs to be imported inline because otherwise we get # django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet. from actstream import registry - registry.register(apps.get_model(model_name='User', app_label='auth')) - registry.register( - apps.get_model(model_name='XForm', app_label='logger')) - registry.register( - apps.get_model(model_name='Project', app_label='logger')) + + registry.register(apps.get_model(model_name="User", app_label="auth")) + registry.register(apps.get_model(model_name="XForm", app_label="logger")) + registry.register(apps.get_model(model_name="Project", app_label="logger")) diff --git a/onadata/apps/messaging/backends/base.py b/onadata/apps/messaging/backends/base.py index 932b5ddab7..782c62f8fd 100644 --- a/onadata/apps/messaging/backends/base.py +++ b/onadata/apps/messaging/backends/base.py @@ -23,7 +23,7 @@ def call_backend(backend, instance_id, backend_options=None): backend_class(options=backend_options).send(instance) -class BaseBackend(object): # pylint: disable=too-few-public-methods +class BaseBackend: # pylint: disable=too-few-public-methods """ Base class for notification backends """ diff --git a/onadata/apps/messaging/backends/mqtt.py b/onadata/apps/messaging/backends/mqtt.py index e8711052bc..35c0ab12ce 100644 --- a/onadata/apps/messaging/backends/mqtt.py +++ b/onadata/apps/messaging/backends/mqtt.py @@ -7,14 +7,19 @@ import json import ssl -import paho.mqtt.publish as publish from django.conf import settings +from paho.mqtt import publish + from onadata.apps.logger.models import XForm from onadata.apps.messaging.backends.base import BaseBackend -from onadata.apps.messaging.constants import MESSAGE -from onadata.apps.messaging.constants import PROJECT, USER, XFORM, \ - VERB_TOPIC_DICT +from onadata.apps.messaging.constants import ( + MESSAGE, + PROJECT, + USER, + VERB_TOPIC_DICT, + XFORM, +) def get_target_metadata(target_obj): @@ -24,12 +29,12 @@ def get_target_metadata(target_obj): target_obj_type = target_obj._meta.model_name metadata = dict(id=target_obj.pk) if target_obj_type == PROJECT: - metadata['name'] = target_obj.name + metadata["name"] = target_obj.name elif target_obj_type == XFORM: - metadata['name'] = target_obj.title - metadata['form_id'] = target_obj.id_string + metadata["name"] = target_obj.title + metadata["form_id"] = target_obj.id_string elif target_obj_type == USER: - metadata['name'] = target_obj.get_full_name() + metadata["name"] = target_obj.get_full_name() return metadata @@ -37,8 +42,9 @@ def get_payload(instance, verbose_payload: bool = False): """ Constructs the message payload """ - full_message_payload = getattr( - settings, 'FULL_MESSAGE_PAYLOAD', False) or verbose_payload + full_message_payload = ( + getattr(settings, "FULL_MESSAGE_PAYLOAD", False) or verbose_payload + ) try: description = json.loads(instance.description) except json.JSONDecodeError: @@ -50,24 +56,24 @@ def get_payload(instance, verbose_payload: bool = False): "verb": instance.verb, "message": description, "user": instance.actor.username, - "timestamp": instance.timestamp.isoformat() + "timestamp": instance.timestamp.isoformat(), } else: payload = { - 'id': instance.id, - 'time': instance.timestamp.isoformat(), - 'payload': { - 'author': { - 'username': instance.actor.username, - 'real_name': instance.actor.get_full_name() + "id": instance.id, + "time": instance.timestamp.isoformat(), + "payload": { + "author": { + "username": instance.actor.username, + "real_name": instance.actor.get_full_name(), }, - 'context': { - 'type': instance.target._meta.model_name, - 'metadata': get_target_metadata(instance.target), - 'verb': instance.verb + "context": { + "type": instance.target._meta.model_name, + "metadata": get_target_metadata(instance.target), + "verb": instance.verb, }, - 'message': description - } + "message": description, + }, } return json.dumps(payload) @@ -79,30 +85,32 @@ class MQTTBackend(BaseBackend): """ def __init__(self, options=None): - super(MQTTBackend, self).__init__() + super().__init__() if not options: raise Exception("MQTT Backend expects configuration options.") - self.host = options.get('HOST') + self.host = options.get("HOST") if not self.host: raise Exception("An MQTT host is required.") - self.port = options.get('PORT') + self.port = options.get("PORT") self.cert_info = None - secure = options.get('SECURE', False) + secure = options.get("SECURE", False) if secure: - if options.get('CA_CERT_FILE') is None: - raise Exception("The Certificate Authority certificate file " - "is required.") + if options.get("CA_CERT_FILE") is None: + raise Exception( + "The Certificate Authority certificate file is required." + ) self.cert_info = dict( - ca_certs=options.get('CA_CERT_FILE'), - certfile=options.get('CERT_FILE'), - keyfile=options.get('KEY_FILE'), + ca_certs=options.get("CA_CERT_FILE"), + certfile=options.get("CERT_FILE"), + keyfile=options.get("KEY_FILE"), tls_version=ssl.PROTOCOL_TLSv1_2, - cert_reqs=ssl.CERT_NONE) + cert_reqs=ssl.CERT_NONE, + ) - self.qos = options.get('QOS', 0) - self.retain = options.get('RETAIN', False) - self.topic_base = options.get('TOPIC_BASE', 'onadata') + self.qos = options.get("QOS", 0) + self.retain = options.get("RETAIN", False) + self.topic_base = options.get("TOPIC_BASE", "onadata") def get_topic(self, instance): """ @@ -114,26 +122,28 @@ def get_topic(self, instance): /onadata/users/[pk or uuid]/[verb]/messages/publish """ kwargs = { - 'target_id': instance.target_object_id, - 'target_name': instance.target._meta.model_name, - 'topic_base': self.topic_base, - 'verb': instance.verb + "target_id": instance.target_object_id, + "target_name": instance.target._meta.model_name, + "topic_base": self.topic_base, + "verb": instance.verb, } - if kwargs.get('target_name') == XFORM: + if kwargs.get("target_name") == XFORM: xform = XForm.objects.get(id=instance.target_object_id) - kwargs[ - 'organization_username'] = xform.project.organization.username - kwargs['verb'] = VERB_TOPIC_DICT[instance.verb] - kwargs['project_id'] = xform.project.id - return ('/{topic_base}/organization/{organization_username}/' - 'project/{project_id}/{target_name}/{target_id}/{verb}/' - 'messages/publish').format(**kwargs) - - elif kwargs.get('verb') == MESSAGE: + kwargs["organization_username"] = xform.project.organization.username + kwargs["verb"] = VERB_TOPIC_DICT[instance.verb] + kwargs["project_id"] = xform.project.id return ( - '/{topic_base}/{target_name}/{target_id}/' - 'messages/publish'.format( - **kwargs)) + "/{topic_base}/organization/{organization_username}/" + "project/{project_id}/{target_name}/{target_id}/{verb}/" + "messages/publish" + ).format(**kwargs) + + if kwargs.get("verb") == MESSAGE: + return "/{topic_base}/{target_name}/{target_id}/" "messages/publish".format( + **kwargs + ) + + return "" def send(self, instance): """ @@ -143,6 +153,12 @@ def send(self, instance): payload = get_payload(instance) # send it - return publish.single(topic, payload=payload, hostname=self.host, - port=self.port, tls=self.cert_info, qos=self.qos, - retain=self.retain) + return publish.single( + topic, + payload=payload, + hostname=self.host, + port=self.port, + tls=self.cert_info, + qos=self.qos, + retain=self.retain, + ) diff --git a/onadata/apps/messaging/filters.py b/onadata/apps/messaging/filters.py index 8250e2d703..9a2e7fffc5 100644 --- a/onadata/apps/messaging/filters.py +++ b/onadata/apps/messaging/filters.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals from actstream.models import Action -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.utils.translation import gettext as _ from rest_framework import exceptions, filters from django_filters import rest_framework as rest_filters @@ -13,6 +13,9 @@ from onadata.apps.messaging.utils import TargetDoesNotExist, get_target +User = get_user_model() + + DATETIME_LOOKUPS = [ "exact", "gt", @@ -50,10 +53,7 @@ class ActionFilterSet(rest_filters.FilterSet): class Meta: model = Action - fields = { - "verb": ["exact"], - "timestamp": DATETIME_LOOKUPS - } + fields = {"verb": ["exact"], "timestamp": DATETIME_LOOKUPS} class TargetTypeFilterBackend(filters.BaseFilterBackend): @@ -61,26 +61,25 @@ class TargetTypeFilterBackend(filters.BaseFilterBackend): A filter backend that filters by target type. """ - # pylint: disable=no-self-use def filter_queryset(self, request, queryset, view): """ Return a filtered queryset. """ - if view.action == 'list': - target_type = request.query_params.get('target_type') + if view.action == "list": + target_type = request.query_params.get("target_type") if target_type: try: target = get_target(target_type) - except TargetDoesNotExist: + except TargetDoesNotExist as exc: raise exceptions.ParseError( - "Unknown target_type {}".format(target_type)) + f"Unknown target_type {target_type}" + ) from exc return queryset.filter(target_content_type=target) - raise exceptions.ParseError( - _("Parameter 'target_type' is missing.")) + raise exceptions.ParseError(_("Parameter 'target_type' is missing.")) return queryset @@ -90,14 +89,13 @@ class TargetIDFilterBackend(filters.BaseFilterBackend): A filter backend that filters by target id. """ - # pylint: disable=no-self-use def filter_queryset(self, request, queryset, view): """ Return a filtered queryset. """ - if view.action == 'list': - target_id = request.query_params.get('target_id') + if view.action == "list": + target_id = request.query_params.get("target_id") if target_id: return queryset.filter(target_object_id=target_id) @@ -113,14 +111,13 @@ class UserFilterBackend(filters.BaseFilterBackend): A filter backend that filters by username. """ - # pylint: disable=no-self-use def filter_queryset(self, request, queryset, view): """ Return a filtered queryset. """ - if view.action == 'list': - username = request.query_params.get('user') + if view.action == "list": + username = request.query_params.get("user") try: user = User.objects.get(username=username) return queryset.filter(actor_object_id=user.id) diff --git a/onadata/apps/messaging/permissions.py b/onadata/apps/messaging/permissions.py index 7e1f776039..42c1c6727e 100644 --- a/onadata/apps/messaging/permissions.py +++ b/onadata/apps/messaging/permissions.py @@ -4,22 +4,26 @@ """ from __future__ import unicode_literals -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from rest_framework import exceptions, permissions +User = get_user_model() + + class TargetObjectPermissions(permissions.BasePermission): """ Check target object permissions """ + perms_map = { - 'GET': ['%(app_label)s.view_%(model_name)s'], - 'OPTIONS': ['%(app_label)s.view_%(model_name)s'], - 'HEAD': ['%(app_label)s.view_%(model_name)s'], - 'POST': ['%(app_label)s.change_%(model_name)s'], - 'PUT': ['%(app_label)s.change_%(model_name)s'], - 'PATCH': ['%(app_label)s.change_%(model_name)s'], - 'DELETE': ['%(app_label)s.delete_%(model_name)s'], + "GET": ["%(app_label)s.view_%(model_name)s"], + "OPTIONS": ["%(app_label)s.view_%(model_name)s"], + "HEAD": ["%(app_label)s.view_%(model_name)s"], + "POST": ["%(app_label)s.change_%(model_name)s"], + "PUT": ["%(app_label)s.change_%(model_name)s"], + "PATCH": ["%(app_label)s.change_%(model_name)s"], + "DELETE": ["%(app_label)s.delete_%(model_name)s"], } def get_required_object_permissions(self, method, model_cls): @@ -28,8 +32,8 @@ def get_required_object_permissions(self, method, model_cls): """ kwargs = { - 'app_label': model_cls._meta.app_label, - 'model_name': model_cls._meta.model_name + "app_label": model_cls._meta.app_label, + "model_name": model_cls._meta.model_name, } if method not in self.perms_map: diff --git a/onadata/apps/messaging/serializers.py b/onadata/apps/messaging/serializers.py index ca71d4942e..4f24232b80 100644 --- a/onadata/apps/messaging/serializers.py +++ b/onadata/apps/messaging/serializers.py @@ -12,7 +12,7 @@ from actstream.models import Action from actstream.signals import action from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.http import HttpRequest from django.utils.translation import gettext as _ from rest_framework import exceptions, serializers @@ -21,12 +21,14 @@ from onadata.apps.messaging.utils import TargetDoesNotExist, get_target +User = get_user_model() + + class ContentTypeChoiceField(serializers.ChoiceField): """ Custom ChoiceField that gets the model name from a ContentType object """ - # pylint: disable=no-self-use def to_representation(self, value): """ Get the model from ContentType object @@ -38,30 +40,41 @@ class MessageSerializer(serializers.ModelSerializer): """ Serializer class for Message objects """ - TARGET_CHOICES = (('xform', 'XForm'), ('project', 'Project'), - ('user', 'User')) # yapf: disable - - message = serializers.CharField(source='description', allow_blank=False) - target_id = serializers.IntegerField(source='target_object_id') - target_type = ContentTypeChoiceField( - TARGET_CHOICES, source='target_content_type') - user = serializers.CharField(source='actor', required=False) + + TARGET_CHOICES = ( + ("xform", "XForm"), + ("project", "Project"), + ("user", "User"), + ) # yapf: disable + + message = serializers.CharField(source="description", allow_blank=False) + target_id = serializers.IntegerField(source="target_object_id") + target_type = ContentTypeChoiceField(TARGET_CHOICES, source="target_content_type") + user = serializers.CharField(source="actor", required=False) verb = serializers.ChoiceField(MESSAGE_VERBS, default=MESSAGE) class Meta: """ MessageSerializer metadata """ + model = Action - fields = ['id', 'verb', 'message', 'user', 'target_id', 'target_type', - 'timestamp'] + fields = [ + "id", + "verb", + "message", + "user", + "target_id", + "target_type", + "timestamp", + ] def __init__(self, *args, **kwargs): - super(MessageSerializer, self).__init__(*args, **kwargs) - request = self.context.get('request') - full_message_payload = getattr(settings, 'FULL_MESSAGE_PAYLOAD', False) - if request and request.method == 'GET' and not full_message_payload: - extra_fields = ['target_type', 'target_id'] + super().__init__(*args, **kwargs) + request = self.context.get("request") + full_message_payload = getattr(settings, "FULL_MESSAGE_PAYLOAD", False) + if request and request.method == "GET" and not full_message_payload: + extra_fields = ["target_type", "target_id"] for field in extra_fields: self.fields.pop(field) @@ -69,39 +82,47 @@ def create(self, validated_data): """ Creates the Message in the Action model """ - request = self.context['request'] + request = self.context["request"] target_type = validated_data.get("target_content_type") target_id = validated_data.get("target_object_id") verb = validated_data.get("verb", MESSAGE) try: content_type = get_target(target_type) - except TargetDoesNotExist: - raise serializers.ValidationError({ - 'target_type': _('Unknown target type') - }) # yapf: disable + except TargetDoesNotExist as exc: + raise serializers.ValidationError( + {"target_type": _("Unknown target type")} + ) from exc # yapf: disable else: try: - target_object = \ - content_type.get_object_for_this_type(pk=target_id) - except content_type.model_class().DoesNotExist: - raise serializers.ValidationError({ - 'target_id': _('target_id not found') - }) # yapf: disable + target_object = content_type.get_object_for_this_type(pk=target_id) + except content_type.model_class().DoesNotExist as exc: + raise serializers.ValidationError( + {"target_id": _("target_id not found")} + ) from exc # yapf: disable else: # check if request.user has permission to the target_object - permission = '{}.change_{}'.format( - target_object._meta.app_label, - target_object._meta.model_name) - if not request.user.has_perm(permission, target_object) \ - and verb == MESSAGE: - message = (_("You do not have permission to add messages " - "to target_id %s.") % target_object) + permission = ( + f"{target_object._meta.app_label}." + f"change_{target_object._meta.model_name}" + ) + if ( + not request.user.has_perm(permission, target_object) + and verb == MESSAGE + ): + message = ( + _( + "You do not have permission to add messages " + "to target_id %s." + ) + % target_object + ) raise exceptions.PermissionDenied(detail=message) results = action.send( request.user, verb=verb, target=target_object, - description=validated_data.get("description")) + description=validated_data.get("description"), + ) # results will be a list of tuples with the first item in the # tuple being the signal handler function and the second @@ -109,22 +130,28 @@ def create(self, validated_data): # element in the list whose function is `action_handler` try: + # pylint: disable=comparison-with-callable instance = [ - instance for (receiver, instance) in results + instance + for (receiver, instance) in results if receiver == action_handler ].pop() - except IndexError: + except IndexError as exc: # if you get here it means we have no instances raise serializers.ValidationError( - "Message not created. Please retry.") + "Message not created. Please retry." + ) from exc else: return instance def send_message( - instance_id: Union[list, int], - target_id: int, - target_type: str, user: User, message_verb: str): + instance_id: Union[list, int], + target_id: int, + target_type: str, + user: User, + message_verb: str, +): """ Send a message. :param id: A single ID or list of IDs that have been affected by an action @@ -133,7 +160,7 @@ def send_message( :param request: http request object :return: """ - message_id_limit = getattr(settings, 'NOTIFICATION_ID_LIMIT', 100) + message_id_limit = getattr(settings, "NOTIFICATION_ID_LIMIT", 100) if user: if isinstance(instance_id, int): instance_id = [instance_id] @@ -143,25 +170,22 @@ def send_message( data = { "target_id": target_id, "target_type": target_type, - "verb": message_verb + "verb": message_verb, } # If ID is a list and the message limit on the amount of IDs # in one message is passed. Split the ids into # chunks - if isinstance(instance_id, list) and\ - len(instance_id) > message_id_limit: + if isinstance(instance_id, list) and len(instance_id) > message_id_limit: ids = instance_id while len(ids) > 0: - data["message"] = json.dumps({'id': ids[:message_id_limit]}) - message = MessageSerializer( - data=data, context={"request": request}) + data["message"] = json.dumps({"id": ids[:message_id_limit]}) + message = MessageSerializer(data=data, context={"request": request}) del ids[:message_id_limit] if message.is_valid(): message.save() else: - data["message"] = json.dumps({'id': instance_id}) - message = MessageSerializer( - data=data, context={"request": request}) + data["message"] = json.dumps({"id": instance_id}) + message = MessageSerializer(data=data, context={"request": request}) if message.is_valid(): message.save() diff --git a/onadata/apps/messaging/signals.py b/onadata/apps/messaging/signals.py index abd1365cb9..d0b75c1fd3 100644 --- a/onadata/apps/messaging/signals.py +++ b/onadata/apps/messaging/signals.py @@ -13,23 +13,24 @@ from onadata.apps.messaging.tasks import call_backend_async -@receiver(post_save, sender=Action, dispatch_uid='messaging_backends_handler') -def messaging_backends_handler(sender, **kwargs): # pylint: disable=W0613 +@receiver(post_save, sender=Action, dispatch_uid="messaging_backends_handler") +def messaging_backends_handler(sender, **kwargs): # pylint: disable=unused-argument """ Handler to send messages to notification backends e.g MQTT. """ - backends = getattr(settings, 'NOTIFICATION_BACKENDS', {}) - as_task = getattr(settings, 'MESSAGING_ASYNC_NOTIFICATION', False) - created = kwargs.get('created') - instance = kwargs.get('instance') + backends = getattr(settings, "NOTIFICATION_BACKENDS", {}) + as_task = getattr(settings, "MESSAGING_ASYNC_NOTIFICATION", False) + created = kwargs.get("created") + instance = kwargs.get("instance") if instance and created: for name in backends: - backend = backends[name]['BACKEND'] - backend_options = backends[name].get('OPTIONS') + backend = backends[name]["BACKEND"] + backend_options = backends[name].get("OPTIONS") if as_task: # Sometimes the Action isn't created yet, hence # the need to delay 2 seconds call_backend_async.apply_async( - (backend, instance.id, backend_options), countdown=2) + (backend, instance.id, backend_options), countdown=2 + ) else: call_backend(backend, instance.id, backend_options) diff --git a/onadata/apps/messaging/tasks.py b/onadata/apps/messaging/tasks.py index 98a3b9c3e7..3567570cbe 100644 --- a/onadata/apps/messaging/tasks.py +++ b/onadata/apps/messaging/tasks.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals from onadata.apps.messaging.backends.base import call_backend -from onadata.celery import app +from onadata.celeryapp import app @app.task(ignore_result=True) diff --git a/onadata/apps/messaging/urls.py b/onadata/apps/messaging/urls.py index 2fe9163fd8..f3612df94f 100644 --- a/onadata/apps/messaging/urls.py +++ b/onadata/apps/messaging/urls.py @@ -7,9 +7,9 @@ from onadata.apps.messaging.viewsets import MessagingViewSet -router = routers.DefaultRouter(trailing_slash=False) # pylint: disable=C0103 -router.register(r'messaging', MessagingViewSet) +router = routers.DefaultRouter(trailing_slash=False) # pylint: disable=invalid-name +router.register(r"messaging", MessagingViewSet) -urlpatterns = [ # pylint: disable=C0103 - re_path(r'^api/v1/', include(router.urls)), +urlpatterns = [ # pylint: disable=invalid-name + re_path(r"^api/v1/", include(router.urls)), ] diff --git a/onadata/apps/messaging/utils.py b/onadata/apps/messaging/utils.py index 573c4d108a..763626dbff 100644 --- a/onadata/apps/messaging/utils.py +++ b/onadata/apps/messaging/utils.py @@ -14,6 +14,7 @@ class TargetDoesNotExist(Exception): """ Target does not Exist exception class. """ + message = UNKNOWN_TARGET @@ -26,5 +27,5 @@ def get_target(target_type): app_label = APP_LABEL_MAPPING[target_type] return ContentType.objects.get(app_label=app_label, model=target_type) - except (KeyError, ContentType.DoesNotExist): - raise TargetDoesNotExist() + except (KeyError, ContentType.DoesNotExist) as exc: + raise TargetDoesNotExist() from exc diff --git a/onadata/apps/messaging/viewsets.py b/onadata/apps/messaging/viewsets.py index 470a3c93f0..fe2482e9ca 100644 --- a/onadata/apps/messaging/viewsets.py +++ b/onadata/apps/messaging/viewsets.py @@ -13,17 +13,24 @@ from onadata.apps.messaging.constants import MESSAGE_VERBS from onadata.apps.messaging.filters import ( - ActionFilterSet, TargetIDFilterBackend, TargetTypeFilterBackend, - UserFilterBackend) + ActionFilterSet, + TargetIDFilterBackend, + TargetTypeFilterBackend, + UserFilterBackend, +) from onadata.apps.messaging.permissions import TargetObjectPermissions from onadata.apps.messaging.serializers import MessageSerializer from onadata.libs.pagination import StandardPageNumberPagination # pylint: disable=too-many-ancestors -class MessagingViewSet(mixins.CreateModelMixin, mixins.ListModelMixin, - mixins.RetrieveModelMixin, mixins.DestroyModelMixin, - viewsets.GenericViewSet): +class MessagingViewSet( + mixins.CreateModelMixin, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + mixins.DestroyModelMixin, + viewsets.GenericViewSet, +): """ ViewSet for the Messaging app - implements /messaging API endpoint """ @@ -31,8 +38,12 @@ class MessagingViewSet(mixins.CreateModelMixin, mixins.ListModelMixin, serializer_class = MessageSerializer queryset = Action.objects.filter(verb__in=MESSAGE_VERBS) permission_classes = [IsAuthenticated, TargetObjectPermissions] - filter_backends = (TargetTypeFilterBackend, TargetIDFilterBackend, - UserFilterBackend, DjangoFilterBackend) + filter_backends = ( + TargetTypeFilterBackend, + TargetIDFilterBackend, + UserFilterBackend, + DjangoFilterBackend, + ) filterset_class = ActionFilterSet pagination_class = StandardPageNumberPagination @@ -40,24 +51,24 @@ def list(self, request, *args, **kwargs): headers = None queryset = self.filter_queryset(self.get_queryset()) no_of_records = queryset.count() - retrieval_threshold = getattr( - settings, "MESSAGE_RETRIEVAL_THRESHOLD", 10000) - pagination_keys = [self.paginator.page_query_param, - self.paginator.page_size_query_param] + retrieval_threshold = getattr(settings, "MESSAGE_RETRIEVAL_THRESHOLD", 10000) + pagination_keys = [ + self.paginator.page_query_param, + self.paginator.page_size_query_param, + ] query_param_keys = self.request.query_params - should_paginate = any( - [k in query_param_keys for k in pagination_keys]) or \ - no_of_records > retrieval_threshold + should_paginate = ( + any(k in query_param_keys for k in pagination_keys) + or no_of_records > retrieval_threshold + ) - if should_paginate and \ - "page_size" not in self.request.query_params.keys(): + if should_paginate and "page_size" not in self.request.query_params.keys(): self.paginator.page_size = retrieval_threshold if should_paginate: page = self.paginate_queryset(queryset) serializer = self.get_serializer(page, many=True) - headers = self.paginator.generate_link_header( - self.request, queryset) + headers = self.paginator.generate_link_header(self.request, queryset) else: serializer = self.get_serializer(queryset, many=True) diff --git a/onadata/apps/restservice/signals.py b/onadata/apps/restservice/signals.py index fea296a59d..e2db6b72b4 100644 --- a/onadata/apps/restservice/signals.py +++ b/onadata/apps/restservice/signals.py @@ -12,7 +12,7 @@ ) # pylint: disable=invalid-name -trigger_webhook = django.dispatch.Signal(providing_args=["instance"]) +trigger_webhook = django.dispatch.Signal() def call_webhooks(sender, **kwargs): # pylint: disable=unused-argument diff --git a/onadata/apps/restservice/tasks.py b/onadata/apps/restservice/tasks.py index e9acd263d2..1aeaaa4ac6 100644 --- a/onadata/apps/restservice/tasks.py +++ b/onadata/apps/restservice/tasks.py @@ -4,7 +4,7 @@ """ from onadata.apps.logger.models.instance import Instance from onadata.apps.restservice.utils import call_service -from onadata.celery import app +from onadata.celeryapp import app @app.task() diff --git a/onadata/apps/sms_support/parser.py b/onadata/apps/sms_support/parser.py index 05ed039ee1..66677e98af 100644 --- a/onadata/apps/sms_support/parser.py +++ b/onadata/apps/sms_support/parser.py @@ -107,7 +107,7 @@ def media_value(value, medias): return filename except (AttributeError, TypeError, binascii.Error) as e: raise SMSCastingError( - _("Media file format " "incorrect. %(except)r") % {"except": e}, + _("Media file format incorrect. %(except)r") % {"except": e}, xlsf_name, ) from e @@ -122,7 +122,7 @@ def media_value(value, medias): if choice.get("sms_option") == value: return choice.get("name") raise SMSCastingError( - _("No matching choice " "for '%(input)s'") % {"input": value}, xlsf_name + _("No matching choice for '%(input)s'") % {"input": value}, xlsf_name ) if xlsf_type == "select all that apply": values = [s.strip() for s in value.split()] @@ -272,7 +272,7 @@ def process_incoming_smses(username, incomings, id_string=None): # noqa C901 json_submissions = [] resp_str = { "success": _( - "[SUCCESS] Your submission has been accepted. " "It's ID is {{ id }}." + "[SUCCESS] Your submission has been accepted. It's ID is {{ id }}." ) } @@ -289,7 +289,7 @@ def process_incoming(incoming, id_string): responses.append( { "code": SMS_API_ERROR, - "text": _("Missing 'identity' " "or 'text' field."), + "text": _("Missing 'identity' or 'text' field."), } ) return @@ -298,7 +298,7 @@ def process_incoming(incoming, id_string): responses.append( { "code": SMS_API_ERROR, - "text": _("'identity' and 'text' fields can " "not be empty."), + "text": _("'identity' and 'text' fields can not be empty."), } ) return @@ -316,7 +316,7 @@ def process_incoming(incoming, id_string): { "code": SMS_SUBMISSION_REFUSED, "text": _( - "The form '%(id_string)s' does not " "accept SMS submissions." + "The form '%(id_string)s' does not accept SMS submissions." ) % {"id_string": xform.id_string}, } @@ -334,14 +334,12 @@ def process_incoming(incoming, id_string): resp_str.update({"success": json_survey.get("sms_response")}) # check that the form contains at least one filled group - meta_groups = sum([1 for k in list(json_submission) if k.startswith("meta")]) + meta_groups = sum(1 for k in list(json_submission) if k.startswith("meta")) if len(list(json_submission)) <= meta_groups: responses.append( { "code": SMS_PARSING_ERROR, - "text": _( - "There must be at least one group of " "questions filled." - ), + "text": _("There must be at least one group of questions filled."), } ) return @@ -362,7 +360,7 @@ def process_incoming(incoming, id_string): responses.append( { "code": SMS_SUBMISSION_REFUSED, - "text": _(f"Required field `{field}` is " "missing."), + "text": _(f"Required field `{field}` is missing."), } ) return diff --git a/onadata/apps/sms_support/providers/smssync.py b/onadata/apps/sms_support/providers/smssync.py index b146c90b04..c8258a00fa 100644 --- a/onadata/apps/sms_support/providers/smssync.py +++ b/onadata/apps/sms_support/providers/smssync.py @@ -36,14 +36,14 @@ def autodoc(url_root, username, id_string): "Ushaidi's SMS Sync" } + "

  1. " - + _("Download the SMS Sync App on your phone serving " "as a gateway.") + + _("Download the SMS Sync App on your phone serving as a gateway.") + "
  2. " + _("Configure the app to point to one of the following URLs") + '
    %(urla)s' + "
    %(urlb)s
    " - + _("Optionnaly set a keyword to prevent non-formhub " "messages to be sent.") + + _("Optionnaly set a keyword to prevent non-formhub messages to be sent.") + "
  3. " - + _("In the preferences, tick the box to allow " "replies from the server.") + + _("In the preferences, tick the box to allow replies from the server.") + "

" + _( "That's it. Now Send an SMS Formhub submission to the number " diff --git a/onadata/apps/sms_support/tools.py b/onadata/apps/sms_support/tools.py index 033dc6cc89..07e39b08c7 100644 --- a/onadata/apps/sms_support/tools.py +++ b/onadata/apps/sms_support/tools.py @@ -248,7 +248,7 @@ def prep_return(msg, comp=None): sensitive_fields += ("datetime",) # must not contain out-of-group questions - if sum([1 for e in groups if e.get("type") != "group"]): + if sum(1 for e in groups if e.get("type") != "group"): return prep_return(_("All your questions must be in groups.")) # all groups must have an sms_field bad_groups = [ @@ -333,7 +333,7 @@ def prep_return(msg, comp=None): # has date field with no sms_date_format if not json_survey.get("sms_date_format", ""): for group in groups: - if sum([1 for e in group.get("children", [{}]) if e.get("type") == "date"]): + if sum(1 for e in group.get("children", [{}]) if e.get("type") == "date"): warnings.append( "

  • You have 'date' fields without " "explicitly setting a date format. " @@ -344,7 +344,7 @@ def prep_return(msg, comp=None): if not json_survey.get("sms_date_format", ""): for group in groups: if sum( - [1 for e in group.get("children", [{}]) if e.get("type") == "datetime"] + 1 for e in group.get("children", [{}]) if e.get("type") == "datetime" ): warnings.append( "
  • You have 'datetime' fields without " diff --git a/onadata/apps/viewer/admin.py b/onadata/apps/viewer/admin.py index 8ca3929619..d6ef7188b8 100644 --- a/onadata/apps/viewer/admin.py +++ b/onadata/apps/viewer/admin.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from reversion.admin import VersionAdmin from django.contrib import admin @@ -6,13 +7,13 @@ class DataDictionaryAdmin(VersionAdmin, admin.ModelAdmin): - exclude = ('user',) + exclude = ("user",) def get_queryset(self, request): - qs = super(DataDictionaryAdmin, self).get_queryset(request) + queryset = super().get_queryset(request) if request.user.is_superuser: - return qs - return qs.filter(user=request.user) + return queryset + return queryset.filter(user=request.user) admin.site.register(DataDictionary, DataDictionaryAdmin) diff --git a/onadata/apps/viewer/management/commands/mark_start_times.py b/onadata/apps/viewer/management/commands/mark_start_times.py index 096fab9b39..9a92fcfe97 100644 --- a/onadata/apps/viewer/management/commands/mark_start_times.py +++ b/onadata/apps/viewer/management/commands/mark_start_times.py @@ -1,20 +1,23 @@ +# -*- coding: utf-8 -*- +""" +mark_start_times command - This is a one-time command to mark start times of old surveys +""" from django.core.management.base import BaseCommand -from django.utils.translation import gettext_lazy, gettext as _ +from django.utils.translation import gettext_lazy from onadata.apps.viewer.models.data_dictionary import DataDictionary class Command(BaseCommand): + """ + This is a one-time command to mark start times of old surveys. + """ + help = gettext_lazy( - "This is a one-time command to " "mark start times of old surveys." + "This is a one-time command to mark start times of old surveys." ) def handle(self, *args, **kwargs): - for dd in DataDictionary.objects.all(): - try: - dd.mark_start_time_boolean() - dd.save() - except Exception: - self.stderr.write( - _("Could not mark start time for DD: %(data)s") % {"data": repr(dd)} - ) + for xform in DataDictionary.objects.all(): + xform.mark_start_time_boolean() + xform.save() diff --git a/onadata/apps/viewer/management/commands/set_uuid_in_xml.py b/onadata/apps/viewer/management/commands/set_uuid_in_xml.py index 0faf46ed7b..919b0d1bec 100644 --- a/onadata/apps/viewer/management/commands/set_uuid_in_xml.py +++ b/onadata/apps/viewer/management/commands/set_uuid_in_xml.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +set_uuid_in_xml command - Insert UUID into XML of all existing XForms. +""" from django.core.management.base import BaseCommand from django.utils.translation import gettext as _, gettext_lazy @@ -6,15 +10,20 @@ class Command(BaseCommand): + """ + set_uuid_in_xml command - Insert UUID into XML of all existing XForms. + """ + help = gettext_lazy("Insert UUID into XML of all existing XForms") def handle(self, *args, **kwargs): - self.stdout.write(_('%(nb)d XForms to update') - % {'nb': DataDictionary.objects.count()}) - for i, dd in enumerate( - queryset_iterator(DataDictionary.objects.all())): - if dd.xls: - dd.set_uuid_in_xml() - super(DataDictionary, dd).save() + self.stdout.write( + _("%(nb)d XForms to update") % {"nb": DataDictionary.objects.count()} + ) + for i, xform in enumerate(queryset_iterator(DataDictionary.objects.all())): + if xform.xls: + xform.set_uuid_in_xml() + # pylint: disable=bad-super-call + super(DataDictionary, xform).save() if (i + 1) % 10 == 0: - self.stdout.write(_('Updated %(nb)d XForms...') % {'nb': i}) + self.stdout.write(_(f"Updated {i} XForms...")) diff --git a/onadata/apps/viewer/models/data_dictionary.py b/onadata/apps/viewer/models/data_dictionary.py index 59d15d2154..4de9a2eeaa 100644 --- a/onadata/apps/viewer/models/data_dictionary.py +++ b/onadata/apps/viewer/models/data_dictionary.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ DataDictionary model. """ @@ -65,6 +65,7 @@ def process_xlsform(xls, default_name): # adopted from pyxform.utils.sheet_to_csv +# pylint: disable=too-many-branches,too-many-locals def sheet_to_csv(xls_content, sheet_name): """Writes a csv file of a specified sheet from a an excel file @@ -78,9 +79,7 @@ def sheet_to_csv(xls_content, sheet_name): sheet = workbook.get_sheet_by_name(sheet_name) if not sheet or sheet.max_column < 2: - raise Exception( - _("Sheet <'%(sheet_name)s'> has no data." % {"sheet_name": sheet_name}) - ) + raise Exception(_(f"Sheet <'{sheet_name}'> has no data.")) csv_file = BytesIO() @@ -143,7 +142,7 @@ def __init__(self, *args, **kwargs): self.instances_for_export = lambda d: d.instances.all() self.has_external_choices = False self._id_string_changed = False - super(DataDictionary, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def __str__(self): return getattr(self, "id_string", "") @@ -200,7 +199,7 @@ def save(self, *args, **kwargs): if "skip_xls_read" in kwargs: del kwargs["skip_xls_read"] - super(DataDictionary, self).save(*args, **kwargs) + super().save(*args, **kwargs) def file_name(self): return os.path.split(self.xls.name)[-1] @@ -214,13 +213,14 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): """ if instance.project: # clear cache - safe_delete("{}{}".format(PROJ_FORMS_CACHE, instance.project.pk)) - safe_delete("{}{}".format(PROJ_BASE_FORMS_CACHE, instance.project.pk)) + safe_delete(f"{PROJ_FORMS_CACHE}{instance.project.pk}") + safe_delete(f"{PROJ_BASE_FORMS_CACHE}{instance.project.pk}") # seems the super is not called, have to get xform from here xform = XForm.objects.get(pk=instance.pk) if created: + # pylint: disable=import-outside-toplevel from onadata.libs.permissions import OwnerRole OwnerRole.add(instance.user, xform) @@ -228,6 +228,7 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): if instance.created_by and instance.user != instance.created_by: OwnerRole.add(instance.created_by, xform) + # pylint: disable=import-outside-toplevel from onadata.libs.utils.project_utils import ( set_project_perms_to_xform_async, ) # noqa @@ -235,6 +236,7 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): try: set_project_perms_to_xform_async.delay(xform.pk, instance.project.pk) except OperationalError: + # pylint: disable=import-outside-toplevel from onadata.libs.utils.project_utils import ( set_project_perms_to_xform, ) # noqa @@ -248,6 +250,7 @@ def set_object_permissions(sender, instance=None, created=False, **kwargs): size = f.tell() f.seek(0) + # pylint: disable=import-outside-toplevel from onadata.apps.main.models.meta_data import MetaData data_file = InMemoryUploadedFile( diff --git a/onadata/apps/viewer/models/export.py b/onadata/apps/viewer/models/export.py index fc3ea9bf3f..b506ebc651 100644 --- a/onadata/apps/viewer/models/export.py +++ b/onadata/apps/viewer/models/export.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Export model. """ @@ -38,7 +38,7 @@ def get_export_options_query_kwargs(options): if field in options: field_value = options.get(field) - key = "options__{}".format(field) + key = f"options__{field}" options_kwargs[key] = field_value return options_kwargs @@ -162,7 +162,7 @@ class Meta: unique_together = (("xform", "filename"),) def __str__(self): - return "%s - %s (%s)" % (self.export_type, self.xform, self.filename) + return f"{self.export_type} - {self.xform} ({self.filename})" def save(self, *args, **kwargs): # pylint: disable=arguments-differ if not self.pk and self.xform: @@ -177,11 +177,11 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ # update time_of_last_submission with # xform.time_of_last_submission_update - # pylint: disable=E1101 + # pylint: disable=no-member self.time_of_last_submission = self.xform.time_of_last_submission_update() if self.filename: self.internal_status = Export.SUCCESSFUL - super(Export, self).save(*args, **kwargs) + super().save(*args, **kwargs) @classmethod def _delete_oldest_export(cls, xform, export_type): @@ -213,7 +213,7 @@ def status(self): # need to have this since existing models will have their # internal_status set to PENDING - the default return Export.SUCCESSFUL - elif self.internal_status == Export.FAILED: + if self.internal_status == Export.FAILED: return Export.FAILED return Export.PENDING @@ -230,7 +230,7 @@ def set_filename(self, filename): def _update_filedir(self): if not self.filename: raise AssertionError() - # pylint: disable=E1101 + # pylint: disable=no-member self.filedir = os.path.join( self.xform.user.username, "exports", self.xform.id_string, self.export_type ) @@ -258,6 +258,7 @@ def full_filepath(self): except NotImplementedError: # read file from s3 _name, ext = os.path.splitext(self.filepath) + # pylint: disable=consider-using-with tmp = NamedTemporaryFile(suffix=ext, delete=False) f = default_storage.open(self.filepath) tmp.write(f.read()) @@ -280,7 +281,7 @@ def exports_outdated(cls, xform, export_type, options=None): xform=xform, export_type=export_type, internal_status__in=[Export.SUCCESSFUL, Export.PENDING], - **export_options + **export_options, ).latest("created_on") except cls.DoesNotExist: return True diff --git a/onadata/apps/viewer/signals.py b/onadata/apps/viewer/signals.py index 02a211fe68..5020262964 100644 --- a/onadata/apps/viewer/signals.py +++ b/onadata/apps/viewer/signals.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Viewer signals module. """ @@ -11,14 +11,15 @@ from onadata.apps.viewer.models import ParsedInstance from onadata.libs.utils.osm import save_osm_data_async -ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = \ - getattr(settings, 'ASYNC_POST_SUBMISSION_PROCESSING_ENABLED', False) +ASYNC_POST_SUBMISSION_PROCESSING_ENABLED = getattr( + settings, "ASYNC_POST_SUBMISSION_PROCESSING_ENABLED", False +) -# pylint: disable=C0103 -process_submission = django.dispatch.Signal(providing_args=['instance']) +# pylint: disable=invalid-name +process_submission = django.dispatch.Signal() -def post_save_osm_data(instance_id): # pylint: disable=W0613 +def post_save_osm_data(instance_id): # pylint: disable=unused-argument """ Process OSM data post submission. """ @@ -34,31 +35,31 @@ def _post_process_submissions(instance): post_save_osm_data(instance.pk) -def post_save_submission(sender, **kwargs): # pylint: disable=W0613 +def post_save_submission(sender, **kwargs): # pylint: disable=unused-argument """ Calls webhooks and OSM data processing for ParsedInstance model. """ - parsed_instance = kwargs.get('instance') - created = kwargs.get('created') + parsed_instance = kwargs.get("instance") + created = kwargs.get("created") if created: _post_process_submissions(parsed_instance.instance) post_save.connect( - post_save_submission, - sender=ParsedInstance, - dispatch_uid='post_save_submission') + post_save_submission, sender=ParsedInstance, dispatch_uid="post_save_submission" +) -def process_saved_submission(sender, **kwargs): # pylint: disable=W0613 +def process_saved_submission(sender, **kwargs): # pylint: disable=unused-argument """ Calls webhooks and OSM data processing for Instance model. """ - instance = kwargs.get('instance') + instance = kwargs.get("instance") if instance: _post_process_submissions(instance) -process_submission.connect(process_saved_submission, sender=Instance, - dispatch_uid='process_saved_submission') +process_submission.connect( + process_saved_submission, sender=Instance, dispatch_uid="process_saved_submission" +) diff --git a/onadata/apps/viewer/tasks.py b/onadata/apps/viewer/tasks.py index 19da86e727..28ec746657 100644 --- a/onadata/apps/viewer/tasks.py +++ b/onadata/apps/viewer/tasks.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Export tasks. """ @@ -18,7 +18,7 @@ ExportConnectionError, ExportTypeError, ) -from onadata.celery import app +from onadata.celeryapp import app from onadata.libs.exceptions import NoRecordsFoundError from onadata.libs.utils.common_tools import get_boolean_value, report_exception from onadata.libs.utils.export_tools import ( diff --git a/onadata/apps/viewer/xls_writer.py b/onadata/apps/viewer/xls_writer.py index 9854d78d02..f7e1079bc3 100644 --- a/onadata/apps/viewer/xls_writer.py +++ b/onadata/apps/viewer/xls_writer.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +XlsWriter module - generate a spreadsheet workbook in XLSX format. +""" from builtins import str as text from collections import defaultdict from io import StringIO @@ -7,13 +11,16 @@ from onadata.apps.logger.models.xform import question_types_to_exclude -class XlsWriter(object): +# pylint: disable=too-many-instance-attributes, +class XlsWriter: + """XlsWriter class - generate a spreadsheet workbook in XLSX format.""" def __init__(self): self.set_file() self.reset_workbook() self.sheet_name_limit = 30 self._generated_sheet_name_dict = {} + self._data_dictionary = None def set_file(self, file_object=None): """ @@ -25,6 +32,7 @@ def set_file(self, file_object=None): self._file = StringIO() def reset_workbook(self): + """Reset a Workbook to sensible default.""" self._workbook = Workbook() self._sheets = {} self._columns = defaultdict(list) @@ -32,11 +40,13 @@ def reset_workbook(self): self._generated_sheet_name_dict = {} def add_sheet(self, name): + """Add a given ``name`` sheet to this workbook.""" unique_sheet_name = self._unique_name_for_xls(name) sheet = self._workbook.add_sheet(unique_sheet_name) self._sheets[unique_sheet_name] = sheet def add_column(self, sheet_name, column_name): + """Add a ``column_name`` to the given ``sheet_name`` to this workbook.""" index = len(self._columns[sheet_name]) sheet = self._sheets.get(sheet_name) if sheet: @@ -44,6 +54,7 @@ def add_column(self, sheet_name, column_name): self._columns[sheet_name].append(column_name) def add_row(self, sheet_name, row): + """Add a ``row`` to the given ``sheet_name`` to this workbook.""" i = self._current_index[sheet_name] columns = self._columns[sheet_name] for key in list(row): @@ -51,25 +62,27 @@ def add_row(self, sheet_name, row): self.add_column(sheet_name, key) for j, column_name in enumerate(self._columns[sheet_name]): # leaving this untranslated as I'm not sure it's in django context - self._sheets[sheet_name].write(i, j, row.get(column_name, u"n/a")) + self._sheets[sheet_name].write(i, j, row.get(column_name, "n/a")) self._current_index[sheet_name] += 1 def add_obs(self, obs): + """Add data in ``obs`` dictionary into specified sheets to this workbook.""" self._fix_indices(obs) for sheet_name, rows in obs.items(): for row in rows: actual_sheet_name = self._generated_sheet_name_dict.get( - sheet_name, sheet_name) + sheet_name, sheet_name + ) self.add_row(actual_sheet_name, row) def _fix_indices(self, obs): for sheet_name, rows in obs.items(): for row in rows: - row[u'_index'] += self._current_index[sheet_name] - if row[u'_parent_index'] == -1: + row["_index"] += self._current_index[sheet_name] + if row["_parent_index"] == -1: continue - i = self._current_index[row[u'_parent_table_name']] - row[u'_parent_index'] += i + i = self._current_index[row["_parent_table_name"]] + row["_parent_index"] += i def write_tables_to_workbook(self, tables): """ @@ -88,10 +101,12 @@ def write_tables_to_workbook(self, tables): return self._workbook def save_workbook_to_file(self): + """Saves the XLSX workbook to a file.""" self._workbook.save(self._file) return self._file def set_data_dictionary(self, data_dictionary): + """Set the data_dictionary XForm model object for this class.""" self._data_dictionary = data_dictionary self.reset_workbook() self._add_sheets() @@ -100,18 +115,20 @@ def set_data_dictionary(self, data_dictionary): self.add_obs(obs) def _add_sheets(self): - for e in self._data_dictionary.get_survey_elements(): - if isinstance(e, Section): - sheet_name = e.name - self.add_sheet(sheet_name) - for f in e.children: - if isinstance(f, Question) and\ - not question_types_to_exclude(f.type): - self.add_column(sheet_name, f.name) + if self._data_dictionary: + for e in self._data_dictionary.get_survey_elements(): + if isinstance(e, Section): + sheet_name = e.name + self.add_sheet(sheet_name) + for f in e.children: + if isinstance(f, Question) and not question_types_to_exclude( + f.type + ): + self.add_column(sheet_name, f.name) def _unique_name_for_xls(self, sheet_name): # excel worksheet name limit seems to be 31 characters (30 to be safe) - unique_sheet_name = sheet_name[0:self.sheet_name_limit] + unique_sheet_name = sheet_name[0 : self.sheet_name_limit] unique_sheet_name = self._generate_unique_sheet_name(unique_sheet_name) self._generated_sheet_name_dict[sheet_name] = unique_sheet_name return unique_sheet_name @@ -120,15 +137,15 @@ def _generate_unique_sheet_name(self, sheet_name): # check if sheet name exists if sheet_name not in self._sheets: return sheet_name - else: - i = 1 - unique_name = sheet_name - while(unique_name in self._sheets): - number_len = len(text(i)) - allowed_name_len = self.sheet_name_limit - number_len - # make name required len - if(len(unique_name) > allowed_name_len): - unique_name = unique_name[0:allowed_name_len] - unique_name = "{0}{1}".format(unique_name, i) - i = i + 1 - return unique_name + + i = 1 + unique_name = sheet_name + while unique_name in self._sheets: + number_len = len(text(i)) + allowed_name_len = self.sheet_name_limit - number_len + # make name required len + if len(unique_name) > allowed_name_len: + unique_name = unique_name[0:allowed_name_len] + unique_name = f"{unique_name}{i}" + i = i + 1 + return unique_name diff --git a/onadata/celery.py b/onadata/celery.py deleted file mode 100644 index 2c598e2c0e..0000000000 --- a/onadata/celery.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Celery module for onadata. -""" -from __future__ import absolute_import, unicode_literals - -import os - -from django.conf import settings - -import celery -import sentry_sdk -from sentry_sdk.integrations.celery import CeleryIntegration - -# set the default Django settings module for the 'celery' program. -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "onadata.settings.common") - - -class Celery(celery.Celery): - """ - Celery class that allows Sentry configuration. - """ - - def on_configure(self): # pylint: disable=method-hidden - """ - Register Sentry for celery tasks. - """ - if getattr(settings, "RAVEN_CONFIG", None): - sentry_sdk.init( - dsn=settings.RAVEN_CONFIG["dsn"], integrations=[CeleryIntegration()] - ) - - -app = Celery(__name__) # pylint: disable=invalid-name - -# Using a string here means the worker will not have to -# pickle the object when using Windows. -app.config_from_object("django.conf:settings", namespace="CELERY") -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) -app.conf.broker_transport_options = {"visibility_timeout": 10} - - -@app.task -def debug_task(): - """A test task""" - print("Hello!") - return True diff --git a/onadata/celeryapp.py b/onadata/celeryapp.py new file mode 100644 index 0000000000..8cd3e75bde --- /dev/null +++ b/onadata/celeryapp.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +""" +Celery module for onadata. +""" +import os + +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "onadata.settings.common") + +app = Celery(__name__) # pylint: disable=invalid-name + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object("django.conf:settings", namespace="CELERY") +app.autodiscover_tasks() +app.conf.broker_transport_options = {"visibility_timeout": 10} + + +@app.task +def debug_task(): + """A test task""" + print("Hello!") + return True diff --git a/onadata/devwsgi.py b/onadata/devwsgi.py index 0b30f727e4..d84bdb0830 100644 --- a/onadata/devwsgi.py +++ b/onadata/devwsgi.py @@ -1,5 +1,5 @@ """ -WSGI config for mspray project. +WSGI config It exposes the WSGI callable as a module-level variable named ``application``. @@ -8,11 +8,12 @@ """ import os -import uwsgi -from uwsgidecorators import timer -from django.utils import autoreload from django.core.wsgi import get_wsgi_application +from django.utils import autoreload + +import uwsgi # pylint: disable=import-error +from uwsgidecorators import timer os.environ.setdefault("DJANGO_SETTINGS_MODULE", "onadata.settings.common") @@ -20,6 +21,7 @@ @timer(3) -def change_code_gracefull_reload(sig): - if autoreload.code_changed(): +def change_code_gracefull_reload(sig): # pylint: disable=unused-argument + """Reload uWSGI whenever the code changes""" + if autoreload.file_changed: uwsgi.reload() diff --git a/onadata/libs/authentication.py b/onadata/libs/authentication.py index 739a6cffae..81bd4cae1f 100644 --- a/onadata/libs/authentication.py +++ b/onadata/libs/authentication.py @@ -21,6 +21,7 @@ from oauth2_provider.models import AccessToken from oauth2_provider.oauth2_validators import OAuth2Validator from oauth2_provider.settings import oauth2_settings +from oidc.utils import authenticate_sso from rest_framework import exceptions from rest_framework.authentication import ( BaseAuthentication, @@ -29,16 +30,10 @@ ) from rest_framework.authtoken.models import Token from rest_framework.exceptions import AuthenticationFailed -from oidc.utils import authenticate_sso from onadata.apps.api.models.temp_token import TempToken from onadata.apps.api.tasks import send_account_lockout_email -from onadata.libs.utils.cache_tools import ( - LOCKOUT_IP, - LOGIN_ATTEMPTS, - cache, - safe_key, -) +from onadata.libs.utils.cache_tools import LOCKOUT_IP, LOGIN_ATTEMPTS, cache, safe_key from onadata.libs.utils.common_tags import API_TOKEN from onadata.libs.utils.email import get_account_lockout_email_data @@ -152,7 +147,7 @@ def authenticate(self, request): raise exceptions.AuthenticationFailed(error_msg) if len(auth) > 2: error_msg = _( - "Invalid token header. " "Token string should not contain spaces." + "Invalid token header. Token string should not contain spaces." ) raise exceptions.AuthenticationFailed(error_msg) @@ -240,7 +235,7 @@ class SSOHeaderAuthentication(BaseAuthentication): cookie or HTTP_SSO header. """ - def authenticate(self, request): # pylint: disable=no-self-use + def authenticate(self, request): return authenticate_sso(request) @@ -367,9 +362,22 @@ class MasterReplicaOAuth2Validator(OAuth2Validator): """ Custom OAuth2Validator class that takes into account replication lag between Master & Replica databases - https://github.com/jazzband/django-oauth-toolkit/blob/3bde632d5722f1f85ffcd8277504955321f00fff/oauth2_provider/oauth2_validators.py#L49 + https://github.com/jazzband/django-oauth-toolkit/blob/ + 3bde632d5722f1f85ffcd8277504955321f00fff/oauth2_provider/oauth2_validators.py#L49 """ + def introspect_token(self, token, token_type_hint, request, *args, **kwargs): + """See oauthlib.oauth2.rfc6749.request_validator""" + raise NotImplementedError("Subclasses must implement this method.") + + def validate_silent_authorization(self, request): + """See oauthlib.oauth2.rfc6749.request_validator""" + raise NotImplementedError("Subclasses must implement this method.") + + def validate_silent_login(self, request): + """See oauthlib.oauth2.rfc6749.request_validator""" + raise NotImplementedError("Subclasses must implement this method.") + def validate_bearer_token(self, token, scopes, request): if not token: return False diff --git a/onadata/libs/baseviewset.py b/onadata/libs/baseviewset.py index 7748951883..9329b85651 100644 --- a/onadata/libs/baseviewset.py +++ b/onadata/libs/baseviewset.py @@ -1,2 +1,10 @@ -class DefaultBaseViewset(object): - pass +# -*- coding: utf-8 -*- +""" +The DefaultBaseViewset class +""" + + +class DefaultBaseViewset: + """ + The DefaultBaseViewset class + """ diff --git a/onadata/libs/data/__init__.py b/onadata/libs/data/__init__.py index a8efc0e3a5..45540a6fcf 100644 --- a/onadata/libs/data/__init__.py +++ b/onadata/libs/data/__init__.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Data utility functions. """ diff --git a/onadata/libs/data/query.py b/onadata/libs/data/query.py index c58a19a4e6..614cf27add 100644 --- a/onadata/libs/data/query.py +++ b/onadata/libs/data/query.py @@ -1,11 +1,14 @@ +# -*- coding: utf-8 -*- +""" +Query data utility functions. +""" import logging + from django.conf import settings from django.db import connection -from onadata.libs.utils.common_tags import ( - SUBMISSION_TIME, SUBMITTED_BY) from onadata.apps.logger.models.data_view import DataView - +from onadata.libs.utils.common_tags import SUBMISSION_TIME, SUBMITTED_BY logger = logging.getLogger(__name__) @@ -14,10 +17,7 @@ def _dictfetchall(cursor): "Returns all rows from a cursor as a dict" desc = cursor.description - return [ - dict(zip([col[0] for col in desc], row)) - for row in cursor.fetchall() - ] + return [dict(zip([col[0] for col in desc], row)) for row in cursor.fetchall()] def _execute_query(query, to_dict=True): @@ -29,8 +29,7 @@ def _execute_query(query, to_dict=True): def _get_fields_of_type(xform, types): k = [] - survey_elements = flatten( - [xform.get_survey_elements_of_type(t) for t in types]) + survey_elements = flatten([xform.get_survey_elements_of_type(t) for t in types]) for element in survey_elements: name = element.get_abbreviated_xpath() @@ -40,14 +39,15 @@ def _get_fields_of_type(xform, types): def _additional_data_view_filters(data_view): + # pylint: disable=protected-access where, where_params = DataView._get_where_clause(data_view) data_view_where = "" if where: - data_view_where = u" AND " + u" AND ".join(where) + data_view_where = " AND " + " AND ".join(where) - for it in where_params: - data_view_where = data_view_where.replace('%s', "'{}'".format(it), 1) + for param in where_params: + data_view_where = data_view_where.replace("%s", f"'{param}'", 1) return data_view_where @@ -55,29 +55,36 @@ def _additional_data_view_filters(data_view): def _json_query(field): if not field: logger.info("Field is empty") - return "json->>'%s'" % field - return "json->>'%s'" % field.replace("'", "''") + return f"json->>'{field}'" + + _field = field.replace("'", "''") + return f"json->>'{_field}'" -def _postgres_count_group_field_n_group_by(field, name, xform, group_by, - data_view): + +def _postgres_count_group_field_n_group_by(field, name, xform, group_by, data_view): string_args = _query_args(field, name, xform, group_by) if is_date_field(xform, field): - string_args['json'] = "to_char(to_date(%(json)s, 'YYYY-MM-DD'), 'YYYY"\ - "-MM-DD')" % string_args + string_args["json"] = ( + "to_char(to_date(%(json)s, 'YYYY-MM-DD'), 'YYYY" "-MM-DD')" % string_args + ) additional_filters = "" if data_view: additional_filters = _additional_data_view_filters(data_view) restricted_string = _restricted_query(xform) - query = "SELECT %(json)s AS \"%(name)s\", "\ - "%(group_by)s AS \"%(group_name)s\", "\ - "count(*) as count "\ - "FROM %(table)s WHERE " + restricted_string + \ - "AND deleted_at IS NULL " + additional_filters + \ - " GROUP BY %(json)s, %(group_by)s" + \ - " ORDER BY %(json)s, %(group_by)s" + query = ( + 'SELECT %(json)s AS "%(name)s", ' + '%(group_by)s AS "%(group_name)s", ' + "count(*) as count " + "FROM %(table)s WHERE " + + restricted_string + + "AND deleted_at IS NULL " + + additional_filters + + " GROUP BY %(json)s, %(group_by)s" + + " ORDER BY %(json)s, %(group_by)s" + ) query = query % string_args return query @@ -86,8 +93,9 @@ def _postgres_count_group_field_n_group_by(field, name, xform, group_by, def _postgres_count_group(field, name, xform, data_view=None): string_args = _query_args(field, name, xform) if is_date_field(xform, field): - string_args['json'] = "to_char(to_date(%(json)s, 'YYYY-MM-DD'), 'YYYY"\ - "-MM-DD')" % string_args + string_args["json"] = ( + "to_char(to_date(%(json)s, 'YYYY-MM-DD'), 'YYYY" "-MM-DD')" % string_args + ) additional_filters = "" if data_view: @@ -99,10 +107,15 @@ def _postgres_count_group(field, name, xform, data_view=None): string_args["join"] = "i LEFT JOIN auth_user au ON au.id = i.user_id" restricted_string = _restricted_query(xform) - sql_query = "SELECT %(json)s AS \"%(name)s\", COUNT(*) AS count FROM " \ - "%(table)s %(join)s WHERE " + restricted_string + \ - " AND deleted_at IS NULL " + additional_filters + " GROUP BY %(json)s"\ + sql_query = ( + 'SELECT %(json)s AS "%(name)s", COUNT(*) AS count FROM ' + "%(table)s %(join)s WHERE " + + restricted_string + + " AND deleted_at IS NULL " + + additional_filters + + " GROUP BY %(json)s" " ORDER BY %(json)s" + ) sql_query = sql_query % string_args return sql_query @@ -111,8 +124,9 @@ def _postgres_count_group(field, name, xform, data_view=None): def _postgres_aggregate_group_by(field, name, xform, group_by, data_view=None): string_args = _query_args(field, name, xform, group_by) if is_date_field(xform, field): - string_args['json'] = "to_char(to_date(%(json)s, 'YYYY-MM-DD'), 'YYYY"\ - "-MM-DD')" % string_args + string_args["json"] = ( + "to_char(to_date(%(json)s, 'YYYY-MM-DD'), 'YYYY" "-MM-DD')" % string_args + ) additional_filters = "" if data_view: @@ -123,27 +137,37 @@ def _postgres_aggregate_group_by(field, name, xform, group_by, data_view=None): if isinstance(group_by, list): group_by_group_by = [] for i, __ in enumerate(group_by): - group_by_select += "%(group_by" + str(i) + \ - ")s AS \"%(group_name" + str(i) + ")s\", " + group_by_select += ( + "%(group_by" + str(i) + ')s AS "%(group_name' + str(i) + ')s", ' + ) group_by_group_by.append("%(group_by" + str(i) + ")s") group_by_group_by = ",".join(group_by_group_by) else: - group_by_select = "%(group_by)s AS \"%(group_name)s\"," + group_by_select = '%(group_by)s AS "%(group_name)s",' group_by_group_by = "%(group_by)s" restricted_string = _restricted_query(xform) aggregation_string = "COUNT(%(json)s) AS count " if field in get_numeric_fields(xform) or not isinstance(group_by, list): - aggregation_string += ", SUM((%(json)s)::numeric) AS sum, " \ - "AVG((%(json)s)::numeric) AS mean " + aggregation_string += ( + ", SUM((%(json)s)::numeric) AS sum, " "AVG((%(json)s)::numeric) AS mean " + ) else: group_by_select = "%(json)s AS %(name)s, " + group_by_select group_by_group_by = "%(json)s, " + group_by_group_by - query = "SELECT " + group_by_select + aggregation_string + \ - "FROM %(table)s WHERE " + restricted_string + \ - " AND deleted_at IS NULL " + additional_filters + \ - " GROUP BY " + group_by_group_by + \ - " ORDER BY " + group_by_group_by + query = ( + "SELECT " + + group_by_select + + aggregation_string + + "FROM %(table)s WHERE " + + restricted_string + + " AND deleted_at IS NULL " + + additional_filters + + " GROUP BY " + + group_by_group_by + + " ORDER BY " + + group_by_group_by + ) return query % string_args @@ -151,9 +175,11 @@ def _postgres_aggregate_group_by(field, name, xform, group_by, data_view=None): def _postgres_select_key(field, name, xform): string_args = _query_args(field, name, xform) restricted_string = _restricted_query(xform) - query = "SELECT %(json)s AS \"%(name)s\" FROM %(table)s WHERE " + \ - restricted_string + " AND deleted_at IS NULL "\ - + query = ( + 'SELECT %(json)s AS "%(name)s" FROM %(table)s WHERE ' + + restricted_string + + " AND deleted_at IS NULL " + ) return query % string_args @@ -166,34 +192,36 @@ def _restricted_query(xform): def _query_args(field, name, xform, group_by=None): qargs = { - 'table': 'logger_instance', - 'json': _json_query(field), - 'name': name, - 'restrict_field': 'xform_id', - 'restrict_value': xform.pk, - 'join': '', + "table": "logger_instance", + "json": _json_query(field), + "name": name, + "restrict_field": "xform_id", + "restrict_value": xform.pk, + "join": "", } if xform.is_merged_dataset: xforms = tuple( - __ for __ in xform.mergedxform.xforms.filter( - deleted_at__isnull=True).values_list('id', flat=True) + __ + for __ in xform.mergedxform.xforms.filter( + deleted_at__isnull=True + ).values_list("id", flat=True) ) or (xform.pk, xform.pk) - qargs['restrict_value'] = xforms + qargs["restrict_value"] = xforms if isinstance(group_by, list): for i, v in enumerate(group_by): - qargs['group_name%d' % i] = v - qargs['group_by%d' % i] = _json_query(v) + qargs[f"group_name{i}"] = v + qargs[f"group_by{i}"] = _json_query(v) else: - qargs['group_name'] = group_by - qargs['group_by'] = _json_query(group_by) + qargs["group_name"] = group_by + qargs["group_by"] = _json_query(group_by) return qargs def _select_key(field, name, xform): - if using_postgres: + if using_postgres(): result = _postgres_select_key(field, name, xform) else: raise Exception("Unsupported Database") @@ -202,23 +230,25 @@ def _select_key(field, name, xform): def flatten(lst): + """Flattens a list of lists.""" return [item for sublist in lst for item in sublist] def get_date_fields(xform): """List of date field names for specified xform""" return [SUBMISSION_TIME] + _get_fields_of_type( - xform, ['date', 'datetime', 'start', 'end', 'today']) + xform, ["date", "datetime", "start", "end", "today"] + ) def get_field_records(field, xform): - result = _execute_query(_select_key(field, field, xform), - to_dict=False) + """Queries and returns all records of the given field.""" + result = _execute_query(_select_key(field, field, xform), to_dict=False) return [float(i[0]) for i in result if i[0] is not None] -def get_form_submissions_grouped_by_field(xform, field, name=None, - data_view=None): +# pylint: disable=invalid-name +def get_form_submissions_grouped_by_field(xform, field, name=None, data_view=None): """Number of submissions grouped by field""" if not name: name = field @@ -226,41 +256,43 @@ def get_form_submissions_grouped_by_field(xform, field, name=None, return _execute_query(_postgres_count_group(field, name, xform, data_view)) -def get_form_submissions_aggregated_by_select_one(xform, field, name=None, - group_by=None, - data_view=None): +# pylint: disable=invalid-name +def get_form_submissions_aggregated_by_select_one( + xform, field, name=None, group_by=None, data_view=None +): """Number of submissions grouped and aggregated by select_one field""" if not name: name = field - return _execute_query(_postgres_aggregate_group_by(field, - name, - xform, - group_by, - data_view)) + return _execute_query( + _postgres_aggregate_group_by(field, name, xform, group_by, data_view) + ) -def get_form_submissions_grouped_by_select_one(xform, field, group_by, - name=None, data_view=None): +# pylint: disable=invalid-name +def get_form_submissions_grouped_by_select_one( + xform, field, group_by, name=None, data_view=None +): """Number of submissions disaggregated by select_one field""" if not name: name = field - return _execute_query(_postgres_count_group_field_n_group_by(field, - name, - xform, - group_by, - data_view)) + return _execute_query( + _postgres_count_group_field_n_group_by(field, name, xform, group_by, data_view) + ) def get_numeric_fields(xform): """List of numeric field names for specified xform""" - return _get_fields_of_type(xform, ['decimal', 'integer']) + return _get_fields_of_type(xform, ["decimal", "integer"]) def is_date_field(xform, field): + """Returns True if an XForm field is a date field.""" return field in get_date_fields(xform) -@property def using_postgres(): - return settings.DATABASES[ - 'default']['ENGINE'] == 'django.db.backends.postgresql' + """Returns True if django.db.backends.postgresql is the DB engine in use""" + return settings.DATABASES["default"]["ENGINE"] in [ + "django.db.backends.postgresql", + "django.contrib.gis.db.backends.postgis", + ] diff --git a/onadata/libs/data/statistics.py b/onadata/libs/data/statistics.py index bb5216a9b3..661f0e4324 100644 --- a/onadata/libs/data/statistics.py +++ b/onadata/libs/data/statistics.py @@ -1,9 +1,14 @@ +# -*- coding: utf-8 -*- +""" +Statistics utility functions. +""" import numpy as np + from onadata.apps.api.tools import DECIMAL_PRECISION from onadata.libs.data.query import get_field_records, get_numeric_fields -def _chk_asarray(a, axis): +def _chk_asarray(a, axis): # pylint: disable=invalid-name if axis is None: a = np.ravel(a) outaxis = 0 @@ -14,10 +19,12 @@ def _chk_asarray(a, axis): def get_mean(values): + """Returns numpy.mean() of values.""" return np.mean(values) def get_median(values, axis=None): + """Returns numpy.median() of values for the given axis""" return np.median(values, axis) @@ -26,14 +33,14 @@ def get_mode(values, axis=0): Adapted from https://github.com/scipy/scipy/blob/master/scipy/stats/stats.py#L568 """ - a, axis = _chk_asarray(values, axis) - scores = np.unique(np.ravel(a)) # get ALL unique values + a, axis = _chk_asarray(values, axis) # pylint: disable=invalid-name + scores = np.unique(np.ravel(a)) # get ALL unique values testshape = list(a.shape) testshape[axis] = 1 oldmostfreq = np.zeros(testshape) oldcounts = np.zeros(testshape) for score in scores: - template = (a == score) + template = a == score counts = np.expand_dims(np.sum(template, axis), axis) mostfrequent = np.where(counts > oldcounts, score, oldmostfreq) oldcounts = np.maximum(counts, oldcounts) @@ -42,10 +49,16 @@ def get_mode(values, axis=0): def get_median_for_field(field, xform): + """Returns numpy.median() of values in the given field.""" return np.median(get_field_records(field, xform)) +# pylint: disable=invalid-name def get_median_for_numeric_fields_in_form(xform, field=None): + """Get's numpy.median() of values in numeric fields. + + Returns a dict with the fields as key and the median as a value. + """ data = {} for field_name in [field] if field else get_numeric_fields(xform): median = get_median_for_field(field_name, xform) @@ -54,10 +67,16 @@ def get_median_for_numeric_fields_in_form(xform, field=None): def get_mean_for_field(field, xform): + """Returns numpy.mean() of values in the given field.""" return np.mean(get_field_records(field, xform)) +# pylint: disable=invalid-name def get_mean_for_numeric_fields_in_form(xform, field): + """Get's numpy.mean() of values in numeric fields. + + Returns a dict with the fields as key and the mean as a value. + """ data = {} for field_name in [field] if field else get_numeric_fields(xform): mean = get_mean_for_field(field_name, xform) @@ -66,12 +85,18 @@ def get_mean_for_numeric_fields_in_form(xform, field): def get_mode_for_field(field, xform): - a = np.array(get_field_records(field, xform)) - m, count = get_mode(a) + """Returns mode of values in the given field.""" + a = np.array(get_field_records(field, xform)) # pylint: disable=invalid-name + m, _count = get_mode(a) # pylint: disable=invalid-name return m +# pylint: disable=invalid-name def get_mode_for_numeric_fields_in_form(xform, field=None): + """Get's mode of values in numeric fields. + + Returns a dict with the fields as key and the mode as a value. + """ data = {} for field_name in [field] if field else get_numeric_fields(xform): mode = get_mode_for_field(field_name, xform) @@ -80,7 +105,8 @@ def get_mode_for_numeric_fields_in_form(xform, field=None): def get_min_max_range_for_field(field, xform): - a = np.array(get_field_records(field, xform)) + """Returns min, max, range of values in the given field.""" + a = np.array(get_field_records(field, xform)) # pylint: disable=invalid-name _max = np.max(a) _min = np.min(a) _range = _max - _min @@ -88,14 +114,23 @@ def get_min_max_range_for_field(field, xform): def get_min_max_range(xform, field=None): + """Get's min, max, range of values in numeric fields. + + Returns a dict with the fields as key and the min, max, range as a value. + """ data = {} for field_name in [field] if field else get_numeric_fields(xform): _min, _max, _range = get_min_max_range_for_field(field_name, xform) - data[field_name] = {'max': _max, 'min': _min, 'range': _range} + data[field_name] = {"max": _max, "min": _min, "range": _range} return data def get_all_stats(xform, field=None): + """Get's mean, median, mode, min, max, range of values in numeric fields. + + Returns a dict with the fields as key and the mean, median, mode, min, max, + range as a value. + """ data = {} for field_name in [field] if field else get_numeric_fields(xform): _min, _max, _range = get_min_max_range_for_field(field_name, xform) @@ -103,11 +138,11 @@ def get_all_stats(xform, field=None): mean = get_mean_for_field(field_name, xform) median = get_median_for_field(field_name, xform) data[field_name] = { - 'mean': np.round(mean, DECIMAL_PRECISION), - 'median': median, - 'mode': np.round(mode, DECIMAL_PRECISION), - 'max': _max, - 'min': _min, - 'range': _range + "mean": np.round(mean, DECIMAL_PRECISION), + "median": median, + "mode": np.round(mode, DECIMAL_PRECISION), + "max": _max, + "min": _min, + "range": _range, } return data diff --git a/onadata/libs/filters.py b/onadata/libs/filters.py index 78ad7d7c1f..3704c28f8e 100644 --- a/onadata/libs/filters.py +++ b/onadata/libs/filters.py @@ -4,14 +4,14 @@ """ from uuid import UUID -import six - from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist from django.db.models import Q from django.http import Http404 from django.shortcuts import get_object_or_404 + +import six from django_filters import rest_framework as django_filter_filters from rest_framework import filters from rest_framework_guardian.filters import ObjectPermissionsFilter @@ -19,10 +19,9 @@ from onadata.apps.api.models import OrganizationProfile, Team from onadata.apps.logger.models import Instance, Project, XForm from onadata.apps.viewer.models import Export -from onadata.libs.utils.numeric import int_or_parse_error -from onadata.libs.utils.common_tags import MEDIA_FILE_TYPES from onadata.libs.permissions import exclude_items_from_queryset_using_xform_meta_perms - +from onadata.libs.utils.common_tags import MEDIA_FILE_TYPES +from onadata.libs.utils.numeric import int_or_parse_error # pylint: disable=invalid-name User = get_user_model() @@ -55,7 +54,7 @@ def filter_queryset(self, request, queryset, view): if form_id: if lookup_field == "pk": int_or_parse_error( - form_id, "Invalid form ID. It must be a positive" " integer" + form_id, "Invalid form ID. It must be a positive integer" ) try: @@ -104,7 +103,6 @@ class XFormListObjectPermissionFilter(AnonDjangoObjectPermissionFilter): class XFormListXFormPKFilter: """Filter forms via 'xform_pk' param.""" - # pylint: disable=no-self-use def filter_queryset(self, request, queryset, view): """Returns an XForm queryset filtered by the 1xform_pk' param.""" xform_pk = view.kwargs.get("xform_pk") @@ -393,7 +391,7 @@ def _instance_filter(self, request, view, keyword): for object_id in [instance_id, project_id]: int_or_parse_error( object_id, - "Invalid value for instanceid. It must be" " a positive integer.", + "Invalid value for instanceid. It must be a positive integer.", ) instance = get_object_or_404(Instance, pk=instance_id) @@ -506,7 +504,7 @@ def filter_queryset(self, request, queryset, view): if instance_id: int_or_parse_error( instance_id, - "Invalid value for instance_id. It must be" " a positive integer.", + "Invalid value for instance_id. It must be a positive integer.", ) instance = get_object_or_404(Instance, pk=instance_id) queryset = queryset.filter(instance=instance) @@ -615,8 +613,8 @@ class UserProfileFilter(filters.BaseFilterBackend): """Filter by the ``users`` query parameter.""" def filter_queryset(self, request, queryset, view): - """Filter by the ``users`` query parameter - returns a queryset of only the users - in the users parameter when `view.action == "list"`""" + """Filter by the ``users`` query parameter - returns a queryset of only the + users in the users parameter when `view.action == "list"`""" if view.action == "list": users = request.GET.get("users") if users: @@ -641,7 +639,7 @@ def filter_queryset(self, request, queryset, view): if instance_id: int_or_parse_error( instance_id, - "Invalid value for instance_id. It must be" " a positive integer", + "Invalid value for instance_id. It must be a positive integer", ) instance = get_object_or_404(Instance, pk=instance_id) @@ -692,7 +690,7 @@ def filter_queryset(self, request, queryset, view): class PublicDatasetsFilter: """Public data set filter where the share attribute is True""" - # pylint: disable=unused-argument,no-self-use + # pylint: disable=unused-argument def filter_queryset(self, request, queryset, view): """Return a queryset of shared=True data if the user is anonymous.""" if request and request.user.is_anonymous: diff --git a/onadata/libs/mixins/anonymous_user_mixin.py b/onadata/libs/mixins/anonymous_user_mixin.py index 7b8110ea21..a920c00086 100644 --- a/onadata/libs/mixins/anonymous_user_mixin.py +++ b/onadata/libs/mixins/anonymous_user_mixin.py @@ -1,14 +1,30 @@ +# -*- coding: utf-8 -*- +""" +Implements AnonymousUserMixin class + +Sets the DB AnonymousUser object to a request user to allow for object permission +checks. +""" from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.shortcuts import get_object_or_404 +User = get_user_model() + + +class AnonymousUserMixin: + """ + Implements AnonymousUserMixin class -class AnonymousUserMixin(object): + Sets the DB AnonymousUser object to a request user to allow for object permission + checks. + """ def get_queryset(self): """Set AnonymousUser from the database to allow object permissions.""" if self.request and self.request.user.is_anonymous: self.request.user = get_object_or_404( - User, username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME) + User, username__iexact=settings.ANONYMOUS_DEFAULT_USERNAME + ) - return super(AnonymousUserMixin, self).get_queryset() + return super().get_queryset() diff --git a/onadata/libs/mixins/anonymous_user_public_forms_mixin.py b/onadata/libs/mixins/anonymous_user_public_forms_mixin.py index f11ced98e8..6735ebdbcd 100644 --- a/onadata/libs/mixins/anonymous_user_public_forms_mixin.py +++ b/onadata/libs/mixins/anonymous_user_public_forms_mixin.py @@ -1,7 +1,18 @@ +# -*- coding: utf-8 -*- +""" +Implements the AnonymousUserPublicFormsMixin class + +Filters only public forms. +""" from onadata.apps.logger.models.xform import XForm -class AnonymousUserPublicFormsMixin(object): +class AnonymousUserPublicFormsMixin: + """ + Implements the AnonymousUserPublicFormsMixin class + + Filters only public forms. + """ def _get_public_forms_queryset(self): return XForm.objects.filter(shared=True) @@ -11,4 +22,4 @@ def get_queryset(self): if self.request and self.request.user.is_anonymous: return self._get_public_forms_queryset() - return super(AnonymousUserPublicFormsMixin, self).get_queryset() + return super().get_queryset() diff --git a/onadata/libs/mixins/authenticate_header_mixin.py b/onadata/libs/mixins/authenticate_header_mixin.py index 7b22596682..f332ca40f0 100644 --- a/onadata/libs/mixins/authenticate_header_mixin.py +++ b/onadata/libs/mixins/authenticate_header_mixin.py @@ -1,17 +1,31 @@ -from rest_framework.authentication import get_authorization_header -from rest_framework.authentication import TokenAuthentication +# -*- coding: utf-8 -*- +""" +Implements the AuthenticateHeaderMixin class + +Set's the appropriate authentication header using either the TempToken or Token. +""" +from rest_framework.authentication import TokenAuthentication, get_authorization_header + from onadata.libs.authentication import TempTokenAuthentication -class AuthenticateHeaderMixin(object): +class AuthenticateHeaderMixin: + """ + Implements the AuthenticateHeaderMixin class + + Set's the appropriate authentication header using either the TempToken or Token. + """ + def get_authenticate_header(self, request): + """ + Set's the appropriate authentication header using either the TempToken or Token. + """ auth = get_authorization_header(request).split() - if auth and auth[0].lower() == b'token': + if auth and auth[0].lower() == b"token": return TokenAuthentication().authenticate_header(request) - if auth and auth[0].lower() == b'temptoken': + if auth and auth[0].lower() == b"temptoken": return TempTokenAuthentication().authenticate_header(request) - return super(AuthenticateHeaderMixin, self)\ - .get_authenticate_header(request) + return super().get_authenticate_header(request) diff --git a/onadata/libs/mixins/bulk_create_mixin.py b/onadata/libs/mixins/bulk_create_mixin.py index 6ed43607ee..ff5b524319 100644 --- a/onadata/libs/mixins/bulk_create_mixin.py +++ b/onadata/libs/mixins/bulk_create_mixin.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals -class BulkCreateMixin(object): +class BulkCreateMixin: """ Bulk Create Mixin Allows the bulk creation of resources @@ -16,7 +16,7 @@ def get_serializer(self, *args, **kwargs): Gets the appropriate serializer depending on if you are creating a single resource or many resources """ - if isinstance(kwargs.get('data', {}), list): - kwargs['many'] = True + if isinstance(kwargs.get("data", {}), list): + kwargs["many"] = True - return super(BulkCreateMixin, self).get_serializer(*args, **kwargs) + return super().get_serializer(*args, **kwargs) diff --git a/onadata/libs/mixins/cache_control_mixin.py b/onadata/libs/mixins/cache_control_mixin.py index 55ab1c501b..236badf957 100644 --- a/onadata/libs/mixins/cache_control_mixin.py +++ b/onadata/libs/mixins/cache_control_mixin.py @@ -1,21 +1,38 @@ +# -*- coding: utf-8 -*- +""" +Implements the CacheControlMixin class + +Adds Cache headers to a viewsets response. +""" from django.conf import settings from django.utils.cache import patch_cache_control - CACHE_MIXIN_SECONDS = 60 -class CacheControlMixin(object): +class CacheControlMixin: + """ + Implements the CacheControlMixin class + + Adds Cache headers to a viewsets response. + """ + def set_cache_control(self, response, max_age=CACHE_MIXIN_SECONDS): - if hasattr(settings, 'CACHE_MIXIN_SECONDS'): + """Adds Cache headers to a response""" + if hasattr(settings, "CACHE_MIXIN_SECONDS"): max_age = settings.CACHE_MIXIN_SECONDS patch_cache_control(response, max_age=max_age) def finalize_response(self, request, response, *args, **kwargs): - if request.method == 'GET' and not response.streaming and \ - response.status_code in [200, 201, 202]: + """Overrides the finalize_response method + + Adds Cache headers to a response.""" + if ( + request.method == "GET" + and not response.streaming + and response.status_code in [200, 201, 202] + ): self.set_cache_control(response) - return super(CacheControlMixin, self).finalize_response( - request, response, *args, **kwargs) + return super().finalize_response(request, response, *args, **kwargs) diff --git a/onadata/libs/mixins/etags_mixin.py b/onadata/libs/mixins/etags_mixin.py index 4a7e47ba5f..144d81384b 100644 --- a/onadata/libs/mixins/etags_mixin.py +++ b/onadata/libs/mixins/etags_mixin.py @@ -1,12 +1,25 @@ -from builtins import str as text +# -*- coding: utf-8 -*- +""" +Implements the EtagsMixin class + +Adds Etag headers to the viewset response. +""" from hashlib import md5 -MODELS_WITH_DATE_MODIFIED = ('XForm', 'Instance', 'Project', 'Attachment', - 'MetaData', 'Note', 'OrganizationProfile', - 'UserProfile', 'Team') +MODELS_WITH_DATE_MODIFIED = ( + "XForm", + "Instance", + "Project", + "Attachment", + "MetaData", + "Note", + "OrganizationProfile", + "UserProfile", + "Team", +) -class ETagsMixin(object): +class ETagsMixin: """ Applies the Etag on GET responses with status code 200, 201, 202 @@ -15,26 +28,31 @@ class ETagsMixin(object): """ def set_etag_header(self, etag_value, etag_hash=None): + """Updates the response headers with Etag header""" if etag_value: - etag_hash = md5(text(etag_value).encode('utf-8')).hexdigest() + etag_hash = md5(str(etag_value).encode("utf-8")).hexdigest() if etag_hash: - self.headers.update({'ETag': etag_hash}) + self.headers.update({"ETag": etag_hash}) def finalize_response(self, request, response, *args, **kwargs): - if request.method == 'GET' and not response.streaming and \ - response.status_code in [200, 201, 202]: + """Overrides the finalize_response method + + Adds the Etag header to response.""" + if ( + request.method == "GET" + and not response.streaming + and response.status_code in [200, 201, 202] + ): etag_value = None - if hasattr(self, 'etag_data') and self.etag_data: + if hasattr(self, "etag_data") and self.etag_data: etag_value = str(self.etag_data) - elif hasattr(self, 'object'): + elif hasattr(self, "object"): if self.object.__class__.__name__ in MODELS_WITH_DATE_MODIFIED: etag_value = self.object.date_modified - if hasattr(self, 'etag_hash') and self.etag_hash: + if hasattr(self, "etag_hash") and self.etag_hash: self.set_etag_header(None, self.etag_hash) else: self.set_etag_header(etag_value) - return super(ETagsMixin, self).finalize_response( - request, response, *args, **kwargs - ) + return super().finalize_response(request, response, *args, **kwargs) diff --git a/onadata/libs/mixins/last_modified_mixin.py b/onadata/libs/mixins/last_modified_mixin.py index d028ebba76..ab53d12191 100644 --- a/onadata/libs/mixins/last_modified_mixin.py +++ b/onadata/libs/mixins/last_modified_mixin.py @@ -1,30 +1,41 @@ +# -*- coding: utf-8 -*- +""" +Implements the LastModifiedMixin class + +Adds the Last-Modified header to a viewset response. +""" import types -from onadata.libs.utils.timing import last_modified_header, get_date + +from onadata.libs.utils.timing import get_date, last_modified_header -class LastModifiedMixin(object): +class LastModifiedMixin: + """ + Implements the LastModifiedMixin class - last_modified_field = 'modified' + Adds the Last-Modified header to a viewset response. + """ + + last_modified_field = "modified" last_modified_date = None def finalize_response(self, request, response, *args, **kwargs): - if request.method == 'GET' and response.status_code < 300: + """Overrides the finalize_response method + + Adds the Last-Modified header to a viewset response.""" + if request.method == "GET" and response.status_code < 300: if self.last_modified_date is not None: - self.headers.update( - last_modified_header(self.last_modified_date)) + self.headers.update(last_modified_header(self.last_modified_date)) else: obj = None - if hasattr(self, 'object_list'): - generator_type = isinstance(self.object_list, - types.GeneratorType) - if isinstance(self.object_list, list) \ - and len(self.object_list): + if hasattr(self, "object_list"): + generator_type = isinstance(self.object_list, types.GeneratorType) + if isinstance(self.object_list, list) and len(self.object_list): obj = self.object_list[len(self.object_list) - 1] - elif not isinstance(self.object_list, list) and \ - not generator_type: + elif not isinstance(self.object_list, list) and not generator_type: obj = self.object_list.last() - if hasattr(self, 'object'): + if hasattr(self, "object"): obj = self.object if not obj: @@ -32,5 +43,4 @@ def finalize_response(self, request, response, *args, **kwargs): self.headers.update(last_modified_header(get_date(obj))) - return super(LastModifiedMixin, self).finalize_response( - request, response, *args, **kwargs) + return super().finalize_response(request, response, *args, **kwargs) diff --git a/onadata/libs/mixins/multi_lookup_mixin.py b/onadata/libs/mixins/multi_lookup_mixin.py index ca7f6a127e..2f623b20d6 100644 --- a/onadata/libs/mixins/multi_lookup_mixin.py +++ b/onadata/libs/mixins/multi_lookup_mixin.py @@ -1,15 +1,29 @@ +# -*- coding: utf-8 -*- +""" +Implements MultiLookupMixin class + +Looks up an object using multiple lookup fields. +""" from django.shortcuts import get_object_or_404 + from rest_framework import serializers from rest_framework.exceptions import ParseError -class MultiLookupMixin(object): +class MultiLookupMixin: + """ + Implements MultiLookupMixin class + + Looks up an object using multiple lookup fields. + """ + def get_object(self, queryset=None): + """Looks up an object using multiple lookup fields.""" if queryset is None: queryset = self.filter_queryset(self.get_queryset()) filter_kwargs = {} serializer = self.get_serializer() - lookup_fields = getattr(self, 'lookup_fields', []) + lookup_fields = getattr(self, "lookup_fields", []) for field in lookup_fields: lookup_field = field @@ -20,15 +34,13 @@ def get_object(self, queryset=None): if isinstance(k, serializers.HyperlinkedRelatedField): if k.source: lookup_field = k.source - lookup_field = '%s__%s' % (lookup_field, k.lookup_field) + lookup_field = f"{lookup_field}__{k.lookup_field}" if self.kwargs.get(field, None) is None: - raise ParseError( - 'Expected URL keyword argument `%s`.' % field - ) + raise ParseError(f"Expected URL keyword argument `{field}`.") filter_kwargs[lookup_field] = self.kwargs[field] - obj = get_object_or_404(queryset, **filter_kwargs) + obj = get_object_or_404(queryset, **filter_kwargs) # May raise a permission denied self.check_object_permissions(self.request, obj) diff --git a/onadata/libs/mixins/object_lookup_mixin.py b/onadata/libs/mixins/object_lookup_mixin.py index d0ae27d926..fac3584fac 100644 --- a/onadata/libs/mixins/object_lookup_mixin.py +++ b/onadata/libs/mixins/object_lookup_mixin.py @@ -1,18 +1,30 @@ +# -*- coding: utf-8 -*- +""" +Implements ObjectLookupMixin class + +Incase the lookup is on an object that has been hyperlinked +then update the queryset filter appropriately +""" from rest_framework import serializers from rest_framework.exceptions import ParseError from rest_framework.generics import get_object_or_404 -class ObjectLookupMixin(object): +class ObjectLookupMixin: + """ + Implements ObjectLookupMixin class + + Incase the lookup is on an object that has been hyperlinked + then update the queryset filter appropriately + """ + def get_object(self, queryset=None): """ Incase the lookup is on an object that has been hyperlinked then update the queryset filter appropriately """ if self.kwargs.get(self.lookup_field, None) is None: - raise ParseError( - 'Expected URL keyword argument `%s`.' % self.lookup_field - ) + raise ParseError(f"Expected URL keyword argument `{self.lookup_field}`.") if queryset is None: queryset = self.filter_queryset(self.get_queryset()) @@ -23,7 +35,7 @@ def get_object(self, queryset=None): if self.lookup_field in serializer.get_fields(): k = serializer.get_fields()[self.lookup_field] if isinstance(k, serializers.HyperlinkedRelatedField): - lookup_field = '%s__%s' % (self.lookup_field, k.lookup_field) + lookup_field = f"{self.lookup_field}__{k.lookup_field}" filter_kwargs[lookup_field] = self.kwargs[self.lookup_field] diff --git a/onadata/libs/mixins/openrosa_headers_mixin.py b/onadata/libs/mixins/openrosa_headers_mixin.py index 9cc5b10d62..fd61389a9f 100644 --- a/onadata/libs/mixins/openrosa_headers_mixin.py +++ b/onadata/libs/mixins/openrosa_headers_mixin.py @@ -1,14 +1,15 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ OpenRosaHeadersMixin module """ from datetime import datetime -import pytz from django.conf import settings +import pytz + # 10,000,000 bytes -DEFAULT_CONTENT_LENGTH = getattr(settings, 'DEFAULT_CONTENT_LENGTH', 10000000) +DEFAULT_CONTENT_LENGTH = getattr(settings, "DEFAULT_CONTENT_LENGTH", 10000000) def get_openrosa_headers(request, location=True): @@ -18,18 +19,18 @@ def get_openrosa_headers(request, location=True): """ now = datetime.now(pytz.timezone(settings.TIME_ZONE)) data = { - 'Date': now.strftime('%a, %d %b %Y %H:%M:%S %Z'), - 'X-OpenRosa-Version': '1.0', - 'X-OpenRosa-Accept-Content-Length': DEFAULT_CONTENT_LENGTH + "Date": now.strftime("%a, %d %b %Y %H:%M:%S %Z"), + "X-OpenRosa-Version": "1.0", + "X-OpenRosa-Accept-Content-Length": DEFAULT_CONTENT_LENGTH, } if location: - data['Location'] = request.build_absolute_uri(request.path) + data["Location"] = request.build_absolute_uri(request.path) return data -class OpenRosaHeadersMixin(object): # pylint: disable=R0903 +class OpenRosaHeadersMixin: """ OpenRosaHeadersMixin class - sets OpenRosa headers in a response for a View or Viewset. @@ -41,5 +42,4 @@ def finalize_response(self, request, response, *args, **kwargs): """ self.headers.update(get_openrosa_headers(request)) - return super(OpenRosaHeadersMixin, self).finalize_response( - request, response, *args, **kwargs) + return super().finalize_response(request, response, *args, **kwargs) diff --git a/onadata/libs/mixins/profiler_mixin.py b/onadata/libs/mixins/profiler_mixin.py index e4b6e08e46..991f7598f0 100644 --- a/onadata/libs/mixins/profiler_mixin.py +++ b/onadata/libs/mixins/profiler_mixin.py @@ -1,64 +1,80 @@ +# -*- coding: utf-8 -*- +""" +Implements a ProfilerMixin - profiles a Django Rest Framework viewset. +""" import logging import time + from django.conf import settings -from django.core.signals import request_started, request_finished +from django.core.signals import request_finished, request_started from django.http import StreamingHttpResponse from rest_framework.fields import empty +project_viewset_profiler = logging.getLogger("profiler_logger") -project_viewset_profiler = logging.getLogger('profiler_logger') +DISPATCH_TIME = 0 +RENDER_TIME = 0 +STARTED = 0 +SERIALIZER_TIME = 0 -class ProfilerMixin(object): +class ProfilerMixin: + """ + Implements a ProfilerMixin - profiles a Django Rest Framework viewset. + """ def get_serializer(self, instance=None, data=empty, **kwargs): + """Override the get_serializer() method.""" serializer_class = self.get_serializer_class() - kwargs['context'] = self.get_serializer_context() + kwargs["context"] = self.get_serializer_context() if settings.PROFILE_API_ACTION_FUNCTION: - global serializer_time + global SERIALIZER_TIME # pylint: disable=global-statement serializer_start = time.time() serializer = serializer_class(instance, data=data, **kwargs) - serializer_time = time.time() - serializer_start + SERIALIZER_TIME = time.time() - serializer_start return serializer return serializer_class(instance, data=data, **kwargs) def dispatch(self, request, *args, **kwargs): - global render_time - global dispatch_time + """Override the viewset dispatch method.""" + global RENDER_TIME # pylint: disable=global-statement + global DISPATCH_TIME # pylint: disable=global-statement dispatch_start = time.time() - ret = super(ProfilerMixin, self).dispatch(request, *args, **kwargs) + ret = super().dispatch(request, *args, **kwargs) if not isinstance(ret, StreamingHttpResponse): render_start = time.time() ret.render() - render_time = time.time() - render_start + RENDER_TIME = time.time() - render_start else: - render_time = 0 - dispatch_time = time.time() - dispatch_start + RENDER_TIME = 0 + DISPATCH_TIME = time.time() - dispatch_start return ret -def started(sender, **kwargs): - global started - started = time.time() +def started(sender, **kwargs): # pylint: disable=unused-argument + """Signal that starts the timer""" + global STARTED # pylint: disable=global-statement + STARTED = time.time() -def finished(sender, **kwargs): +def finished(sender, **kwargs): # pylint: disable=unused-argument + """Signal that captures the end of the timer""" try: - total = time.time() - started + total = time.time() - STARTED api_view_time = dispatch_time - (render_time + serializer_time) request_response_time = total - dispatch_time + # pylint: disable=consider-using-f-string output = "\n" output += "Serialization | %.4fs\n" % serializer_time - output += "Django request/response | %.4fs\n" %\ - request_response_time + output += "Django request/response | %.4fs\n" % request_response_time output += "API view | %.4fs\n" % api_view_time output += "Response rendering | %.4fs\n" % render_time diff --git a/onadata/libs/mixins/xform_id_string_lookup.py b/onadata/libs/mixins/xform_id_string_lookup.py index c41dc2fc00..a42689257a 100644 --- a/onadata/libs/mixins/xform_id_string_lookup.py +++ b/onadata/libs/mixins/xform_id_string_lookup.py @@ -1,11 +1,27 @@ +# -*- coding: utf-8 -*- +""" +XForm id_strng lookup mixin class + +Looks up an XForm using the id_string. +""" from django.core.exceptions import ImproperlyConfigured from django.shortcuts import get_object_or_404 -class XFormIdStringLookupMixin(object): - lookup_id_string = 'id_string' +class XFormIdStringLookupMixin: + """ + XForm id_strng lookup mixin class + + Looks up an XForm using the id_string. + """ + + lookup_id_string = "id_string" def get_object(self, queryset=None): + """Looks up an XForm object using the ``id_string`` + + Returns the XForm object or raises a 404 HTTP response exception + """ if queryset is None: queryset = self.filter_queryset(self.get_queryset()) @@ -19,10 +35,10 @@ def get_object(self, queryset=None): lookup_field = self.lookup_id_string else: raise ImproperlyConfigured( - 'Expected view %s to be called with a URL keyword argument ' - 'named "%s". Fix your URL conf, or set the `.lookup_field` ' - 'attribute on the view correctly.' % - (self.__class__.__name__, self.lookup_field) + f"Expected view {self.__class__.__name__} to be called with a " + f'URL keyword argument named "{self.lookup_field}". ' + "Fix your URL conf, or set the `.lookup_field` " + "attribute on the view correctly." ) filter_kwargs = {lookup_field: lookup} diff --git a/onadata/libs/models/signals.py b/onadata/libs/models/signals.py index 5c395078a2..e8711a13cc 100644 --- a/onadata/libs/models/signals.py +++ b/onadata/libs/models/signals.py @@ -4,9 +4,9 @@ from onadata.apps.logger.models import XForm -# pylint: disable=unexpected-keyword-arg -xform_tags_add = django.dispatch.Signal(providing_args=["xform", "tags"]) -xform_tags_delete = django.dispatch.Signal(providing_args=["xform", "tag"]) +# pylint: disable=invalid-name +xform_tags_add = django.dispatch.Signal() +xform_tags_delete = django.dispatch.Signal() # pylint: disable=unused-argument diff --git a/onadata/libs/models/sorting.py b/onadata/libs/models/sorting.py index 7e0935f7f7..708db7b209 100644 --- a/onadata/libs/models/sorting.py +++ b/onadata/libs/models/sorting.py @@ -1,12 +1,18 @@ +# -*- coding: utf-8 -*- +""" +Sorting utility functions. +""" import json -import six from typing import Dict +import six + def sort_from_mongo_sort_str(sort_str): + """Create a sort query list based on MongoDB sort string input.""" sort_values = [] if isinstance(sort_str, six.string_types): - if sort_str.startswith('{'): + if sort_str.startswith("{"): sort_dict = json.loads(sort_str) for k, v in sort_dict.items(): try: @@ -14,7 +20,7 @@ def sort_from_mongo_sort_str(sort_str): except ValueError: pass if v < 0: - k = u'-{}'.format(k) + k = f"-{k}" sort_values.append(k) else: sort_values.append(sort_str) @@ -22,34 +28,41 @@ def sort_from_mongo_sort_str(sort_str): return sort_values -def json_order_by( - sort_list, none_json_fields: Dict = {}, model_name: str = ""): +def json_order_by(sort_list, none_json_fields: Dict = None, model_name: str = ""): + """Returns SQL ORDER BY string portion based on JSON input.""" _list = [] + if none_json_fields is None: + none_json_fields = {} for field in sort_list: - field_key = field.lstrip('-') - _str = u" json->>%s"\ - if field_key not in none_json_fields.keys() else\ - f'"{model_name}"."{none_json_fields.get(field_key)}"' + field_key = field.lstrip("-") + _str = ( + " json->>%s" + if field_key not in none_json_fields + else f'"{model_name}"."{none_json_fields.get(field_key)}"' + ) - if field.startswith('-'): - _str += u" DESC" + if field.startswith("-"): + _str += " DESC" else: - _str += u" ASC" + _str += " ASC" _list.append(_str) if len(_list) > 0: - return u"ORDER BY {}".format(u",".join(_list)) + return f'ORDER BY {",".join(_list)}' - return u"" + return "" -def json_order_by_params(sort_list, none_json_fields: Dict = {}): +def json_order_by_params(sort_list, none_json_fields: Dict = None): + """Creates the ORDER BY parameters list from JSON input.""" params = [] + if none_json_fields is None: + none_json_fields = {} for field in sort_list: - field = field.lstrip('-') - if field not in none_json_fields.keys(): - params.append(field.lstrip('-')) + field = field.lstrip("-") + if field not in none_json_fields: + params.append(field.lstrip("-")) return params diff --git a/onadata/libs/pagination.py b/onadata/libs/pagination.py index 06888f4e62..10b6317409 100644 --- a/onadata/libs/pagination.py +++ b/onadata/libs/pagination.py @@ -1,38 +1,58 @@ -from django.core.paginator import Paginator +# -*- coding: utf-8 -*- +""" +Pagination classes. +""" from django.conf import settings +from django.core.paginator import Paginator from django.db.models import QuerySet +from django.utils.functional import cached_property + from rest_framework.pagination import ( - PageNumberPagination, InvalidPage, NotFound, replace_query_param) + InvalidPage, + NotFound, + PageNumberPagination, + replace_query_param, +) from rest_framework.request import Request class StandardPageNumberPagination(PageNumberPagination): + """The Standard PageNumberPagination class + + Set's the default ``page_size`` to 1000 with a maximum page_size of 10,000 records + per page. + """ + page_size = 1000 - page_size_query_param = 'page_size' - max_page_size = getattr( - settings, "STANDARD_PAGINATION_MAX_PAGE_SIZE", 10000) + page_size_query_param = "page_size" + max_page_size = getattr(settings, "STANDARD_PAGINATION_MAX_PAGE_SIZE", 10000) def get_first_page_link(self): + """Returns the URL to the first page.""" if self.page.number == 1: return None + url = self.request.build_absolute_uri() + return replace_query_param(url, self.page_query_param, 1) def get_last_page_link(self): + """Returns the URL to the last page.""" if self.page.number == self.paginator.num_pages: return None + url = self.request.build_absolute_uri() - return replace_query_param( - url, self.page_query_param, self.paginator.num_pages) - def generate_link_header( - self, request: Request, queryset: QuerySet - ): + return replace_query_param(url, self.page_query_param, self.paginator.num_pages) + + def generate_link_header(self, request: Request, queryset: QuerySet): + """Generates pagination headers for a HTTP response object""" links = [] page_size = self.get_page_size(request) if not page_size: return {} page_number = request.query_params.get(self.page_query_param, 1) + # pylint: disable=attribute-defined-outside-init self.paginator = self.django_paginator_class(queryset, page_size) self.request = request @@ -42,27 +62,41 @@ def generate_link_header( return {} for rel, link in ( - ('prev', self.get_previous_link()), - ('next', self.get_next_link()), - ('last', self.get_last_page_link()), - ('first', self.get_first_page_link())): + ("prev", self.get_previous_link()), + ("next", self.get_next_link()), + ("last", self.get_last_page_link()), + ("first", self.get_first_page_link()), + ): if link: links.append(f'<{link}>; rel="{rel}"') - return {'Link': ', '.join(links)} + return {"Link": ", ".join(links)} class CountOverridablePaginator(Paginator): + """Count override Paginator + + Allows overriding the count especially in the event it may be expensive request. + """ + + # pylint: disable=too-many-arguments def __init__( - self, object_list, per_page, - orphans: int = 0, allow_empty_first_page: bool = True, - count_override: int = None) -> None: + self, + object_list, + per_page, + orphans: int = 0, + allow_empty_first_page: bool = True, + count_override: int = None, + ) -> None: self.count_override = count_override super().__init__( - object_list, per_page, - orphans=orphans, allow_empty_first_page=allow_empty_first_page) + object_list, + per_page, + orphans=orphans, + allow_empty_first_page=allow_empty_first_page, + ) - @property + @cached_property def count(self): if self.count_override: return self.count_override @@ -70,17 +104,21 @@ def count(self): class CountOverridablePageNumberPagination(StandardPageNumberPagination): + """Count override PageNumberPagination + + Allows overriding the count especially in the event it may be expensive request. + """ + django_paginator_class = CountOverridablePaginator def paginate_queryset(self, queryset, request, view, count=None): + # pylint: disable=attribute-defined-outside-init page_size = self.get_page_size(request) if not page_size: return None paginator = self.django_paginator_class( - queryset, - page_size, - count_override=count + queryset, page_size, count_override=count ) page_number = request.query_params.get(self.page_query_param, 1) if page_number in self.last_page_strings: @@ -89,9 +127,10 @@ def paginate_queryset(self, queryset, request, view, count=None): try: self.page = paginator.page(page_number) except InvalidPage as exc: - msg = self.invalid_page_message.format(page_number=page_number, - message=str(exc)) - raise NotFound(msg) + msg = self.invalid_page_message.format( + page_number=page_number, message=str(exc) + ) + raise NotFound(msg) from exc if paginator.num_pages > 1 and self.template is not None: self.display_page_controls = True diff --git a/onadata/libs/permissions.py b/onadata/libs/permissions.py index 91b8b8ced3..c3715f3420 100644 --- a/onadata/libs/permissions.py +++ b/onadata/libs/permissions.py @@ -1,84 +1,93 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Permissions module. """ import json from collections import defaultdict -import six -from django.db.models.base import ModelBase +from django.apps import apps from django.db.models import Q -from guardian.shortcuts import (assign_perm, get_perms, get_users_with_perms, - remove_perm) - -from onadata.apps.api.models import OrganizationProfile -from onadata.apps.logger.models import MergedXForm, Project, XForm,\ - Attachment -from onadata.apps.logger.models.project import (ProjectGroupObjectPermission, - ProjectUserObjectPermission) -from onadata.apps.logger.models.xform import (XFormGroupObjectPermission, - XFormUserObjectPermission) -from onadata.apps.main.models.user_profile import UserProfile -from onadata.apps.viewer.models import DataDictionary +from django.db.models.base import ModelBase + +import six +from guardian.shortcuts import assign_perm, get_perms, get_users_with_perms, remove_perm + +from onadata.apps.logger.models.attachment import Attachment +from onadata.apps.logger.models.project import ( + Project, + ProjectGroupObjectPermission, + ProjectUserObjectPermission, +) +from onadata.apps.logger.models.xform import ( + XForm, + XFormGroupObjectPermission, + XFormUserObjectPermission, +) from onadata.libs.exceptions import NoRecordsPermission from onadata.libs.utils.common_tags import XFORM_META_PERMS from onadata.libs.utils.model_tools import queryset_iterator # Userprofile Permissions -CAN_ADD_USERPROFILE = 'add_userprofile' -CAN_CHANGE_USERPROFILE = 'change_userprofile' -CAN_DELETE_USERPROFILE = 'delete_userprofile' -CAN_ADD_PROJECT_TO_PROFILE = 'can_add_project' -CAN_ADD_XFORM_TO_PROFILE = 'can_add_xform' -CAN_VIEW_PROFILE = 'view_profile' +CAN_ADD_USERPROFILE = "add_userprofile" +CAN_CHANGE_USERPROFILE = "change_userprofile" +CAN_DELETE_USERPROFILE = "delete_userprofile" +CAN_ADD_PROJECT_TO_PROFILE = "can_add_project" +CAN_ADD_XFORM_TO_PROFILE = "can_add_xform" +CAN_VIEW_PROFILE = "view_profile" # Organization Permissions -CAN_VIEW_ORGANIZATION_PROFILE = 'view_organizationprofile' -CAN_ADD_ORGANIZATION_PROFILE = 'add_organizationprofile' -CAN_ADD_ORGANIZATION_PROJECT = 'can_add_project' -CAN_ADD_ORGANIZATION_XFORM = 'can_add_xform' -CAN_CHANGE_ORGANIZATION_PROFILE = 'change_organizationprofile' -CAN_DELETE_ORGANIZATION_PROFILE = 'delete_organizationprofile' -IS_ORGANIZATION_OWNER = 'is_org_owner' - -# Xform Permissions -CAN_CHANGE_XFORM = 'change_xform' -CAN_ADD_XFORM = 'add_xform' -CAN_DELETE_XFORM = 'delete_xform' -CAN_VIEW_XFORM = 'view_xform' -CAN_VIEW_XFORM_DATA = 'view_xform_data' -CAN_VIEW_XFORM_ALL = 'view_xform_all' -CAN_ADD_SUBMISSIONS = 'report_xform' -CAN_DELETE_SUBMISSION = 'delete_submission' -CAN_TRANSFER_OWNERSHIP = 'transfer_xform' -CAN_MOVE_TO_FOLDER = 'move_xform' -CAN_EXPORT_XFORM = 'can_export_xform_data' +CAN_VIEW_ORGANIZATION_PROFILE = "view_organizationprofile" +CAN_ADD_ORGANIZATION_PROFILE = "add_organizationprofile" +CAN_ADD_ORGANIZATION_PROJECT = "can_add_project" +CAN_ADD_ORGANIZATION_XFORM = "can_add_xform" +CAN_CHANGE_ORGANIZATION_PROFILE = "change_organizationprofile" +CAN_DELETE_ORGANIZATION_PROFILE = "delete_organizationprofile" +IS_ORGANIZATION_OWNER = "is_org_owner" + +# XForm Permissions +CAN_CHANGE_XFORM = "change_xform" +CAN_ADD_XFORM = "add_xform" +CAN_DELETE_XFORM = "delete_xform" +CAN_VIEW_XFORM = "view_xform" +CAN_VIEW_XFORM_DATA = "view_xform_data" +CAN_VIEW_XFORM_ALL = "view_xform_all" +CAN_ADD_SUBMISSIONS = "report_xform" +CAN_DELETE_SUBMISSION = "delete_submission" +CAN_TRANSFER_OWNERSHIP = "transfer_xform" +CAN_MOVE_TO_FOLDER = "move_xform" +CAN_EXPORT_XFORM = "can_export_xform_data" # MergedXform Permissions -CAN_VIEW_MERGED_XFORM = 'view_mergedxform' +CAN_VIEW_MERGED_XFORM = "view_mergedxform" # Project Permissions -CAN_ADD_PROJECT = 'add_project' -CAN_VIEW_PROJECT = 'view_project' -CAN_VIEW_PROJECT_DATA = 'view_project_data' -CAN_VIEW_PROJECT_ALL = 'view_project_all' -CAN_CHANGE_PROJECT = 'change_project' -CAN_TRANSFER_PROJECT_OWNERSHIP = 'transfer_project' -CAN_DELETE_PROJECT = 'delete_project' -CAN_ADD_PROJECT_XFORM = 'add_project_xform' -CAN_ADD_SUBMISSIONS_PROJECT = 'report_project_xform' -CAN_EXPORT_PROJECT = 'can_export_project_data' +CAN_ADD_PROJECT = "add_project" +CAN_VIEW_PROJECT = "view_project" +CAN_VIEW_PROJECT_DATA = "view_project_data" +CAN_VIEW_PROJECT_ALL = "view_project_all" +CAN_CHANGE_PROJECT = "change_project" +CAN_TRANSFER_PROJECT_OWNERSHIP = "transfer_project" +CAN_DELETE_PROJECT = "delete_project" +CAN_ADD_PROJECT_XFORM = "add_project_xform" +CAN_ADD_SUBMISSIONS_PROJECT = "report_project_xform" +CAN_EXPORT_PROJECT = "can_export_project_data" # Data dictionary permissions -CAN_ADD_DATADICTIONARY = 'add_datadictionary' -CAN_CHANGE_DATADICTIONARY = 'change_datadictionary' -CAN_DELETE_DATADICTIONARY = 'delete_datadictionary' +CAN_ADD_DATADICTIONARY = "add_datadictionary" +CAN_CHANGE_DATADICTIONARY = "change_datadictionary" +CAN_DELETE_DATADICTIONARY = "delete_datadictionary" + +DataDictionary = apps.get_model("viewer", "DataDictionary") +MergedXForm = apps.get_model("logger", "MergedXForm") +OrganizationProfile = apps.get_model("api", "OrganizationProfile") +UserProfile = apps.get_model("main", "UserProfile") -class Role(object): +class Role: """ Base Role class. """ + class_to_permissions = defaultdict(list) name = None @@ -135,12 +144,16 @@ class ReadOnlyRoleNoDownload(Role): """ Read-only no download Role class. """ - name = 'readonly-no-download' - permissions = ((CAN_VIEW_ORGANIZATION_PROFILE, - OrganizationProfile), (CAN_VIEW_XFORM, XForm), - (CAN_VIEW_PROJECT, Project), (CAN_VIEW_XFORM_ALL, XForm), - (CAN_VIEW_PROJECT_ALL, Project), (CAN_VIEW_MERGED_XFORM, - MergedXForm), ) + + name = "readonly-no-download" + permissions = ( + (CAN_VIEW_ORGANIZATION_PROFILE, OrganizationProfile), + (CAN_VIEW_XFORM, XForm), + (CAN_VIEW_PROJECT, Project), + (CAN_VIEW_XFORM_ALL, XForm), + (CAN_VIEW_PROJECT_ALL, Project), + (CAN_VIEW_MERGED_XFORM, MergedXForm), + ) class_to_permissions = { MergedXForm: [CAN_VIEW_MERGED_XFORM], @@ -153,7 +166,8 @@ class ReadOnlyRole(Role): """ Read-only Role class. """ - name = 'readonly' + + name = "readonly" class_to_permissions = { MergedXForm: [CAN_VIEW_MERGED_XFORM], @@ -167,13 +181,13 @@ class DataEntryOnlyRole(Role): """ Data-Entry only Role class. """ - name = 'dataentry-only' + + name = "dataentry-only" class_to_permissions = { MergedXForm: [CAN_VIEW_MERGED_XFORM], OrganizationProfile: [CAN_VIEW_ORGANIZATION_PROFILE], - Project: - [CAN_ADD_SUBMISSIONS_PROJECT, CAN_EXPORT_PROJECT, CAN_VIEW_PROJECT], + Project: [CAN_ADD_SUBMISSIONS_PROJECT, CAN_EXPORT_PROJECT, CAN_VIEW_PROJECT], XForm: [CAN_ADD_SUBMISSIONS], } @@ -183,17 +197,22 @@ class DataEntryMinorRole(Role): Data-Entry minor Role class - user can submit and has readonly access to data they submitted. """ - name = 'dataentry-minor' + + name = "dataentry-minor" class_to_permissions = { MergedXForm: [CAN_VIEW_MERGED_XFORM], OrganizationProfile: [CAN_VIEW_ORGANIZATION_PROFILE], Project: [ - CAN_ADD_SUBMISSIONS_PROJECT, CAN_EXPORT_PROJECT, CAN_VIEW_PROJECT, - CAN_VIEW_PROJECT_DATA + CAN_ADD_SUBMISSIONS_PROJECT, + CAN_EXPORT_PROJECT, + CAN_VIEW_PROJECT, + CAN_VIEW_PROJECT_DATA, ], XForm: [ - CAN_ADD_SUBMISSIONS, CAN_EXPORT_XFORM, CAN_VIEW_XFORM, - CAN_VIEW_XFORM_DATA + CAN_ADD_SUBMISSIONS, + CAN_EXPORT_XFORM, + CAN_VIEW_XFORM, + CAN_VIEW_XFORM_DATA, ], } @@ -203,17 +222,24 @@ class DataEntryRole(Role): Data-Entry Role class - user can submit data and has readonly permissions to all the data including data submitted by others. """ - name = 'dataentry' + + name = "dataentry" class_to_permissions = { MergedXForm: [CAN_VIEW_MERGED_XFORM], OrganizationProfile: [CAN_VIEW_ORGANIZATION_PROFILE], Project: [ - CAN_ADD_SUBMISSIONS_PROJECT, CAN_EXPORT_PROJECT, CAN_VIEW_PROJECT, - CAN_VIEW_PROJECT_ALL, CAN_VIEW_PROJECT_DATA + CAN_ADD_SUBMISSIONS_PROJECT, + CAN_EXPORT_PROJECT, + CAN_VIEW_PROJECT, + CAN_VIEW_PROJECT_ALL, + CAN_VIEW_PROJECT_DATA, ], XForm: [ - CAN_ADD_SUBMISSIONS, CAN_EXPORT_XFORM, CAN_VIEW_XFORM, - CAN_VIEW_XFORM_ALL, CAN_VIEW_XFORM_DATA + CAN_ADD_SUBMISSIONS, + CAN_EXPORT_XFORM, + CAN_VIEW_XFORM, + CAN_VIEW_XFORM_ALL, + CAN_VIEW_XFORM_DATA, ], } @@ -223,17 +249,25 @@ class EditorMinorRole(Role): Editor-Minor Role class - user can submit data, read and edit only the data they submitted. """ - name = 'editor-minor' + + name = "editor-minor" class_to_permissions = { MergedXForm: [CAN_VIEW_MERGED_XFORM], OrganizationProfile: [CAN_VIEW_ORGANIZATION_PROFILE], Project: [ - CAN_ADD_SUBMISSIONS_PROJECT, CAN_CHANGE_PROJECT, - CAN_EXPORT_PROJECT, CAN_VIEW_PROJECT, CAN_VIEW_PROJECT_DATA + CAN_ADD_SUBMISSIONS_PROJECT, + CAN_CHANGE_PROJECT, + CAN_EXPORT_PROJECT, + CAN_VIEW_PROJECT, + CAN_VIEW_PROJECT_DATA, ], XForm: [ - CAN_ADD_SUBMISSIONS, CAN_CHANGE_XFORM, CAN_DELETE_SUBMISSION, - CAN_EXPORT_XFORM, CAN_VIEW_XFORM, CAN_VIEW_XFORM_DATA + CAN_ADD_SUBMISSIONS, + CAN_CHANGE_XFORM, + CAN_DELETE_SUBMISSION, + CAN_EXPORT_XFORM, + CAN_VIEW_XFORM, + CAN_VIEW_XFORM_DATA, ], } @@ -242,19 +276,27 @@ class EditorRole(Role): """ Editor Role class - user can submit, read and edit any submitted data. """ - name = 'editor' + + name = "editor" class_to_permissions = { MergedXForm: [CAN_VIEW_MERGED_XFORM], OrganizationProfile: [CAN_VIEW_ORGANIZATION_PROFILE], Project: [ - CAN_ADD_SUBMISSIONS_PROJECT, CAN_CHANGE_PROJECT, - CAN_EXPORT_PROJECT, CAN_VIEW_PROJECT, CAN_VIEW_PROJECT_ALL, - CAN_VIEW_PROJECT_DATA + CAN_ADD_SUBMISSIONS_PROJECT, + CAN_CHANGE_PROJECT, + CAN_EXPORT_PROJECT, + CAN_VIEW_PROJECT, + CAN_VIEW_PROJECT_ALL, + CAN_VIEW_PROJECT_DATA, ], XForm: [ - CAN_ADD_SUBMISSIONS, CAN_CHANGE_XFORM, CAN_DELETE_SUBMISSION, - CAN_EXPORT_XFORM, CAN_VIEW_XFORM, CAN_VIEW_XFORM_ALL, - CAN_VIEW_XFORM_DATA + CAN_ADD_SUBMISSIONS, + CAN_CHANGE_XFORM, + CAN_DELETE_SUBMISSION, + CAN_EXPORT_XFORM, + CAN_VIEW_XFORM, + CAN_VIEW_XFORM_ALL, + CAN_VIEW_XFORM_DATA, ], } @@ -264,24 +306,40 @@ class ManagerRole(Role): Manager Role class - user can add,delete,edit forms and data as well as control access to data, forms and projects. """ - name = 'manager' + + name = "manager" class_to_permissions = { MergedXForm: [CAN_VIEW_MERGED_XFORM], - OrganizationProfile: - [CAN_ADD_ORGANIZATION_PROJECT, CAN_ADD_ORGANIZATION_XFORM, - CAN_VIEW_ORGANIZATION_PROFILE], + OrganizationProfile: [ + CAN_ADD_ORGANIZATION_PROJECT, + CAN_ADD_ORGANIZATION_XFORM, + CAN_VIEW_ORGANIZATION_PROFILE, + ], Project: [ - CAN_ADD_PROJECT, CAN_ADD_PROJECT_XFORM, - CAN_ADD_SUBMISSIONS_PROJECT, CAN_CHANGE_PROJECT, - CAN_EXPORT_PROJECT, CAN_VIEW_PROJECT, CAN_VIEW_PROJECT_ALL, - CAN_VIEW_PROJECT_DATA + CAN_ADD_PROJECT, + CAN_ADD_PROJECT_XFORM, + CAN_ADD_SUBMISSIONS_PROJECT, + CAN_CHANGE_PROJECT, + CAN_EXPORT_PROJECT, + CAN_VIEW_PROJECT, + CAN_VIEW_PROJECT_ALL, + CAN_VIEW_PROJECT_DATA, + ], + UserProfile: [ + CAN_ADD_PROJECT_TO_PROFILE, + CAN_ADD_XFORM_TO_PROFILE, + CAN_VIEW_PROFILE, ], - UserProfile: [CAN_ADD_PROJECT_TO_PROFILE, CAN_ADD_XFORM_TO_PROFILE, - CAN_VIEW_PROFILE], XForm: [ - CAN_ADD_SUBMISSIONS, CAN_ADD_XFORM, CAN_CHANGE_XFORM, - CAN_DELETE_SUBMISSION, CAN_DELETE_XFORM, CAN_EXPORT_XFORM, - CAN_VIEW_XFORM, CAN_VIEW_XFORM_ALL, CAN_VIEW_XFORM_DATA + CAN_ADD_SUBMISSIONS, + CAN_ADD_XFORM, + CAN_CHANGE_XFORM, + CAN_DELETE_SUBMISSION, + CAN_DELETE_XFORM, + CAN_EXPORT_XFORM, + CAN_VIEW_XFORM, + CAN_VIEW_XFORM_ALL, + CAN_VIEW_XFORM_DATA, ], } @@ -290,52 +348,80 @@ class MemberRole(Role): """ This is a role for a member of an organization. """ - name = 'member' + + name = "member" class OwnerRole(Role): """ This is a role for an owner of a dataset, organization, or project. """ - name = 'owner' + + name = "owner" class_to_permissions = { DataDictionary: [ - CAN_ADD_DATADICTIONARY, CAN_CHANGE_DATADICTIONARY, - CAN_DELETE_DATADICTIONARY + CAN_ADD_DATADICTIONARY, + CAN_CHANGE_DATADICTIONARY, + CAN_DELETE_DATADICTIONARY, ], MergedXForm: [CAN_VIEW_MERGED_XFORM], OrganizationProfile: [ - CAN_ADD_ORGANIZATION_PROJECT, CAN_ADD_ORGANIZATION_XFORM, - CAN_ADD_ORGANIZATION_PROFILE, CAN_ADD_ORGANIZATION_PROJECT, - CAN_ADD_ORGANIZATION_XFORM, CAN_CHANGE_ORGANIZATION_PROFILE, - CAN_DELETE_ORGANIZATION_PROFILE, CAN_VIEW_ORGANIZATION_PROFILE, - IS_ORGANIZATION_OWNER + CAN_ADD_ORGANIZATION_PROJECT, + CAN_ADD_ORGANIZATION_XFORM, + CAN_ADD_ORGANIZATION_PROFILE, + CAN_ADD_ORGANIZATION_PROJECT, + CAN_ADD_ORGANIZATION_XFORM, + CAN_CHANGE_ORGANIZATION_PROFILE, + CAN_DELETE_ORGANIZATION_PROFILE, + CAN_VIEW_ORGANIZATION_PROFILE, + IS_ORGANIZATION_OWNER, ], Project: [ - CAN_ADD_PROJECT, CAN_ADD_PROJECT_XFORM, - CAN_ADD_SUBMISSIONS_PROJECT, CAN_CHANGE_PROJECT, - CAN_DELETE_PROJECT, CAN_EXPORT_PROJECT, - CAN_TRANSFER_PROJECT_OWNERSHIP, CAN_VIEW_PROJECT, - CAN_VIEW_PROJECT_ALL, CAN_VIEW_PROJECT_DATA + CAN_ADD_PROJECT, + CAN_ADD_PROJECT_XFORM, + CAN_ADD_SUBMISSIONS_PROJECT, + CAN_CHANGE_PROJECT, + CAN_DELETE_PROJECT, + CAN_EXPORT_PROJECT, + CAN_TRANSFER_PROJECT_OWNERSHIP, + CAN_VIEW_PROJECT, + CAN_VIEW_PROJECT_ALL, + CAN_VIEW_PROJECT_DATA, ], UserProfile: [ - CAN_ADD_PROJECT_TO_PROFILE, CAN_ADD_XFORM_TO_PROFILE, - CAN_ADD_USERPROFILE, CAN_CHANGE_USERPROFILE, - CAN_DELETE_USERPROFILE, CAN_VIEW_PROFILE + CAN_ADD_PROJECT_TO_PROFILE, + CAN_ADD_XFORM_TO_PROFILE, + CAN_ADD_USERPROFILE, + CAN_CHANGE_USERPROFILE, + CAN_DELETE_USERPROFILE, + CAN_VIEW_PROFILE, ], XForm: [ - CAN_ADD_SUBMISSIONS, CAN_ADD_XFORM, CAN_CHANGE_XFORM, - CAN_DELETE_SUBMISSION, CAN_DELETE_XFORM, CAN_EXPORT_XFORM, - CAN_VIEW_XFORM, CAN_VIEW_XFORM_ALL, CAN_VIEW_XFORM_DATA, - CAN_MOVE_TO_FOLDER, CAN_TRANSFER_OWNERSHIP + CAN_ADD_SUBMISSIONS, + CAN_ADD_XFORM, + CAN_CHANGE_XFORM, + CAN_DELETE_SUBMISSION, + CAN_DELETE_XFORM, + CAN_EXPORT_XFORM, + CAN_VIEW_XFORM, + CAN_VIEW_XFORM_ALL, + CAN_VIEW_XFORM_DATA, + CAN_MOVE_TO_FOLDER, + CAN_TRANSFER_OWNERSHIP, ], } ROLES_ORDERED = [ - ReadOnlyRoleNoDownload, ReadOnlyRole, DataEntryOnlyRole, - DataEntryMinorRole, DataEntryRole, EditorMinorRole, EditorRole, - ManagerRole, OwnerRole + ReadOnlyRoleNoDownload, + ReadOnlyRole, + DataEntryOnlyRole, + DataEntryMinorRole, + DataEntryRole, + EditorMinorRole, + EditorRole, + ManagerRole, + OwnerRole, ] ROLES = {role.name: role for role in ROLES_ORDERED} @@ -347,8 +433,7 @@ def is_organization(obj): UserProfiles do. Check for that first since it avoids a database hit. """ try: - return (hasattr(obj, 'userprofile_ptr') - or obj.organizationprofile is not None) + return hasattr(obj, "userprofile_ptr") or obj.organizationprofile is not None except OrganizationProfile.DoesNotExist: return False @@ -369,7 +454,7 @@ def get_role_in_org(user, organization): """ perms = get_perms(user, organization) - if 'is_org_owner' in perms: + if "is_org_owner" in perms: return OwnerRole.name return get_role(perms, organization) or MemberRole.name @@ -382,8 +467,11 @@ def get_user_perms(obj): model = XFormUserObjectPermission if isinstance(obj, XForm) else None model = ProjectUserObjectPermission if isinstance(obj, Project) else model - return queryset_iterator( - model.objects.filter(content_object_id=obj.pk)) if model else None + return ( + queryset_iterator(model.objects.filter(content_object_id=obj.pk)) + if model + else None + ) def get_group_perms(obj): @@ -393,8 +481,11 @@ def get_group_perms(obj): model = XFormGroupObjectPermission if isinstance(obj, XForm) else None model = ProjectGroupObjectPermission if isinstance(obj, Project) else model - return queryset_iterator( - model.objects.filter(content_object_id=obj.pk)) if model else None + return ( + queryset_iterator(model.objects.filter(content_object_id=obj.pk)) + if model + else None + ) def _get_group_users_with_perms(obj, attach_perms=False, user_perms=None): @@ -404,7 +495,8 @@ def _get_group_users_with_perms(obj, attach_perms=False, user_perms=None): group_obj_perms = get_group_perms(obj) if group_obj_perms is None: return get_users_with_perms( - obj, attach_perms=attach_perms, with_group_users=True) + obj, attach_perms=attach_perms, with_group_users=True + ) group_users = {} if attach_perms: if user_perms: @@ -420,9 +512,8 @@ def _get_group_users_with_perms(obj, attach_perms=False, user_perms=None): group_users[user] = set([perm.permission.codename]) else: group_users = set() if not user_perms else set(user_perms) - for perm in group_obj_perms.distinct('group'): - group_users.union( - set([user for user in perm.group.user_set.all()])) + for perm in group_obj_perms.distinct("group"): + group_users.union(set(user for user in perm.group.user_set.all())) group_users = list(group_obj_perms) return group_users @@ -435,7 +526,8 @@ def _get_users_with_perms(obj, attach_perms=False, with_group_users=None): user_obj_perms = get_user_perms(obj) if user_obj_perms is None: return get_users_with_perms( - obj, attach_perms=attach_perms, with_group_users=with_group_users) + obj, attach_perms=attach_perms, with_group_users=with_group_users + ) user_perms = {} if attach_perms: for perm in user_obj_perms: @@ -445,7 +537,7 @@ def _get_users_with_perms(obj, attach_perms=False, with_group_users=None): user_perms[perm.user] = set([perm.permission.codename]) else: user_perms = [ - perm.user for perm in user_obj_perms.only('user').distinct('user') + perm.user for perm in user_obj_perms.only("user").distinct("user") ] if with_group_users: @@ -454,9 +546,10 @@ def _get_users_with_perms(obj, attach_perms=False, with_group_users=None): return user_perms -def get_object_users_with_permissions(obj, # pylint: disable=invalid-name - username=False, - with_group_users=False): +# pylint: disable=invalid-name +def get_object_users_with_permissions( + obj, username=False, with_group_users=False # pylint: disable=invalid-name +): """ Returns users, roles and permissions for an object. @@ -467,17 +560,21 @@ def get_object_users_with_permissions(obj, # pylint: disable=invalid-name if obj: users_with_perms = _get_users_with_perms( - obj, attach_perms=True, with_group_users=with_group_users).items() - - result = [{ - 'user': user.username if username else user, - 'first_name': user.first_name, - 'last_name': user.last_name, - 'role': get_role(permissions, obj), - 'is_org': is_organization(user.profile), - 'gravatar': user.profile.gravatar, - 'metadata': user.profile.metadata - } for user, permissions in users_with_perms] + obj, attach_perms=True, with_group_users=with_group_users + ).items() + + result = [ + { + "user": user.username if username else user, + "first_name": user.first_name, + "last_name": user.last_name, + "role": get_role(permissions, obj), + "is_org": is_organization(user.profile), + "gravatar": user.profile.gravatar, + "metadata": user.profile.metadata, + } + for user, permissions in users_with_perms + ] return result @@ -494,45 +591,49 @@ def get_team_project_default_permissions(team, project): def _check_meta_perms_enabled(xform): """ - Check for meta-perms settings in the xform metadata model. - :param xform: - :return: bool + Check for meta-perms settings in the xform metadata model. + :param xform: + :return: bool """ return xform.metadata_set.filter(data_type=XFORM_META_PERMS).count() > 0 -def exclude_items_from_queryset_using_xform_meta_perms( - xform, user, queryset): +# pylint: disable=invalid-name +def exclude_items_from_queryset_using_xform_meta_perms(xform, user, queryset): """ Exclude instances from the queryset if meta-perms have been enabled """ - if user.has_perm(CAN_VIEW_XFORM_ALL, xform) or xform.shared_data \ - or not _check_meta_perms_enabled(xform): + if ( + user.has_perm(CAN_VIEW_XFORM_ALL, xform) + or xform.shared_data + or not _check_meta_perms_enabled(xform) + ): return queryset - elif user.has_perm(CAN_VIEW_XFORM_DATA, xform): + if user.has_perm(CAN_VIEW_XFORM_DATA, xform): if queryset.model is Attachment: - return queryset.exclude( - ~Q(instance__user=user), instance__xform=xform) - else: - return queryset.exclude( - ~Q(user=user), xform=xform) + return queryset.exclude(~Q(instance__user=user), instance__xform=xform) + return queryset.exclude(~Q(user=user), xform=xform) + return queryset.none() def filter_queryset_xform_meta_perms(xform, user, instance_queryset): """ - Check for the specific perms if meta-perms have been enabled - CAN_VIEW_XFORM_ALL ==> User should be able to view all the data - CAN_VIEW_XFORM_DATA ===> User should be able to view his/her submitted - data. Otherwise should raise forbidden error. - :param xform: - :param user: - :param instance_queryset: - :return: data - """ - if user.has_perm(CAN_VIEW_XFORM_ALL, xform) or xform.shared_data \ - or not _check_meta_perms_enabled(xform): + Check for the specific perms if meta-perms have been enabled + CAN_VIEW_XFORM_ALL ==> User should be able to view all the data + CAN_VIEW_XFORM_DATA ===> User should be able to view his/her submitted + data. Otherwise should raise forbidden error. + :param xform: + :param user: + :param instance_queryset: + :return: data + """ + if ( + user.has_perm(CAN_VIEW_XFORM_ALL, xform) + or xform.shared_data + or not _check_meta_perms_enabled(xform) + ): return instance_queryset - elif user.has_perm(CAN_VIEW_XFORM_DATA, xform): + if user.has_perm(CAN_VIEW_XFORM_DATA, xform): return instance_queryset.filter(user=user) return instance_queryset.none() @@ -540,33 +641,35 @@ def filter_queryset_xform_meta_perms(xform, user, instance_queryset): def filter_queryset_xform_meta_perms_sql(xform, user, query): """ - Check for the specific perms if meta-perms have been enabled - CAN_VIEW_XFORM_ALL ==> User should be able to view all the data - CAN_VIEW_XFORM_DATA ===> User should be able to view his/her submitted - data. Otherwise should raise forbidden error. - :param xform: - :param user: - :param instance_queryset: - :return: data - """ - if user.has_perm(CAN_VIEW_XFORM_ALL, xform) or xform.shared_data\ - or not _check_meta_perms_enabled(xform): + Check for the specific perms if meta-perms have been enabled + CAN_VIEW_XFORM_ALL ==> User should be able to view all the data + CAN_VIEW_XFORM_DATA ===> User should be able to view his/her submitted + data. Otherwise should raise forbidden error. + :param xform: + :param user: + :param instance_queryset: + :return: data + """ + if ( + user.has_perm(CAN_VIEW_XFORM_ALL, xform) + or xform.shared_data + or not _check_meta_perms_enabled(xform) + ): return query - elif user.has_perm(CAN_VIEW_XFORM_DATA, xform): + if user.has_perm(CAN_VIEW_XFORM_DATA, xform): try: if query and isinstance(query, six.string_types): query = json.loads(query) if isinstance(query, list): query = query[0] else: - query = dict() + query = {} query.update({"_submitted_by": user.username}) return query except (ValueError, AttributeError): - query_list = list() + query_list = [] query_list.append({"_submitted_by": user.username}) query_list.append(query) return query_list - else: - raise NoRecordsPermission() + raise NoRecordsPermission() diff --git a/onadata/libs/profiling/sql.py b/onadata/libs/profiling/sql.py index 1c26037464..c9b24fe5fc 100644 --- a/onadata/libs/profiling/sql.py +++ b/onadata/libs/profiling/sql.py @@ -6,14 +6,14 @@ from django.db import connection -sql_log = logging.getLogger('sql_logger') # pylint: disable=C0103 -totals_log = logging.getLogger('sql_totals_logger') # pylint: disable=C0103 +SQL_LOG = logging.getLogger("sql_logger") +TOTALS_LOG = logging.getLogger("sql_totals_logger") # modified from # http://johnparsons.net/index.php/2013/08/15/easy-sql-query-counting-in-django -class SqlTimingMiddleware(object): # pylint: disable=R0903 +class SqlTimingMiddleware: """ Logs the time taken by each sql query over requests. Logs the total time taken to run sql queries and the number of sql queries @@ -24,17 +24,16 @@ def __init__(self, get_response): self.get_response = get_response def __call__(self, request): - path_info = '%s %s' % (request.method, request.path_info) + path_info = f"{request.method} {request.path_info}" response = self.get_response(request) sqltime = 0 # Variable to store execution time for query in connection.queries: # Add the time that the query took to the total sqltime += float(query["time"]) - sql_log.debug(path_info, extra=query) + SQL_LOG.debug(path_info, extra=query) - totals_log.debug( - path_info, - extra={'time': sqltime, - 'num_queries': len(connection.queries)}) + TOTALS_LOG.debug( + path_info, extra={"time": sqltime, "num_queries": len(connection.queries)} + ) return response diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index 8a0604f617..b068037977 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -8,14 +8,13 @@ from io import BytesIO, StringIO from typing import Tuple -import pytz -import six - from django.utils import timezone from django.utils.dateparse import parse_datetime -from django.utils.encoding import smart_str, force_str +from django.utils.encoding import force_str, smart_str from django.utils.xmlutils import SimplerXMLGenerator -from six import iteritems + +import pytz +import six from rest_framework import negotiation from rest_framework.renderers import ( BaseRenderer, @@ -25,6 +24,7 @@ ) from rest_framework.utils.encoders import JSONEncoder from rest_framework_xml.renderers import XMLRenderer +from six import iteritems from onadata.libs.utils.osm import get_combined_osm @@ -122,7 +122,6 @@ class XLSRenderer(BaseRenderer): format = "xls" charset = None - # pylint: disable=no-self-use,unused-argument def render(self, data, accepted_media_type=None, renderer_context=None): """ Encode ``data`` string to 'utf-8'. @@ -226,8 +225,7 @@ class MediaFileContentNegotiation(negotiation.DefaultContentNegotiation): matching format. """ - # pylint: disable=redefined-builtin,no-self-use - def filter_renderers(self, renderers, format): + def filter_renderers(self, renderers, format): # pylint: disable=redefined-builtin """ If there is a '.json' style format suffix, filter the renderers so that we only negotiation against those that accept that format. @@ -360,6 +358,7 @@ def _get_current_buffer_data(self): return None def stream_data(self, data, serializer): + """Returns a streaming response.""" if data is None: yield "" @@ -372,7 +371,7 @@ def stream_data(self, data, serializer): yield self._get_current_buffer_data() - data = data.__iter__() + data = iter(data) try: out = next(data) diff --git a/onadata/libs/serializers/attachment_serializer.py b/onadata/libs/serializers/attachment_serializer.py index 7c676b2fc4..174fc4ca6f 100644 --- a/onadata/libs/serializers/attachment_serializer.py +++ b/onadata/libs/serializers/attachment_serializer.py @@ -3,13 +3,11 @@ Attachments serializer. """ -from six import itervalues - from rest_framework import serializers +from six import itervalues from onadata.apps.logger.models.attachment import Attachment -from onadata.apps.logger.models.instance import get_attachment_url -from onadata.apps.logger.models.instance import Instance +from onadata.apps.logger.models.instance import Instance, get_attachment_url from onadata.libs.utils.decorators import check_obj @@ -110,7 +108,6 @@ def get_medium_download_url(self, obj): return request.build_absolute_uri(path) if request else path return "" - # pylint: disable=no-self-use def get_field_xpath(self, obj): """ Return question xpath diff --git a/onadata/libs/serializers/clone_xform_serializer.py b/onadata/libs/serializers/clone_xform_serializer.py index 0392a41342..0ab1b589f1 100644 --- a/onadata/libs/serializers/clone_xform_serializer.py +++ b/onadata/libs/serializers/clone_xform_serializer.py @@ -19,7 +19,6 @@ class CloneXFormSerializer(serializers.Serializer): username = serializers.CharField(max_length=255) project = ProjectField(required=False) - # pylint: disable=no-self-use def create(self, validated_data): """Uses the CloneXForm class to clone/copy an XForm. @@ -29,7 +28,6 @@ def create(self, validated_data): return instance - # pylint: disable=no-self-use def update(self, instance, validated_data): instance.xform = validated_data.get("xform", instance.xform) instance.username = validated_data.get("username", instance.username) @@ -38,7 +36,6 @@ def update(self, instance, validated_data): return instance - # pylint: disable=no-self-use def validate_username(self, value): """Check that the username exists""" # pylint: disable=invalid-name diff --git a/onadata/libs/serializers/data_serializer.py b/onadata/libs/serializers/data_serializer.py index 1638ff493c..e2b3a47fb8 100644 --- a/onadata/libs/serializers/data_serializer.py +++ b/onadata/libs/serializers/data_serializer.py @@ -1,28 +1,43 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Submission data serializers module. """ -import xmltodict from io import BytesIO from django.shortcuts import get_object_or_404 from django.utils.translation import gettext as _ + +import xmltodict from rest_framework import exceptions, serializers from rest_framework.reverse import reverse -from onadata.apps.logger.models.instance import Instance, InstanceHistory from onadata.apps.logger.models import Project, XForm +from onadata.apps.logger.models.instance import Instance, InstanceHistory from onadata.libs.serializers.fields.json_field import JsonField +from onadata.libs.utils.analytics import TrackObjectEvent from onadata.libs.utils.common_tags import ( - METADATA_FIELDS, NOTES, TAGS, DATE_MODIFIED, VERSION, GEOLOCATION, - XFORM_ID, ATTACHMENTS, XFORM_ID_STRING, UUID) -from onadata.libs.utils.logger_tools import remove_metadata_fields -from onadata.libs.utils.dict_tools import (dict_lists2strings, dict_paths2dict, - query_list_to_dict, - floip_response_headers_dict) -from onadata.libs.utils.logger_tools import dict2xform, safe_create_instance -from onadata.libs.utils.analytics import track_object_event - + ATTACHMENTS, + DATE_MODIFIED, + GEOLOCATION, + METADATA_FIELDS, + NOTES, + TAGS, + UUID, + VERSION, + XFORM_ID, + XFORM_ID_STRING, +) +from onadata.libs.utils.dict_tools import ( + dict_lists2strings, + dict_paths2dict, + floip_response_headers_dict, + query_list_to_dict, +) +from onadata.libs.utils.logger_tools import ( + dict2xform, + remove_metadata_fields, + safe_create_instance, +) NUM_FLOIP_COLUMNS = 6 @@ -31,11 +46,11 @@ def get_request_and_username(context): """ Returns request object and username """ - request = context['request'] - view = context['view'] - username = view.kwargs.get('username') - form_pk = view.kwargs.get('xform_pk') - project_pk = view.kwargs.get('project_pk') + request = context["request"] + view = context["view"] + username = view.kwargs.get("username") + form_pk = view.kwargs.get("xform_pk") + project_pk = view.kwargs.get("project_pk") if not username: # get the username from the XForm object if form_id is @@ -45,22 +60,19 @@ def get_request_and_username(context): elif project_pk: username = Project.objects.get(pk=project_pk).user.username else: - username = (request.user and request.user.username) + username = request.user and request.user.username return (request, username) -def create_submission( - request, username, data_dict, xform_id, gen_uuid: bool = False): +def create_submission(request, username, data_dict, xform_id, gen_uuid: bool = False): """ Returns validated data object instances """ - xml_string = dict2xform( - data_dict, xform_id, username=username, gen_uuid=gen_uuid) - xml_file = BytesIO(xml_string.encode('utf-8')) + xml_string = dict2xform(data_dict, xform_id, username=username, gen_uuid=gen_uuid) + xml_file = BytesIO(xml_string.encode("utf-8")) - error, instance = safe_create_instance(username, xml_file, [], None, - request) + error, instance = safe_create_instance(username, xml_file, [], None, request) if error: raise serializers.ValidationError(error.message) @@ -72,19 +84,25 @@ class DataSerializer(serializers.HyperlinkedModelSerializer): DataSerializer class - used for the list view to show `id`, `id_string`, `title` and `description`. """ - url = serializers.HyperlinkedIdentityField( - view_name='data-list', lookup_field='pk') + + url = serializers.HyperlinkedIdentityField(view_name="data-list", lookup_field="pk") class Meta: model = XForm - fields = ('id', 'id_string', 'title', 'description', 'url') + fields = ("id", "id_string", "title", "description", "url") -class JsonDataSerializer(serializers.Serializer): # pylint: disable=W0223 +class JsonDataSerializer(serializers.Serializer): """ JSON DataSerializer class - for json field data representation. """ + def create(self, validated_data): + pass + + def update(self, instance, validated_data): + pass + def to_representation(self, instance): return instance @@ -93,17 +111,17 @@ class InstanceHistorySerializer(serializers.ModelSerializer): """ InstanceHistorySerializer class - for the json field data representation. """ + json = JsonField() class Meta: model = InstanceHistory - fields = ('json', ) + fields = ("json",) def to_representation(self, instance): - ret = super(InstanceHistorySerializer, - self).to_representation(instance) + ret = super().to_representation(instance) - return ret['json'] if 'json' in ret else ret + return ret["json"] if "json" in ret else ret class DataInstanceXMLSerializer(serializers.ModelSerializer): @@ -114,51 +132,56 @@ class DataInstanceXMLSerializer(serializers.ModelSerializer): class Meta: model = Instance - fields = ('xml', ) + fields = ("xml",) def _convert_metadata_field_to_attribute(self, field: str) -> str: """ Converts a metadata field such as `_review_status` into a camel cased attribute `reviewStatus` """ - split_field = field.split('_')[1:] - return split_field[0] + ''.join( - word.title() for word in split_field[1:]) + split_field = field.split("_")[1:] + return split_field[0] + "".join(word.title() for word in split_field[1:]) def to_representation(self, instance): - ret = super( - DataInstanceXMLSerializer, self).to_representation(instance) - if 'xml' in ret: - ret = xmltodict.parse(ret['xml'], cdata_key="") + ret = super().to_representation(instance) + if "xml" in ret: + ret = xmltodict.parse(ret["xml"], cdata_key="") # Add Instance attributes to representation instance_attributes = { - '@formVersion': instance.version, - '@lastModified': instance.date_modified.isoformat(), - '@dateCreated': instance.date_created.isoformat(), - '@objectID': str(instance.id) + "@formVersion": instance.version, + "@lastModified": instance.date_modified.isoformat(), + "@dateCreated": instance.date_created.isoformat(), + "@objectID": str(instance.id), } ret.update(instance_attributes) excluded_metadata = [ - NOTES, TAGS, GEOLOCATION, XFORM_ID, DATE_MODIFIED, VERSION, - ATTACHMENTS, XFORM_ID_STRING, UUID] + NOTES, + TAGS, + GEOLOCATION, + XFORM_ID, + DATE_MODIFIED, + VERSION, + ATTACHMENTS, + XFORM_ID_STRING, + UUID, + ] additional_attributes = [ (self._convert_metadata_field_to_attribute(metadata), metadata) for metadata in METADATA_FIELDS - if metadata not in excluded_metadata] + if metadata not in excluded_metadata + ] for attrib, meta_field in additional_attributes: meta_value = instance.json.get(meta_field, "") if not isinstance(meta_value, str): meta_value = str(meta_value) - ret.update({ - f"@{attrib}": meta_value - }) + ret.update({f"@{attrib}": meta_value}) # Include linked resources linked_resources = { - 'linked-resources': { - 'attachments': instance.json.get(ATTACHMENTS), - 'notes': instance.json.get(NOTES) + "linked-resources": { + "attachments": instance.json.get(ATTACHMENTS), + "notes": instance.json.get(NOTES), } } ret.update(linked_resources) @@ -170,16 +193,17 @@ class DataInstanceSerializer(serializers.ModelSerializer): DataInstanceSerializer class - for json field data representation on the Instance (submission) model. """ + json = JsonField() class Meta: model = Instance - fields = ('json', ) + fields = ("json",) def to_representation(self, instance): - ret = super(DataInstanceSerializer, self).to_representation(instance) - if 'json' in ret: - ret = ret['json'] + ret = super().to_representation(instance) + if "json" in ret: + ret = ret["json"] return ret @@ -187,23 +211,24 @@ class TableauDataSerializer(serializers.ModelSerializer): """ TableauDataSerializer class - cleans out instance fields. """ + json = JsonField() class Meta: model = Instance - fields = ('json', ) + fields = ("json",) def to_representation(self, instance): - ret = super(TableauDataSerializer, self).to_representation(instance) - if 'json' in ret: - ret = ret['json'] + ret = super().to_representation(instance) + if "json" in ret: + ret = ret["json"] # Remove metadata fields from the instance remove_metadata_fields(ret) return ret -class SubmissionSuccessMixin(object): # pylint: disable=R0903 +class SubmissionSuccessMixin: """ SubmissionSuccessMixin - prepares submission success data/message. """ @@ -213,65 +238,77 @@ def to_representation(self, instance): Returns a dict with a successful submission message. """ if instance is None: - return super(SubmissionSuccessMixin, self)\ - .to_representation(instance) + return super().to_representation(instance) return { - 'message': _("Successful submission."), - 'formid': instance.xform.id_string, - 'encrypted': instance.xform.encrypted, - 'instanceID': u'uuid:%s' % instance.uuid, - 'submissionDate': instance.date_created.isoformat(), - 'markedAsCompleteDate': instance.date_modified.isoformat() + "message": _("Successful submission."), + "formid": instance.xform.id_string, + "encrypted": instance.xform.encrypted, + "instanceID": f"uuid:{instance.uuid}", + "submissionDate": instance.date_created.isoformat(), + "markedAsCompleteDate": instance.date_modified.isoformat(), } -class BaseRapidProSubmissionSerializer(SubmissionSuccessMixin, - serializers.Serializer): +class BaseRapidProSubmissionSerializer(SubmissionSuccessMixin, serializers.Serializer): """ Base Rapidpro SubmissionSerializer - Implements the basic functionalities of a Rapidpro webhook serializer """ + + def create(self, validated_data): + pass + + def update(self, instance, validated_data): + pass + def validate(self, attrs): """ Validate that the XForm ID is passed in view kwargs """ - view = self.context['view'] + view = self.context["view"] - if 'xform_pk' in view.kwargs: - xform_pk = view.kwargs.get('xform_pk') + if "xform_pk" in view.kwargs: + xform_pk = view.kwargs.get("xform_pk") xform = get_object_or_404(XForm, pk=xform_pk) - attrs.update({'id_string': xform.id_string}) + attrs.update({"id_string": xform.id_string}) else: - raise serializers.ValidationError({ - 'xform_pk': - _(u'Incorrect url format. Use format ' - u'https://api.ona.io/username/formid/submission')}) + raise serializers.ValidationError( + { + "xform_pk": _( + "Incorrect url format. Use format " + "https://api.ona.io/username/formid/submission" + ) + } + ) - return super(BaseRapidProSubmissionSerializer, self).validate(attrs) + return super().validate(attrs) -# pylint: disable=W0223 class SubmissionSerializer(SubmissionSuccessMixin, serializers.Serializer): """ XML SubmissionSerializer - handles creating a submission from XML. """ + def update(self, instance, validated_data): + pass + def validate(self, attrs): request, __ = get_request_and_username(self.context) - if not request.FILES or 'xml_submission_file' not in request.FILES: + if not request.FILES or "xml_submission_file" not in request.FILES: raise serializers.ValidationError(_("No XML submission file.")) - return super(SubmissionSerializer, self).validate(attrs) + return super().validate(attrs) - @track_object_event( - user_field='xform__user', + @TrackObjectEvent( + user_field="xform__user", properties={ - 'submitted_by': 'user', - 'xform_id': 'xform__pk', - 'project_id': 'xform__project__pk', - 'organization': 'xform__user__profile__organization'}, - additional_context={'from': 'XML Submissions'} + "submitted_by": "user", + "xform_id": "xform__pk", + "project_id": "xform__project__pk", + "organization": "xform__user__profile__organization", + }, + additional_context={"from": "XML Submissions"}, ) def create(self, validated_data): """ @@ -279,12 +316,13 @@ def create(self, validated_data): """ request, username = get_request_and_username(self.context) - xml_file_list = request.FILES.pop('xml_submission_file', []) + xml_file_list = request.FILES.pop("xml_submission_file", []) xml_file = xml_file_list[0] if xml_file_list else None media_files = request.FILES.values() - error, instance = safe_create_instance(username, xml_file, media_files, - None, request) + error, instance = safe_create_instance( + username, xml_file, media_files, None, request + ) if error: exc = exceptions.APIException(detail=error) exc.response = error @@ -300,24 +338,30 @@ class OSMSerializer(serializers.Serializer): OSM Serializer - represents OSM data. """ + def create(self, validated_data): + pass + + def update(self, instance, validated_data): + pass + def to_representation(self, instance): """ Return a list of osm file objects from attachments. """ return instance - # pylint: disable=W0201 @property def data(self): """ Returns the serialized data on the serializer. """ - if not hasattr(self, '_data'): - if self.instance is not None and \ - not getattr(self, '_errors', None): + # pylint: disable=attribute-defined-outside-init + if not hasattr(self, "_data"): + if self.instance is not None and not getattr(self, "_errors", None): self._data = self.to_representation(self.instance) - elif hasattr(self, '_validated_data') and \ - not getattr(self, '_errors', None): + elif hasattr(self, "_validated_data") and not getattr( + self, "_errors", None + ): self._data = self.to_representation(self.validated_data) else: self._data = self.get_initial() @@ -330,28 +374,27 @@ class OSMSiteMapSerializer(serializers.Serializer): OSM SiteMap Serializer. """ + def create(self, validated_data): + pass + + def update(self, instance, validated_data): + pass + def to_representation(self, instance): """ Return a list of osm file objects from attachments. """ if instance is None: - return super(OSMSiteMapSerializer, self)\ - .to_representation(instance) + return super().to_representation(instance) - id_string = instance.get('instance__xform__id_string') - title = instance.get('instance__xform__title') - user = instance.get('instance__xform__user__username') + id_string = instance.get("instance__xform__id_string") + title = instance.get("instance__xform__title") + user = instance.get("instance__xform__user__username") - kwargs = {'pk': instance.get('instance__xform')} - url = reverse( - 'osm-list', kwargs=kwargs, request=self.context.get('request')) + kwargs = {"pk": instance.get("instance__xform")} + url = reverse("osm-list", kwargs=kwargs, request=self.context.get("request")) - return { - 'url': url, - 'title': title, - 'id_string': id_string, - 'user': user - } + return {"url": url, "title": title, "id_string": id_string, "user": user} class JSONSubmissionSerializer(SubmissionSuccessMixin, serializers.Serializer): @@ -359,52 +402,58 @@ class JSONSubmissionSerializer(SubmissionSuccessMixin, serializers.Serializer): JSON SubmissionSerializer - handles JSON submission data. """ + def update(self, instance, validated_data): + pass + def validate(self, attrs): """ Custom submission validator in request data. """ - request = self.context['request'] + request = self.context["request"] - if 'submission' not in request.data: - raise serializers.ValidationError({ - 'submission': - _(u"No submission key provided.") - }) + if "submission" not in request.data: + raise serializers.ValidationError( + {"submission": _("No submission key provided.")} + ) - submission = request.data.get('submission') + submission = request.data.get("submission") if not submission: - raise serializers.ValidationError({ - 'submission': - _(u"Received empty submission. No instance was created") - }) + raise serializers.ValidationError( + {"submission": _("Received empty submission. No instance was created")} + ) - return super(JSONSubmissionSerializer, self).validate(attrs) + return super().validate(attrs) - @track_object_event( - user_field='xform__user', + @TrackObjectEvent( + user_field="xform__user", properties={ - 'submitted_by': 'user', - 'xform_id': 'xform__pk', - 'project_id': 'xform__project__pk', - 'organization': 'xform__user__profile__organization'}, - additional_context={'from': 'JSON Submission'} + "submitted_by": "user", + "xform_id": "xform__pk", + "project_id": "xform__project__pk", + "organization": "xform__user__profile__organization", + }, + additional_context={"from": "JSON Submission"}, ) def create(self, validated_data): """ Returns object instances based on the validated data """ request, username = get_request_and_username(self.context) - submission = request.data.get('submission') + submission = request.data.get("submission") # convert lists in submission dict to joined strings try: submission_joined = dict_paths2dict(dict_lists2strings(submission)) - except AttributeError: + except AttributeError as exc: raise serializers.ValidationError( - _(u'Incorrect format, see format details here,' - u'https://api.ona.io/static/docs/submissions.html.')) + _( + "Incorrect format, see format details here," + "https://api.ona.io/static/docs/submissions.html." + ) + ) from exc - instance = create_submission(request, username, submission_joined, - request.data.get('id')) + instance = create_submission( + request, username, submission_joined, request.data.get("id") + ) return instance @@ -413,24 +462,28 @@ class RapidProSubmissionSerializer(BaseRapidProSubmissionSerializer): """ Rapidpro SubmissionSerializer - handles Rapidpro webhook post. """ - @track_object_event( - user_field='xform__user', + + def update(self, instance, validated_data): + pass + + @TrackObjectEvent( + user_field="xform__user", properties={ - 'submitted_by': 'user', - 'xform_id': 'xform__pk', - 'project_id': 'xform__project__pk', - }, - additional_context={'from': 'RapidPro'} + "submitted_by": "user", + "xform_id": "xform__pk", + "project_id": "xform__project__pk", + }, + additional_context={"from": "RapidPro"}, ) def create(self, validated_data): """ Returns object instances based on the validated data. """ request, username = get_request_and_username(self.context) - rapidpro_dict = query_list_to_dict(request.data.get('values')) - instance = create_submission(request, username, rapidpro_dict, - validated_data['id_string'], - gen_uuid=True) + rapidpro_dict = query_list_to_dict(request.data.get("values")) + instance = create_submission( + request, username, rapidpro_dict, validated_data["id_string"], gen_uuid=True + ) return instance @@ -439,26 +492,33 @@ class RapidProJSONSubmissionSerializer(BaseRapidProSubmissionSerializer): """ Rapidpro SubmissionSerializer - handles RapidPro JSON webhook posts """ - @track_object_event( - user_field='xform__user', + + def update(self, instance, validated_data): + pass + + @TrackObjectEvent( + user_field="xform__user", properties={ - 'submitted_by': 'user', - 'xform_id': 'xform__pk', - 'project_id': 'xform__project__pk', - }, - additional_context={'from': 'RapidPro(JSON)'} + "submitted_by": "user", + "xform_id": "xform__pk", + "project_id": "xform__project__pk", + }, + additional_context={"from": "RapidPro(JSON)"}, ) def create(self, validated_data): """ Returns object instances based on validated data. """ request, username = get_request_and_username(self.context) - post_data = request.data.get('results') - instance_data_dict = { - k: post_data[k].get('value') for k in post_data.keys()} + post_data = request.data.get("results") + instance_data_dict = {k: post_data[k].get("value") for k in post_data.keys()} instance = create_submission( - request, username, instance_data_dict, - validated_data['id_string'], gen_uuid=True) + request, + username, + instance_data_dict, + validated_data["id_string"], + gen_uuid=True, + ) return instance @@ -466,39 +526,48 @@ class FLOIPListSerializer(serializers.ListSerializer): """ Custom ListSerializer for a FLOIP submission. """ - @track_object_event( - user_field='xform__user', + + def update(self, instance, validated_data): + pass + + @TrackObjectEvent( + user_field="xform__user", properties={ - 'submitted_by': 'user', - 'xform_id': 'xform__pk', - 'project_id': 'xform__project__pk', - }, - additional_context={'from': 'FLOIP'} + "submitted_by": "user", + "xform_id": "xform__pk", + "project_id": "xform__project__pk", + }, + additional_context={"from": "FLOIP"}, ) def create(self, validated_data): """ Returns object instances based on the validated data. """ request, username = get_request_and_username(self.context) - xform_pk = self.context['view'].kwargs['xform_pk'] + xform_pk = self.context["view"].kwargs["xform_pk"] xform = get_object_or_404(XForm, pk=xform_pk) xform_headers = xform.get_keys() flow_dict = floip_response_headers_dict(request.data, xform_headers) - instance = create_submission(request, username, flow_dict, - xform) + instance = create_submission(request, username, flow_dict, xform) return [instance] -class FLOIPSubmissionSerializer(SubmissionSuccessMixin, - serializers.Serializer): +class FLOIPSubmissionSerializer(SubmissionSuccessMixin, serializers.Serializer): """ FLOIP SubmmissionSerializer - Handles a row of FLOIP specification format. """ + + def create(self, validated_data): + pass + + def update(self, instance, validated_data): + pass + def run_validators(self, value): # Only run default run_validators if we have validators attached to the # serializer. if self.validators: - return super(FLOIPSubmissionSerializer, self).run_validators(value) + return super().run_validators(value) return [] @@ -506,25 +575,29 @@ def validate(self, attrs): """ Custom list data validator. """ - data = self.context['request'].data + data = self.context["request"].data error_msg = None if not isinstance(data, list): - error_msg = u'Invalid format. Expecting a list.' + error_msg = "Invalid format. Expecting a list." elif data: for row_i, row in enumerate(data): if len(row) != NUM_FLOIP_COLUMNS: - error_msg = _(u"Wrong number of values (%(values)d) in row" - " %(row)d, expecting %(expected)d values" - % {'row': row_i, - 'values': (len(row)), - 'expected': NUM_FLOIP_COLUMNS}) + error_msg = _( + "Wrong number of values (%(values)d) in row" + " %(row)d, expecting %(expected)d values" + % { + "row": row_i, + "values": (len(row)), + "expected": NUM_FLOIP_COLUMNS, + } + ) break if error_msg: raise serializers.ValidationError(_(error_msg)) - return super(FLOIPSubmissionSerializer, self).validate(attrs) + return super().validate(attrs) def to_internal_value(self, data): """ @@ -539,4 +612,5 @@ class Meta: """ Call the list serializer class to create an instance. """ + list_serializer_class = FLOIPListSerializer diff --git a/onadata/libs/serializers/dataview_serializer.py b/onadata/libs/serializers/dataview_serializer.py index 0a0e323a39..90f7e94262 100644 --- a/onadata/libs/serializers/dataview_serializer.py +++ b/onadata/libs/serializers/dataview_serializer.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +The DataViewSerializer - manage DataView objects. +""" import datetime from django.utils.translation import gettext as _ @@ -11,58 +15,62 @@ from onadata.apps.logger.models.data_view import SUPPORTED_FILTERS from onadata.apps.logger.models.xform import XForm from onadata.apps.logger.models.project import Project -from onadata.libs.utils.cache_tools import ( - DATAVIEW_COUNT, - DATAVIEW_LAST_SUBMISSION_TIME) +from onadata.libs.utils.cache_tools import DATAVIEW_COUNT, DATAVIEW_LAST_SUBMISSION_TIME from onadata.libs.utils.common_tags import MONGO_STRFTIME, DATE_FORMAT from onadata.libs.utils.model_tools import get_columns_with_hxl from onadata.libs.utils.api_export_tools import include_hxl_row -LAST_SUBMISSION_TIME = '_submission_time' +LAST_SUBMISSION_TIME = "_submission_time" def validate_date(value): + """Returns True if the ``value`` is a date string.""" try: datetime.datetime.strptime(value, DATE_FORMAT) - return True except ValueError: return False + return True def validate_datetime(value): + """Returns True if the ``value`` is a datetime string.""" try: datetime.datetime.strptime(value, MONGO_STRFTIME) - return True except ValueError: return False + return True def match_columns(data, instance=None): - matches_parent = data.get('matches_parent') - xform = data.get('xform', instance.xform if instance else None) - columns = data.get('columns', instance.columns if instance else None) + """Checks if the fields in two forms are a match.""" + matches_parent = data.get("matches_parent") + xform = data.get("xform", instance.xform if instance else None) + columns = data.get("columns", instance.columns if instance else None) if xform and columns: fields = xform.get_field_name_xpaths_only() matched = [col for col in columns if col in fields] matches_parent = len(matched) == len(columns) == len(fields) - data['matches_parent'] = matches_parent + data["matches_parent"] = matches_parent return data class DataViewMinimalSerializer(serializers.HyperlinkedModelSerializer): - dataviewid = serializers.ReadOnlyField(source='id') + """ + The DataViewMinimalSerializer - manage DataView objects. + """ + + dataviewid = serializers.ReadOnlyField(source="id") name = serializers.CharField(max_length=255) - url = serializers.HyperlinkedIdentityField(view_name='dataviews-detail', - lookup_field='pk') + url = serializers.HyperlinkedIdentityField( + view_name="dataviews-detail", lookup_field="pk" + ) xform = serializers.HyperlinkedRelatedField( - view_name='xform-detail', lookup_field='pk', - queryset=XForm.objects.all() + view_name="xform-detail", lookup_field="pk", queryset=XForm.objects.all() ) project = serializers.HyperlinkedRelatedField( - view_name='project-detail', lookup_field='pk', - queryset=Project.objects.all() + view_name="project-detail", lookup_field="pk", queryset=Project.objects.all() ) columns = JsonField() query = JsonField(required=False) @@ -70,23 +78,36 @@ class DataViewMinimalSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = DataView - fields = ('dataviewid', 'name', 'url', 'xform', 'project', 'columns', - 'query', 'matches_parent', 'date_created', - 'instances_with_geopoints', 'date_modified') + fields = ( + "dataviewid", + "name", + "url", + "xform", + "project", + "columns", + "query", + "matches_parent", + "date_created", + "instances_with_geopoints", + "date_modified", + ) class DataViewSerializer(serializers.HyperlinkedModelSerializer): - dataviewid = serializers.ReadOnlyField(source='id') + """ + The DataViewSerializer - manage DataView objects. + """ + + dataviewid = serializers.ReadOnlyField(source="id") name = serializers.CharField(max_length=255) - url = serializers.HyperlinkedIdentityField(view_name='dataviews-detail', - lookup_field='pk') + url = serializers.HyperlinkedIdentityField( + view_name="dataviews-detail", lookup_field="pk" + ) xform = serializers.HyperlinkedRelatedField( - view_name='xform-detail', lookup_field='pk', - queryset=XForm.objects.all() + view_name="xform-detail", lookup_field="pk", queryset=XForm.objects.all() ) project = serializers.HyperlinkedRelatedField( - view_name='project-detail', lookup_field='pk', - queryset=Project.objects.all() + view_name="project-detail", lookup_field="pk", queryset=Project.objects.all() ) columns = JsonField() query = JsonField(required=False) @@ -98,85 +119,93 @@ class DataViewSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = DataView - fields = ('dataviewid', 'name', 'xform', 'project', 'columns', 'query', - 'matches_parent', 'count', 'instances_with_geopoints', - 'last_submission_time', 'has_hxl_support', 'url', - 'date_created', 'deleted_at', 'deleted_by') + fields = ( + "dataviewid", + "name", + "xform", + "project", + "columns", + "query", + "matches_parent", + "count", + "instances_with_geopoints", + "last_submission_time", + "has_hxl_support", + "url", + "date_created", + "deleted_at", + "deleted_by", + ) validators = [ serializers.UniqueTogetherValidator( - queryset=DataView.objects.all(), - fields=('name', 'xform') + queryset=DataView.objects.all(), fields=("name", "xform") ) ] def create(self, validated_data): validated_data = match_columns(validated_data) - return super(DataViewSerializer, self).create(validated_data) + return super().create(validated_data) def update(self, instance, validated_data): validated_data = match_columns(validated_data, instance) - return super(DataViewSerializer, self).update(instance, validated_data) + return super().update(instance, validated_data) def validate_query(self, value): + """Checks if the query filters in ``value`` are known.""" if value: - for q in value: - if 'column' not in q: - raise serializers.ValidationError(_( - u"`column` not set in query" - )) + for query in value: + if "column" not in query: + raise serializers.ValidationError(_("`column` not set in query")) - if 'filter' not in q: - raise serializers.ValidationError(_( - u"`filter` not set in query" - )) + if "filter" not in query: + raise serializers.ValidationError(_("`filter` not set in query")) - if 'value' not in q: - raise serializers.ValidationError(_( - u"`value` not set in query" - )) + if "value" not in query: + raise serializers.ValidationError(_("`value` not set in query")) - comp = q.get('filter') + comp = query.get("filter") if comp not in SUPPORTED_FILTERS: - raise serializers.ValidationError(_( - u"Filter not supported" - )) + raise serializers.ValidationError(_("Filter not supported")) return value def validate_columns(self, value): + """Checks the ``value`` is a list.""" if not isinstance(value, list): - raise serializers.ValidationError(_( - u"`columns` should be a list of columns" - )) + raise serializers.ValidationError( + _("`columns` should be a list of columns") + ) return value def validate(self, attrs): - if 'xform' in attrs and attrs.get('xform'): - xform = attrs.get('xform') - know_dates = [e.name for e in - xform.get_survey_elements_of_type('date')] - know_dates.append('_submission_time') - for q in attrs.get('query', []): - column = q.get('column') - value = q.get('value') - - if column in know_dates and not \ - (validate_datetime(value) or validate_date(value)): - raise serializers.ValidationError(_( - u"Date value in {} should be yyyy-mm-ddThh:m:s or " - u"yyyy-mm-dd" - .format(column) - )) - - return super(DataViewSerializer, self).validate(attrs) + if "xform" in attrs and attrs.get("xform"): + xform = attrs.get("xform") + know_dates = [e.name for e in xform.get_survey_elements_of_type("date")] + know_dates.append("_submission_time") + for query in attrs.get("query", []): + column = query.get("column") + value = query.get("value") + + if column in know_dates and not ( + validate_datetime(value) or validate_date(value) + ): + raise serializers.ValidationError( + _( + f"Date value in {column} should be yyyy-mm-ddThh:m:s or " + "yyyy-mm-dd" + ) + ) + + return super().validate(attrs) def get_count(self, obj): + """Returns the submission count for the data view,""" if obj: - count_dict = cache.get('{}{}'.format(DATAVIEW_COUNT, obj.xform.pk)) + count_dict = cache.get(f"{DATAVIEW_COUNT}{obj.xform.pk}") if count_dict: if obj.pk in count_dict: @@ -185,50 +214,52 @@ def get_count(self, obj): count_dict = {} count_rows = DataView.query_data(obj, count=True) - if 'error' in count_rows: - raise ParseError(count_rows.get('error')) + if "error" in count_rows: + raise ParseError(count_rows.get("error")) count_row = count_rows[0] - if 'count' in count_row: - count = count_row.get('count') + if "count" in count_row: + count = count_row.get("count") count_dict.setdefault(obj.pk, count) - cache.set('{}{}'.format(DATAVIEW_COUNT, obj.xform.pk), - count_dict) + cache.set(f"{DATAVIEW_COUNT}{obj.xform.pk}", count_dict) return count return None def get_last_submission_time(self, obj): + """Returns the last submission timestamp.""" if obj: - last_submission_time = cache.get('{}{}'.format( - DATAVIEW_LAST_SUBMISSION_TIME, obj.xform.pk)) + last_submission_time = cache.get( + f"{DATAVIEW_LAST_SUBMISSION_TIME}{obj.xform.pk}" + ) if last_submission_time: return last_submission_time last_submission_rows = DataView.query_data( - obj, last_submission_time=True) # data is returned as list + obj, last_submission_time=True + ) # data is returned as list - if 'error' in last_submission_rows: - raise ParseError(last_submission_rows.get('error')) + if "error" in last_submission_rows: + raise ParseError(last_submission_rows.get("error")) if len(last_submission_rows): last_submission_row = last_submission_rows[0] if LAST_SUBMISSION_TIME in last_submission_row: - last_submission_time = last_submission_row.get( - LAST_SUBMISSION_TIME) + last_submission_time = last_submission_row.get(LAST_SUBMISSION_TIME) cache.set( - '{}{}'.format( - DATAVIEW_LAST_SUBMISSION_TIME, obj.xform.pk), - last_submission_time) + f"{DATAVIEW_LAST_SUBMISSION_TIME}{obj.xform.pk}", + last_submission_time, + ) return last_submission_time return None def get_instances_with_geopoints(self, obj): + """Returns True if a DataView has submissions with geopoints.""" if obj: check_geo = obj.has_geo_columnn_n_data() @@ -241,8 +272,7 @@ def get_instances_with_geopoints(self, obj): return False def get_has_hxl_support(self, obj): - columns_with_hxl = get_columns_with_hxl( - obj.xform.survey.get('children') - ) + """Returns true if a DataView has columns with HXL tags.""" + columns_with_hxl = get_columns_with_hxl(obj.xform.survey.get("children")) return include_hxl_row(obj.columns, list(columns_with_hxl)) diff --git a/onadata/libs/serializers/export_serializer.py b/onadata/libs/serializers/export_serializer.py index c10eab1029..237bc9b0e9 100644 --- a/onadata/libs/serializers/export_serializer.py +++ b/onadata/libs/serializers/export_serializer.py @@ -1,40 +1,59 @@ +# -*- coding: utf-8 -*- +""" +The ExportSerializer class - create, list exports. +""" from rest_framework import serializers +from rest_framework.reverse import reverse from onadata.apps.viewer.models.export import Export from onadata.libs.utils.async_status import status_msg -from rest_framework.reverse import reverse - class ExportSerializer(serializers.HyperlinkedModelSerializer): - date_created = serializers.ReadOnlyField(source='created_on') + """ + The ExportSerializer class - create, list exports. + """ + + date_created = serializers.ReadOnlyField(source="created_on") job_status = serializers.SerializerMethodField() type = serializers.SerializerMethodField() export_url = serializers.SerializerMethodField() class Meta: model = Export - fields = ('id', 'job_status', 'type', 'task_id', 'xform', - 'date_created', 'filename', 'options', 'export_url', - 'error_message') + fields = ( + "id", + "job_status", + "type", + "task_id", + "xform", + "date_created", + "filename", + "options", + "export_url", + "error_message", + ) def get_job_status(self, obj): + """Returns export async status text.""" return status_msg.get(obj.internal_status) def get_type(self, obj): + """Returns export type - CSV,XLS,...""" return obj.export_type def get_export_url(self, obj): + """Returns the export download URL.""" if obj.export_url: return obj.export_url - request = self.context.get('request') + request = self.context.get("request") if request: - export_url = reverse( - 'export-detail', - kwargs={'pk': obj.pk}, + return reverse( + "export-detail", + kwargs={"pk": obj.pk}, request=request, - format=obj.export_type.replace('_', '') + format=obj.export_type.replace("_", ""), ) - return export_url + return None diff --git a/onadata/libs/serializers/fields/hyperlinked_multi_identity_field.py b/onadata/libs/serializers/fields/hyperlinked_multi_identity_field.py index 89d81b7dff..5d977c5515 100644 --- a/onadata/libs/serializers/fields/hyperlinked_multi_identity_field.py +++ b/onadata/libs/serializers/fields/hyperlinked_multi_identity_field.py @@ -1,22 +1,29 @@ +# -*- coding: utf-8 -*- +""" +The HyperlinkedIdentityField class - multi-lookup identity fields. +""" from rest_framework import serializers from rest_framework.reverse import reverse class HyperlinkedMultiIdentityField(serializers.HyperlinkedIdentityField): - lookup_fields = (('pk', 'pk'), ) + """ + The HyperlinkedIdentityField class - multi-lookup identity fields. + """ + + lookup_fields = (("pk", "pk"),) def __init__(self, *args, **kwargs): - lookup_fields = kwargs.pop('lookup_fields', None) + lookup_fields = kwargs.pop("lookup_fields", None) self.lookup_fields = lookup_fields or self.lookup_fields - super(HyperlinkedMultiIdentityField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) + # pylint: disable=redefined-builtin def get_url(self, obj, view_name, request, format): kwargs = {} for slug, field in self.lookup_fields: lookup_field = getattr(obj, field) kwargs[slug] = lookup_field - return reverse( - view_name, kwargs=kwargs, request=request, format=format - ) + return reverse(view_name, kwargs=kwargs, request=request, format=format) diff --git a/onadata/libs/serializers/fields/hyperlinked_multi_related_field.py b/onadata/libs/serializers/fields/hyperlinked_multi_related_field.py index 4133bb5c03..eeb909429f 100644 --- a/onadata/libs/serializers/fields/hyperlinked_multi_related_field.py +++ b/onadata/libs/serializers/fields/hyperlinked_multi_related_field.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +The HyperlinkedRelatedField class - multi-lookup fields. +""" from rest_framework import serializers from rest_framework.reverse import reverse @@ -12,25 +16,30 @@ def get_obj_property_value(obj, field): <<< list """ - _attr_list = field.split('.') + _attr_list = field.split(".") if len(_attr_list) > 1: tmp_obj = getattr(obj, _attr_list[0]) - return get_obj_property_value(tmp_obj, '.'.join(_attr_list[1:])) + return get_obj_property_value(tmp_obj, ".".join(_attr_list[1:])) return getattr(obj, field) class HyperlinkedMultiRelatedField(serializers.HyperlinkedRelatedField): - lookup_fields = (('pk', 'pk'), ) + """ + The HyperlinkedRelatedField class - multi-lookup fields. + """ + + lookup_fields = (("pk", "pk"),) def __init__(self, *args, **kwargs): - lookup_fields = kwargs.pop('lookup_fields', None) + lookup_fields = kwargs.pop("lookup_fields", None) self.lookup_fields = lookup_fields or self.lookup_fields - super(HyperlinkedMultiRelatedField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) + # pylint: disable=redefined-builtin def get_url(self, obj, view_name, request, format): kwargs = {} @@ -38,5 +47,4 @@ def get_url(self, obj, view_name, request, format): lookup_field = get_obj_property_value(obj, field) kwargs[slug] = lookup_field - return reverse( - view_name, kwargs=kwargs, request=request, format=format) + return reverse(view_name, kwargs=kwargs, request=request, format=format) diff --git a/onadata/libs/serializers/fields/json_field.py b/onadata/libs/serializers/fields/json_field.py index d247f84f13..75ea8fa9e6 100644 --- a/onadata/libs/serializers/fields/json_field.py +++ b/onadata/libs/serializers/fields/json_field.py @@ -12,7 +12,6 @@ class JsonField(serializers.Field): Deserialize a string instance containing a JSON document to a Python object. """ - # pylint: disable=no-self-use def to_representation(self, value): """ Deserialize ``value`` a `str` instance containing a @@ -22,7 +21,6 @@ def to_representation(self, value): return json.loads(value) return value - # pylint: disable=no-self-use def to_internal_value(self, data): """ Deserialize ``value`` a `str` instance containing a @@ -32,8 +30,8 @@ def to_internal_value(self, data): try: return json.loads(data) except ValueError as e: - # invalid json - raise serializers.ValidationError(str(e)) + # invalid JSON + raise serializers.ValidationError(str(e)) from e return data @classmethod diff --git a/onadata/libs/serializers/fields/organization_field.py b/onadata/libs/serializers/fields/organization_field.py index 50cd9932f6..37b751b06b 100644 --- a/onadata/libs/serializers/fields/organization_field.py +++ b/onadata/libs/serializers/fields/organization_field.py @@ -12,12 +12,10 @@ class OrganizationField(serializers.Field): """organization serializer field""" - # pylint: disable=no-self-use def to_representation(self, value): """Return the organization pk.""" return value.pk - # pylint: disable=no-self-use def to_internal_value(self, data): """Validate the organization exists.""" if data is not None: @@ -25,10 +23,7 @@ def to_internal_value(self, data): organization = OrganizationProfile.objects.get(pk=data) except OrganizationProfile.DoesNotExist as e: raise serializers.ValidationError( - _( - "Organization with id '%(value)s' does not exist." - % {"value": data} - ) + _(f"Organization with id '{data}' does not exist.") ) from e except ValueError as e: raise serializers.ValidationError(str(e)) from e diff --git a/onadata/libs/serializers/fields/project_field.py b/onadata/libs/serializers/fields/project_field.py index e6673ae752..4b2f973032 100644 --- a/onadata/libs/serializers/fields/project_field.py +++ b/onadata/libs/serializers/fields/project_field.py @@ -12,12 +12,10 @@ class ProjectField(serializers.Field): """Project field for use with a Project object/instance.""" - # pylint: disable=no-self-use def to_representation(self, value): """Returns the project pk.""" return value.pk - # pylint: disable=no-self-use def to_internal_value(self, data): """Validates that a project exists.""" if data is not None: diff --git a/onadata/libs/serializers/fields/team_field.py b/onadata/libs/serializers/fields/team_field.py index 18f6b110df..f73dfd1346 100644 --- a/onadata/libs/serializers/fields/team_field.py +++ b/onadata/libs/serializers/fields/team_field.py @@ -1,10 +1,19 @@ +# -*- coding: utf-8 -*- +""" +The TeamField class. +""" from rest_framework import serializers + from onadata.apps.api.models.team import Team class TeamField(serializers.Field): - def to_representation(self, obj): - return obj.pk + """ + The TeamField class. + """ + + def to_representation(self, value): + return value.pk def to_internal_value(self, data): return Team.objects.get(pk=data) diff --git a/onadata/libs/serializers/fields/utils.py b/onadata/libs/serializers/fields/utils.py index 0cf67b4736..efbce124e3 100644 --- a/onadata/libs/serializers/fields/utils.py +++ b/onadata/libs/serializers/fields/utils.py @@ -6,7 +6,7 @@ def get_object_id_by_content_type(instance, model_class): """Return instance.object_id from a cached model's content type""" - key = "{}-content_type_id".format(model_class.__name__) + key = f"{model_class.__name__}-content_type_id" content_type_id = cache.get(key) if not content_type_id: try: diff --git a/onadata/libs/serializers/fields/xform_field.py b/onadata/libs/serializers/fields/xform_field.py index 18b18a3f06..b75b9aae27 100644 --- a/onadata/libs/serializers/fields/xform_field.py +++ b/onadata/libs/serializers/fields/xform_field.py @@ -1,14 +1,23 @@ +# -*- coding: utf-8 -*- +""" +The XFormField class +""" from rest_framework import serializers + from onadata.apps.logger.models import XForm class XFormField(serializers.Field): - def to_representation(self, obj): - return obj.pk + """ + The XFormField class + """ + + def to_representation(self, value): + return value.pk def to_internal_value(self, data): try: int(data) - except ValueError: - raise serializers.ValidationError(u"Invalid form id") + except ValueError as exc: + raise serializers.ValidationError("Invalid form id") from exc return XForm.objects.get(pk=data) diff --git a/onadata/libs/serializers/fields/xform_related_field.py b/onadata/libs/serializers/fields/xform_related_field.py index c330e1c45f..cb946422b4 100644 --- a/onadata/libs/serializers/fields/xform_related_field.py +++ b/onadata/libs/serializers/fields/xform_related_field.py @@ -21,10 +21,10 @@ def get_attribute(self, instance): def to_internal_value(self, data): try: return XForm.objects.get(id=data) - except ValueError: - raise serializers.ValidationError("xform id should be an integer") - except XForm.DoesNotExist: - raise serializers.ValidationError("XForm does not exist") + except ValueError as exc: + raise serializers.ValidationError("xform id should be an integer") from exc + except XForm.DoesNotExist as exc: + raise serializers.ValidationError("XForm does not exist") from exc def to_representation(self, value): """Serialize xform object""" diff --git a/onadata/libs/serializers/floip_serializer.py b/onadata/libs/serializers/floip_serializer.py index 6dc9994b59..37497a0431 100644 --- a/onadata/libs/serializers/floip_serializer.py +++ b/onadata/libs/serializers/floip_serializer.py @@ -5,12 +5,12 @@ """ import json import os -from uuid import UUID from copy import deepcopy from io import BytesIO +from uuid import UUID from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models import Q @@ -31,6 +31,8 @@ QUESTION_INDEX = getattr(settings, "FLOW_RESULTS_QUESTION_INDEX", 4) ANSWER_INDEX = getattr(settings, "FLOW_RESULTS_ANSWER_INDEX", 5) +User = get_user_model() + def _get_user(username): users = User.objects.filter(username__iexact=username) @@ -45,7 +47,7 @@ def _get_owner(request): owner_obj = _get_user(owner) if owner_obj is None: - raise ValidationError(_("User with username %s does not exist." % owner)) + raise ValidationError(_(f"User with username {owner} does not exist.")) return owner_obj return owner @@ -70,7 +72,7 @@ def parse_responses( current_key = row[session_id_index] if "meta" not in submission: submission["meta"] = { - "instanceID": "uuid:%s" % current_key, + "instanceID": f"uuid:{current_key}", "sessionID": current_key, "contactID": row[contact_id_index], } @@ -88,8 +90,11 @@ class ReadOnlyUUIDField(serializers.ReadOnlyField): Custom ReadOnlyField for UUID """ - def to_representation(self, obj): # pylint: disable=no-self-use - return str(UUID(obj)) + def to_internal_value(self, data): + pass + + def to_representation(self, value): + return str(UUID(value)) # pylint: disable=too-many-ancestors @@ -101,12 +106,12 @@ class FloipListSerializer(serializers.HyperlinkedModelSerializer): url = serializers.HyperlinkedIdentityField( view_name="flow-results-detail", lookup_field="uuid" ) - id = ReadOnlyUUIDField(source="uuid") # pylint: disable=C0103 + id = ReadOnlyUUIDField(source="uuid") # pylint: disable=invalid-name name = serializers.ReadOnlyField(source="id_string") created = serializers.ReadOnlyField(source="date_created") modified = serializers.ReadOnlyField(source="date_modified") - class JSONAPIMeta: # pylint: disable=old-style-class,no-init,R0903 + class JSONAPIMeta: """ JSON API metaclass. """ @@ -133,7 +138,7 @@ class FloipSerializer(serializers.HyperlinkedModelSerializer): flow_results_specification_version = serializers.SerializerMethodField() resources = serializers.SerializerMethodField() - class JSONAPIMeta: # pylint: disable=old-style-class,no-init,R0903 + class JSONAPIMeta: """ JSON API metaclass. """ @@ -154,20 +159,20 @@ class Meta: "resources", ) - def get_profile(self, value): # pylint: disable=no-self-use,W0613 + def get_profile(self, value): # pylint: disable=unused-argument """ Returns the data-package profile. """ return "data-package" - # pylint: disable=no-self-use,unused-argument + # pylint: disable=unused-argument def get_flow_results_specification_version(self, value): """ Returns the flow results specification version. """ return "1.0.0-rc1" - def get_resources(self, value): # pylint: disable=no-self-use,W0613 + def get_resources(self, value): # pylint: disable=unused-argument """ Returns empty dict, a dummy holder for the eventually generated data package resources. @@ -234,7 +239,7 @@ def to_representation(self, instance): return data -class FlowResultsResponse(object): # pylint: disable=too-few-public-methods +class FlowResultsResponse: # pylint: disable=too-few-public-methods """ FLowResultsResponse class to hold a list of submission ids. """ @@ -259,7 +264,7 @@ class FlowResultsResponseSerializer(serializers.Serializer): responses = serializers.ListField() duplicates = serializers.IntegerField(read_only=True) - class JSONAPIMeta: # pylint: disable=old-style-class,no-init,R0903 + class JSONAPIMeta: """ JSON API metaclass. """ diff --git a/onadata/libs/serializers/geojson_serializer.py b/onadata/libs/serializers/geojson_serializer.py index e3e9ce9dcb..aba4133939 100644 --- a/onadata/libs/serializers/geojson_serializer.py +++ b/onadata/libs/serializers/geojson_serializer.py @@ -1,6 +1,10 @@ -import geojson +# -*- coding: utf-8 -*- +""" +The GeoJsonSerializer class - uses the GeoJSON structure for submission data. +""" import json +import geojson from rest_framework_gis import serializers from onadata.apps.logger.models.instance import Instance @@ -82,6 +86,10 @@ def geometry_from_string(points, simple_style): class GeometryField(serializers.GeometryField): + """ + The GeometryField class - representation for single GeometryField. + """ + def to_representation(self, value): if isinstance(value, dict) or value is None: return None @@ -90,6 +98,9 @@ def to_representation(self, value): class GeoJsonSerializer(serializers.GeoFeatureModelSerializer): + """ + The GeoJsonSerializer class - uses the GeoJSON structure for submission data. + """ geom = GeometryField() @@ -100,17 +111,17 @@ class Meta: id_field = False fields = ("id", "xform") - def to_representation(self, obj): - ret = super().to_representation(obj) + def to_representation(self, instance): + ret = super().to_representation(instance) request = self.context.get("request") - if obj and ret and "properties" in ret and request is not None: + if instance and ret and "properties" in ret and request is not None: fields = request.query_params.get("fields") if fields: for field in fields.split(","): - ret["properties"][field] = obj.json.get(field) + ret["properties"][field] = instance.json.get(field) - if obj and ret and request: + if instance and ret and request: fields = request.query_params.get("fields") geo_field = request.query_params.get("geo_field") simple_style = request.query_params.get("simple_style") @@ -118,11 +129,11 @@ def to_representation(self, obj): if geo_field: if "properties" in ret: if title: - ret["properties"]["title"] = obj.json.get(title) + ret["properties"]["title"] = instance.json.get(title) if fields: for field in fields.split(","): - ret["properties"][field] = obj.json.get(field) - points = obj.json.get(geo_field) + ret["properties"][field] = instance.json.get(field) + points = instance.json.get(geo_field) geometry = ( geometry_from_string(points, simple_style) if points @@ -139,30 +150,30 @@ class GeoJsonListSerializer(GeoJsonSerializer): Creates a FeatureCollections """ - def to_representation(self, obj): + def to_representation(self, instance): - if obj is None: - return super().to_representation(obj) + if instance is None: + return super().to_representation(instance) geo_field = None fields = None - if "fields" in obj and obj.get("fields"): - fields = obj.get("fields").split(",") + if "fields" in instance and instance.get("fields"): + fields = instance.get("fields").split(",") - if "instances" in obj and obj.get("instances"): - insts = obj.get("instances") + if "instances" in instance and instance.get("instances"): + insts = instance.get("instances") - if "geo_field" in obj and obj.get("geo_field"): - geo_field = obj.get("geo_field") + if "geo_field" in instance and instance.get("geo_field"): + geo_field = instance.get("geo_field") # Get the instances from the form - instances = [inst for inst in insts[0].instances.all()] + instances = insts[0].instances.all() if not geo_field: return geojson.FeatureCollection( [ super().to_representation( - {"instance": ret, "fields": obj.get("fields")} + {"instance": ret, "fields": instance.get("fields")} ) for ret in instances ] diff --git a/onadata/libs/serializers/merged_xform_serializer.py b/onadata/libs/serializers/merged_xform_serializer.py index 556d3af8c7..be57e792cb 100644 --- a/onadata/libs/serializers/merged_xform_serializer.py +++ b/onadata/libs/serializers/merged_xform_serializer.py @@ -9,10 +9,10 @@ from django.db import transaction from django.utils.translation import gettext as _ -from rest_framework import serializers from pyxform.builder import create_survey_element_from_dict from pyxform.errors import PyXFormError +from rest_framework import serializers from onadata.apps.logger.models import MergedXForm, XForm from onadata.apps.logger.models.xform import XFORM_TITLE_LENGTH @@ -50,6 +50,11 @@ def _get_elements(elements, intersect, parent_prefix=None): return new_elements +def _list_with_name(name, children): + """Returns all children where the name is the same value as ``name``""" + return list(filter(lambda x: x["name"] == name, children)) + + def get_merged_xform_survey(xforms): """ Genertates a new pyxform survey object from the intersection of fields of @@ -67,7 +72,7 @@ def get_merged_xform_survey(xforms): merged_xform_dict["children"] = [] intersect = set(xform_sets[0]).intersection(*xform_sets[1:]) - intersect = set([__ for (__, ___) in intersect]) + intersect = set(__ for (__, ___) in intersect) merged_xform_dict["children"] = _get_elements(children, intersect) @@ -87,10 +92,7 @@ def get_merged_xform_survey(xforms): if "children" in child and child["type"] in SELECTS: children = [] for xform_dict in xform_dicts: - element_name = child["name"] - element_list = list( - filter(lambda x: x["name"] == element_name, xform_dict["children"]) - ) + element_list = _list_with_name(child["name"], xform_dict["children"]) if element_list and element_list[0]: children += element_list[0]["children"] # remove duplicates @@ -194,7 +196,6 @@ class Meta: ) write_only_fields = ("uuid",) - # pylint: disable=no-self-use def get_num_of_submissions(self, obj): """Return number of submissions either from the aggregate 'number_of_submissions' in the queryset or from the xform field @@ -214,6 +215,7 @@ def get_last_submission_time(self, obj): ] if values: return sorted(values, reverse=True)[0] + return None def create(self, validated_data): request = self.context["request"] @@ -230,20 +232,20 @@ def create(self, validated_data): validated_data["xml"] = survey.to_xml() except PyXFormError as error: raise serializers.ValidationError( - {"xforms": _("Problem Merging the Form: {}".format(error))} + {"xforms": _(f"Problem Merging the Form: {error}")} ) validated_data["user"] = validated_data["project"].user validated_data["created_by"] = request.user validated_data["is_merged_dataset"] = True validated_data["num_of_submissions"] = sum( - [__.num_of_submissions for __ in validated_data.get("xforms")] + __.num_of_submissions for __ in validated_data.get("xforms") ) validated_data["instances_with_geopoints"] = any( - [__.instances_with_geopoints for __ in validated_data.get("xforms")] + __.instances_with_geopoints for __ in validated_data.get("xforms") ) with transaction.atomic(): - instance = super(MergedXFormSerializer, self).create(validated_data) + instance = super().create(validated_data) if instance.xforms.all().count() == 0 and xforms: for xform in xforms: diff --git a/onadata/libs/serializers/metadata_serializer.py b/onadata/libs/serializers/metadata_serializer.py index cf7ca3c67b..2d3e05033a 100644 --- a/onadata/libs/serializers/metadata_serializer.py +++ b/onadata/libs/serializers/metadata_serializer.py @@ -13,24 +13,24 @@ from django.db.utils import IntegrityError from django.shortcuts import get_object_or_404 from django.utils.translation import gettext as _ -from six.moves.urllib.parse import urlparse + from rest_framework import serializers from rest_framework.reverse import reverse - +from six.moves.urllib.parse import urlparse from onadata.apps.api.tools import update_role_by_meta_xform_perms -from onadata.libs.utils.api_export_tools import get_metadata_format from onadata.apps.logger.models import DataView, Instance, Project, XForm from onadata.apps.main.models import MetaData from onadata.libs.permissions import ROLES, ManagerRole -from onadata.libs.serializers.fields.json_field import JsonField from onadata.libs.serializers.fields.instance_related_field import InstanceRelatedField +from onadata.libs.serializers.fields.json_field import JsonField from onadata.libs.serializers.fields.project_related_field import ProjectRelatedField from onadata.libs.serializers.fields.xform_related_field import XFormRelatedField +from onadata.libs.utils.api_export_tools import get_metadata_format from onadata.libs.utils.common_tags import ( - XFORM_META_PERMS, - SUBMISSION_REVIEW, IMPORTED_VIA_CSV_BY, + SUBMISSION_REVIEW, + XFORM_META_PERMS, ) UNIQUE_TOGETHER_ERROR = "Object already exists" @@ -174,7 +174,7 @@ def validate(self, attrs): raise serializers.ValidationError( { "missing_field": _( - "`xform` or `project` or `instance`" "field is required." + "`xform` or `project` or `instance` field is required." ) } ) @@ -182,7 +182,7 @@ def validate(self, attrs): if data_file: allowed_types = settings.SUPPORTED_MEDIA_UPLOAD_TYPES # add geojson mimetype - mimetypes.add_type('application/geo+json', '.geojson') + mimetypes.add_type("application/geo+json", ".geojson") data_content_type = ( data_file.content_type if data_file.content_type in allowed_types @@ -219,11 +219,7 @@ def validate(self, attrs): ) if not has_perm: raise serializers.ValidationError( - { - "data_value": _( - "User has no permission to " "the dataview." - ) - } + {"data_value": _("User has no permission to the dataview.")} ) from e else: raise serializers.ValidationError( @@ -251,7 +247,6 @@ def validate(self, attrs): return attrs - # pylint: disable=no-self-use def get_content_object(self, validated_data): """ Returns the validated 'xform' or 'project' or 'instance' ids being @@ -270,7 +265,7 @@ def create(self, validated_data): data_type = validated_data.get("data_type") data_file = validated_data.get("data_file") data_file_type = validated_data.get("data_file_type") - extra_data = validated_data.get('extra_data') + extra_data = validated_data.get("extra_data") content_object = self.get_content_object(validated_data) data_value = data_file.name if data_file else validated_data.get("data_value") diff --git a/onadata/libs/serializers/monthly_submissions_serializer.py b/onadata/libs/serializers/monthly_submissions_serializer.py index 2fa779f690..4f102556a0 100644 --- a/onadata/libs/serializers/monthly_submissions_serializer.py +++ b/onadata/libs/serializers/monthly_submissions_serializer.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Monthly submissions serializer """ @@ -5,22 +6,36 @@ class MonthlySubmissionsListSerializer(serializers.ListSerializer): + """ + Monthly submissions serializer + """ + + def update(self, instance, validated_data): + pass def to_representation(self, data): - result = super(MonthlySubmissionsListSerializer, - self).to_representation(data) + result = super().to_representation(data) result_dictionary = {} for i in result: - label = 'public' if i['xform__shared'] else 'private' - result_dictionary[label] = i['num_instances'] + label = "public" if i["xform__shared"] else "private" + result_dictionary[label] = i["num_instances"] return [result_dictionary] class MonthlySubmissionsSerializer(serializers.Serializer): + """ + Monthly submissions serializer + """ class Meta: list_serializer_class = MonthlySubmissionsListSerializer + def update(self, instance, validated_data): + pass + + def create(self, validated_data): + pass + def to_representation(self, instance): """ Returns the total number of private/public submissions for a user diff --git a/onadata/libs/serializers/note_serializer.py b/onadata/libs/serializers/note_serializer.py index 8eb77c9003..b21c88c877 100644 --- a/onadata/libs/serializers/note_serializer.py +++ b/onadata/libs/serializers/note_serializer.py @@ -14,15 +14,25 @@ class NoteSerializer(serializers.ModelSerializer): """ NoteSerializer class """ + owner = serializers.SerializerMethodField() class Meta: """ Meta Options for NoteSerializer """ + model = Note - fields = ('id', 'note', 'instance', 'instance_field', 'created_by', - 'date_created', 'date_modified', 'owner') + fields = ( + "id", + "note", + "instance", + "instance_field", + "created_by", + "date_created", + "date_modified", + "owner", + ) def get_owner(self, obj): """ @@ -35,14 +45,14 @@ def get_owner(self, obj): return None def create(self, validated_data): - request = self.context.get('request') - obj = super(NoteSerializer, self).create(validated_data) + request = self.context.get("request") + obj = super().create(validated_data) if request: - assign_perm('add_note', request.user, obj) - assign_perm('change_note', request.user, obj) - assign_perm('delete_note', request.user, obj) - assign_perm('view_note', request.user, obj) + assign_perm("add_note", request.user, obj) + assign_perm("change_note", request.user, obj) + assign_perm("delete_note", request.user, obj) + assign_perm("view_note", request.user, obj) # should update instance json obj.instance.save() @@ -50,19 +60,21 @@ def create(self, validated_data): return obj def validate(self, attrs): - instance = attrs.get('instance') - request = self.context.get('request') + instance = attrs.get("instance") + request = self.context.get("request") if request and request.user.is_anonymous: raise exceptions.ParseError( - _(u"You are not authorized to add/change notes on this form.")) + _("You are not authorized to add/change notes on this form.") + ) - attrs['created_by'] = request.user + attrs["created_by"] = request.user - field = attrs.get('instance_field') + field = attrs.get("instance_field") if field and instance.xform.get_label(field) is None: raise exceptions.ValidationError( - "instance_field must be a field on the form") + "instance_field must be a field on the form" + ) return attrs diff --git a/onadata/libs/serializers/open_data_serializer.py b/onadata/libs/serializers/open_data_serializer.py index 479278991a..818acb41b9 100644 --- a/onadata/libs/serializers/open_data_serializer.py +++ b/onadata/libs/serializers/open_data_serializer.py @@ -1,45 +1,55 @@ +# -*- coding: utf-8 -*- +""" +The OpenDataSerializer class - create/list OpenData model data. +""" import collections from django.contrib.contenttypes.models import ContentType +from django.shortcuts import get_object_or_404 + from rest_framework import serializers from onadata.apps.logger.models import OpenData, XForm -from django.shortcuts import get_object_or_404 def get_data(request_data, update=False): - ''' + """ return a namedtuple with error, message and data values. - ''' - fields = ['object_id', 'data_type', 'name'] - results = collections.namedtuple('results', 'error message data') + """ + fields = ["object_id", "data_type", "name"] + results = collections.namedtuple("results", "error message data") if not update: if not set(fields).issubset(list(request_data)): return results( error=True, message="Fields object_id, data_type and name are required.", - data=None + data=None, ) - fields.append('active') + fields.append("active") # check if invalid fields are provided if any(a not in fields for a in list(request_data)): return results( error=True, message="Valid fields are object_id, data_type and name.", - data=None + data=None, ) data = {} for key in fields: available = request_data.get(key) is not None - available and data.update({key: request_data.get(key)}) + if available: + data.update({key: request_data.get(key)}) return results(error=False, message=None, data=data) class OpenDataSerializer(serializers.ModelSerializer): + """ + The OpenDataSerializer class - create/list OpenData model data. + """ + name = serializers.CharField(max_length=255, required=True) data_type = serializers.CharField(max_length=50, required=False) object_id = serializers.IntegerField(required=False) @@ -47,43 +57,41 @@ class OpenDataSerializer(serializers.ModelSerializer): class Meta: model = OpenData - exclude = ('date_created', 'date_modified', 'content_type', 'id') + exclude = ("date_created", "date_modified", "content_type", "id") def create(self, validated_data): results = get_data(validated_data) if results.error: raise serializers.ValidationError(results.message) - name = validated_data.get('name') - data_type = validated_data.get('data_type') - object_id = validated_data.get('object_id') - op = None + name = validated_data.get("name") + data_type = validated_data.get("data_type") + object_id = validated_data.get("object_id") - if data_type == 'xform': + if data_type == "xform": xform = get_object_or_404(XForm, id=object_id) - ct = ContentType.objects.get_for_model(xform) + content_type = ContentType.objects.get_for_model(xform) - op, created = OpenData.objects.get_or_create( + open_data, _created = OpenData.objects.get_or_create( object_id=object_id, defaults={ - 'name': name, - 'content_type': ct, - 'content_object': xform, - } + "name": name, + "content_type": content_type, + "content_object": xform, + }, ) - return op + return open_data + return None def update(self, instance, validated_data): results = get_data(validated_data, update=True) if results.error: raise serializers.ValidationError(results.message) - instance.name = validated_data.get('name', instance.name) - instance.object_id = validated_data.get( - 'object_id', instance.object_id - ) - instance.active = validated_data.get('active', instance.active) + instance.name = validated_data.get("name", instance.name) + instance.object_id = validated_data.get("object_id", instance.object_id) + instance.active = validated_data.get("active", instance.active) instance.save() return instance diff --git a/onadata/libs/serializers/organization_member_serializer.py b/onadata/libs/serializers/organization_member_serializer.py index 09df7131fe..3917b22cb8 100644 --- a/onadata/libs/serializers/organization_member_serializer.py +++ b/onadata/libs/serializers/organization_member_serializer.py @@ -1,36 +1,37 @@ -from django.contrib.auth.models import User -from django.utils.translation import gettext as _ +# -*- coding: utf-8 -*- +""" +The OrganizationMemberSerializer - manages a users access in an organization +""" +from django.contrib.auth import get_user_model from django.core.mail import send_mail +from django.utils.translation import gettext as _ from rest_framework import serializers -from onadata.libs.serializers.fields.organization_field import \ - OrganizationField -from onadata.libs.permissions import ROLES -from onadata.libs.permissions import OwnerRole -from onadata.libs.permissions import is_organization -from onadata.apps.api.tools import add_user_to_organization -from onadata.apps.api.tools import get_or_create_organization_owners_team -from onadata.apps.api.tools import add_user_to_team -from onadata.apps.api.tools import remove_user_from_team -from onadata.apps.api.tools import _get_owners -from onadata.apps.api.tools import get_organization_members -from onadata.apps.api.tools import remove_user_from_organization -from onadata.settings.common import (DEFAULT_FROM_EMAIL, SHARE_ORG_SUBJECT) +from onadata.apps.api.tools import ( + _get_owners, + add_user_to_organization, + add_user_to_team, + get_or_create_organization_owners_team, + get_organization_members, + remove_user_from_organization, + remove_user_from_team, +) from onadata.libs.models.share_project import ShareProject +from onadata.libs.permissions import ROLES, OwnerRole, is_organization +from onadata.libs.serializers.fields.organization_field import OrganizationField +from onadata.settings.common import DEFAULT_FROM_EMAIL, SHARE_ORG_SUBJECT + +User = get_user_model() def _compose_send_email(organization, user, email_msg, email_subject=None): if not email_subject: - email_subject = SHARE_ORG_SUBJECT.format(user.username, - organization.name) + email_subject = SHARE_ORG_SUBJECT.format(user.username, organization.name) # send out email message. - send_mail(email_subject, - email_msg, - DEFAULT_FROM_EMAIL, - (user.email, )) + send_mail(email_subject, email_msg, DEFAULT_FROM_EMAIL, (user.email,)) def _set_organization_role_to_user(organization, user, role): @@ -52,6 +53,10 @@ def _set_organization_role_to_user(organization, user, role): class OrganizationMemberSerializer(serializers.Serializer): + """ + The OrganizationMemberSerializer - manages a users access in an organization + """ + organization = OrganizationField() username = serializers.CharField(max_length=255, required=False) role = serializers.CharField(max_length=50, required=False) @@ -59,41 +64,43 @@ class OrganizationMemberSerializer(serializers.Serializer): email_subject = serializers.CharField(max_length=255, required=False) remove = serializers.BooleanField(default=False) + def update(self, instance, validated_data): + # Do nothing + pass + def validate_username(self, value): """Check that the username exists""" user = None try: user = User.objects.get(username=value) - except User.DoesNotExist: - raise serializers.ValidationError(_( - u"User '%(value)s' does not exist." % {"value": value} - )) + except User.DoesNotExist as exc: + raise serializers.ValidationError( + _(f"User '{value}' does not exist.") + ) from exc else: if not user.is_active: - raise serializers.ValidationError(_(u"User is not active")) + raise serializers.ValidationError(_("User is not active")) if is_organization(user.profile): raise serializers.ValidationError( - _(u"Cannot add org account `{}` as member." - .format(user.username))) + _(f"Cannot add org account `{user.username}` as member.") + ) return value def validate_role(self, value): """check that the role exists""" if value not in ROLES: - raise serializers.ValidationError(_( - u"Unknown role '%(role)s'." % {"role": value} - )) + raise serializers.ValidationError(_(f"Unknown role '{value}'.")) return value def validate(self, attrs): - remove = attrs.get('remove') - role = attrs.get('role') - organization = attrs.get('organization') - username = attrs.get('username') + remove = attrs.get("remove") + role = attrs.get("role") + organization = attrs.get("organization") + username = attrs.get("username") # check if roles are downgrading and the user is the last admin if username and (remove or role != OwnerRole.name): @@ -102,17 +109,18 @@ def validate(self, attrs): owners = _get_owners(organization) if user in owners and len(owners) <= 1: raise serializers.ValidationError( - _("Organization cannot be without an owner")) + _("Organization cannot be without an owner") + ) return attrs def create(self, validated_data): organization = validated_data.get("organization") username = validated_data.get("username") - role = validated_data.get('role') - email_msg = validated_data.get('email_msg') - email_subject = validated_data.get('email_subject') - remove = validated_data.get('remove') + role = validated_data.get("role") + email_msg = validated_data.get("email_msg") + email_subject = validated_data.get("email_subject") + remove = validated_data.get("remove") if username: user = User.objects.get(username=username) @@ -123,14 +131,14 @@ def create(self, validated_data): _set_organization_role_to_user(organization, user, role) if email_msg: - _compose_send_email(organization, user, email_msg, - email_subject) + _compose_send_email(organization, user, email_msg, email_subject) if remove: remove_user_from_organization(organization, user) return organization + @property def data(self): organization = self.validated_data.get("organization") members = get_organization_members(organization) diff --git a/onadata/libs/serializers/organization_serializer.py b/onadata/libs/serializers/organization_serializer.py index a000e611df..eb604f0d0b 100644 --- a/onadata/libs/serializers/organization_serializer.py +++ b/onadata/libs/serializers/organization_serializer.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Organization Serializer """ @@ -92,7 +92,7 @@ def create(self, validated_data): return profile - def validate_org(self, value): # pylint: disable=no-self-use + def validate_org(self, value): """ Validate organization name. """ @@ -116,7 +116,7 @@ def validate_org(self, value): # pylint: disable=no-self-use raise serializers.ValidationError(_(f"Organization {org} already exists.")) - def get_users(self, obj): # pylint: disable=no-self-use + def get_users(self, obj): """ Return organization members. """ diff --git a/onadata/libs/serializers/password_reset_serializer.py b/onadata/libs/serializers/password_reset_serializer.py index 01ab28888c..e014abb34f 100644 --- a/onadata/libs/serializers/password_reset_serializer.py +++ b/onadata/libs/serializers/password_reset_serializer.py @@ -176,7 +176,6 @@ class PasswordResetSerializer(serializers.Serializer): label=_("Email Subject"), required=False, max_length=78, allow_blank=True ) - # pylint: disable=no-self-use def validate_email(self, value): """ Validates the email. @@ -214,7 +213,6 @@ class PasswordResetChangeSerializer(serializers.Serializer): new_password = serializers.CharField(min_length=4, max_length=128) token = serializers.CharField(max_length=128) - # pylint: disable=no-self-use def validate_uid(self, value): """ Validate the user uid. diff --git a/onadata/libs/serializers/project_serializer.py b/onadata/libs/serializers/project_serializer.py index 24c611ac3e..29bdd404ef 100644 --- a/onadata/libs/serializers/project_serializer.py +++ b/onadata/libs/serializers/project_serializer.py @@ -2,8 +2,6 @@ """ Project Serializer module. """ -from six import itervalues - from django.conf import settings from django.contrib.auth import get_user_model from django.core.cache import cache @@ -11,13 +9,15 @@ from django.utils.translation import gettext as _ from rest_framework import serializers +from six import itervalues -from onadata.apps.api.models import OrganizationProfile -from onadata.apps.api.tools import ( +from onadata.apps.api.models.organization_profile import ( + OrganizationProfile, get_or_create_organization_owners_team, get_organization_members_team, ) -from onadata.apps.logger.models import Project, XForm +from onadata.apps.logger.models.project import Project +from onadata.apps.logger.models.xform import XForm from onadata.libs.permissions import ( ManagerRole, OwnerRole, @@ -25,10 +25,9 @@ get_role, is_organization, ) -from onadata.libs.serializers.dataview_serializer import DataViewMinimalSerializer from onadata.libs.serializers.fields.json_field import JsonField from onadata.libs.serializers.tag_list_serializer import TagListSerializer -from onadata.libs.utils.analytics import track_object_event +from onadata.libs.utils.analytics import TrackObjectEvent from onadata.libs.utils.cache_tools import ( PROJ_BASE_FORMS_CACHE, PROJ_FORMS_CACHE, @@ -250,7 +249,7 @@ class Meta: "is_merged_dataset", ) - def get_published_by_formbuilder(self, obj): # pylint: disable=no-self-use + def get_published_by_formbuilder(self, obj): """ Returns true if the form was published by formbuilder. """ @@ -346,19 +345,19 @@ def get_forms(self, obj): return forms - def get_num_datasets(self, obj): # pylint: disable=no-self-use + def get_num_datasets(self, obj): """ Return the number of datasets attached to the project. """ return get_num_datasets(obj) - def get_last_submission_date(self, obj): # pylint: disable=no-self-use + def get_last_submission_date(self, obj): """ Return the most recent submission date to any of the projects datasets. """ return get_last_submission_date(obj) - def get_teams(self, obj): # pylint: disable=no-self-use + def get_teams(self, obj): """ Return the teams with access to the project. """ @@ -448,7 +447,7 @@ def validate(self, attrs): ) return attrs - def validate_public(self, value): # pylint: disable=no-self-use + def validate_public(self, value): """ Validate the public field """ @@ -458,7 +457,7 @@ def validate_public(self, value): # pylint: disable=no-self-use ) return value - def validate_metadata(self, value): # pylint: disable=no-self-use + def validate_metadata(self, value): """ Validate metadaata is a valid JSON value. """ @@ -520,7 +519,7 @@ def update(self, instance, validated_data): return instance - @track_object_event( + @TrackObjectEvent( user_field="created_by", properties={ "created_by": "created_by", @@ -557,7 +556,7 @@ def create(self, validated_data): cache.set(f"{PROJ_OWNER_CACHE}{project.pk}", response) return project - def get_users(self, obj): # pylint: disable=no-self-use + def get_users(self, obj): """ Return a list of users and organizations that have access to the project. @@ -565,7 +564,7 @@ def get_users(self, obj): # pylint: disable=no-self-use return get_users(obj, self.context) @check_obj - def get_forms(self, obj): # pylint: disable=no-self-use + def get_forms(self, obj): """ Return list of xforms in the project. """ @@ -583,25 +582,25 @@ def get_forms(self, obj): # pylint: disable=no-self-use return forms - def get_num_datasets(self, obj): # pylint: disable=no-self-use + def get_num_datasets(self, obj): """ Return the number of datasets attached to the project. """ return get_num_datasets(obj) - def get_last_submission_date(self, obj): # pylint: disable=no-self-use + def get_last_submission_date(self, obj): """ Return the most recent submission date to any of the projects datasets. """ return get_last_submission_date(obj) - def get_starred(self, obj): # pylint: disable=no-self-use + def get_starred(self, obj): """ Return True if request user has starred this project. """ return is_starred(obj, self.context["request"]) - def get_teams(self, obj): # pylint: disable=no-self-use + def get_teams(self, obj): """ Return the teams with access to the project. """ @@ -623,6 +622,11 @@ def get_data_views(self, obj): else obj.dataview_set.filter(deleted_at__isnull=True) ) + # pylint: disable=import-outside-toplevel + from onadata.libs.serializers.dataview_serializer import ( + DataViewMinimalSerializer, + ) + serializer = DataViewMinimalSerializer( data_views_obj, many=True, context=self.context ) diff --git a/onadata/libs/serializers/restservices_serializer.py b/onadata/libs/serializers/restservices_serializer.py index c137fe95b9..a5b2aacaa2 100644 --- a/onadata/libs/serializers/restservices_serializer.py +++ b/onadata/libs/serializers/restservices_serializer.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +The RestServiceSerializer class - create, list a rest service. +""" from rest_framework import serializers from onadata.apps.logger.models import XForm @@ -5,14 +9,25 @@ class RestServiceSerializer(serializers.ModelSerializer): - id = serializers.IntegerField(source='pk', read_only=True) - xform = serializers.PrimaryKeyRelatedField( - queryset=XForm.objects.all() - ) + """ + The RestServiceSerializer class - create, list a rest service. + """ + + # pylint: disable=invalid-name + id = serializers.IntegerField(source="pk", read_only=True) + xform = serializers.PrimaryKeyRelatedField(queryset=XForm.objects.all()) name = serializers.CharField(max_length=50) service_url = serializers.URLField(required=True) class Meta: model = RestService - fields = ('id', 'xform', 'name', 'service_url', 'date_created', - 'date_modified', 'active', 'inactive_reason') + fields = ( + "id", + "xform", + "name", + "service_url", + "date_created", + "date_modified", + "active", + "inactive_reason", + ) diff --git a/onadata/libs/serializers/share_project_serializer.py b/onadata/libs/serializers/share_project_serializer.py index 342d91eea3..0e8eddf1b2 100644 --- a/onadata/libs/serializers/share_project_serializer.py +++ b/onadata/libs/serializers/share_project_serializer.py @@ -1,24 +1,34 @@ -from django.contrib.auth.models import User +# -*- coding: utf-8 -*- +""" +The ShareProjectSerializer class - support sharing a project. +""" +from django.contrib.auth import get_user_model from django.utils.translation import gettext as _ from rest_framework import serializers + from onadata.libs.models.share_project import ShareProject -from onadata.libs.permissions import get_object_users_with_permissions -from onadata.libs.permissions import ROLES -from onadata.libs.permissions import OwnerRole +from onadata.libs.permissions import ROLES, OwnerRole, get_object_users_with_permissions from onadata.libs.serializers.fields.project_field import ProjectField +User = get_user_model() + def attrs_to_instance(attrs, instance): - instance.project = attrs.get('project', instance.project) - instance.username = attrs.get('username', instance.username) - instance.role = attrs.get('role', instance.role) - instance.remove = attrs.get('remove', instance.remove) + """Apply attributes into a class object from a dict.""" + instance.project = attrs.get("project", instance.project) + instance.username = attrs.get("username", instance.username) + instance.role = attrs.get("role", instance.role) + instance.remove = attrs.get("remove", instance.remove) return instance class ShareProjectSerializer(serializers.Serializer): + """ + The ShareProjectSerializer class - support sharing a project. + """ + project = ProjectField() username = serializers.CharField(max_length=255) role = serializers.CharField(max_length=50) @@ -26,8 +36,8 @@ class ShareProjectSerializer(serializers.Serializer): def create(self, validated_data): created_instances = [] - for username in validated_data.pop('username').split(','): - validated_data['username'] = username + for username in validated_data.pop("username").split(","): + validated_data["username"] = username instance = ShareProject(**validated_data) instance.save() created_instances.append(instance) @@ -41,24 +51,26 @@ def update(self, instance, validated_data): return instance def validate(self, attrs): - for username in attrs.get('username').split(','): + for username in attrs.get("username").split(","): user = User.objects.get(username=username) - project = attrs.get('project') + project = attrs.get("project") # check if the user is the owner of the project if user and project: if user == project.organization: - raise serializers.ValidationError({ - 'username': - _(u"Cannot share project with the owner (%(value)s)" % - {"value": user.username}) - }) + raise serializers.ValidationError( + { + "username": _( + f"Cannot share project with the owner ({user.username})" + ) + } + ) return attrs def validate_username(self, value): """Check that the username exists""" - usernames = [u.strip() for u in value.split(',')] + usernames = [u.strip() for u in value.split(",")] user = None non_existent_users = [] inactive_users = [] @@ -73,31 +85,30 @@ def validate_username(self, value): inactive_users.append(username) if non_existent_users: - non_existent_users = ', '.join(non_existent_users) + non_existent_users = ", ".join(non_existent_users) raise serializers.ValidationError( - _('The following user(s) does/do not exist:' - f' {non_existent_users}' - )) + _(f"The following user(s) does/do not exist: {non_existent_users}") + ) if inactive_users: - inactive_users = ', '.join(inactive_users) + inactive_users = ", ".join(inactive_users) raise serializers.ValidationError( - _(f'The following user(s) is/are not active: {inactive_users}') + _(f"The following user(s) is/are not active: {inactive_users}") ) - return (',').join(usernames) + return (",").join(usernames) def validate_role(self, value): """check that the role exists""" if value not in ROLES: - raise serializers.ValidationError(_( - u"Unknown role '%(role)s'." % {"role": value} - )) + raise serializers.ValidationError(_(f"Unknown role '{value}'.")) return value class RemoveUserFromProjectSerializer(ShareProjectSerializer): + """RemoveUserFromProjectSerializer class - removes a user's access to a project.""" + remove = serializers.BooleanField() def update(self, instance, validated_data): @@ -113,20 +124,18 @@ def create(self, validated_data): return instance def validate(self, attrs): - """ Check and confirm that the project will be left with at least one - owner. Raises a validation error if only one owner found""" + """Check and confirm that the project will be left with at least one + owner. Raises a validation error if only one owner found""" - if attrs.get('role') == OwnerRole.name: - results = get_object_users_with_permissions(attrs.get('project')) + if attrs.get("role") == OwnerRole.name: + results = get_object_users_with_permissions(attrs.get("project")) # count all the owners - count = len( - [res for res in results if res.get('role') == OwnerRole.name] - ) + count = len([res for res in results if res.get("role") == OwnerRole.name]) if count <= 1: - raise serializers.ValidationError({ - 'remove': _(u"Project requires at least one owner") - }) + raise serializers.ValidationError( + {"remove": _("Project requires at least one owner")} + ) return attrs diff --git a/onadata/libs/serializers/share_team_project_serializer.py b/onadata/libs/serializers/share_team_project_serializer.py index 780be95304..bf0c472e63 100644 --- a/onadata/libs/serializers/share_team_project_serializer.py +++ b/onadata/libs/serializers/share_team_project_serializer.py @@ -19,7 +19,6 @@ class ShareTeamProjectSerializer(serializers.Serializer): project = ProjectField() role = serializers.CharField(max_length=50) - # pylint: disable=no-self-use def update(self, instance, validated_data): """Update project sharing properties.""" instance.team = validated_data.get("team", instance.team) @@ -29,7 +28,6 @@ def update(self, instance, validated_data): return instance - # pylint: disable=no-self-use def create(self, validated_data): """Shares a project to a team.""" instance = ShareTeamProject(**validated_data) @@ -37,7 +35,6 @@ def create(self, validated_data): return instance - # pylint: disable=no-self-use def validate_role(self, value): """check that the role exists""" @@ -52,7 +49,6 @@ class RemoveTeamFromProjectSerializer(ShareTeamProjectSerializer): remove = serializers.BooleanField() - # pylint: disable=no-self-use def update(self, instance, validated_data): """Remove a team from a project""" instance.remove = validated_data.get("remove", instance.remove) @@ -60,7 +56,6 @@ def update(self, instance, validated_data): return instance - # pylint: disable=no-self-use def create(self, validated_data): """Remove a team from a project""" instance = ShareTeamProject(**validated_data) diff --git a/onadata/libs/serializers/share_xform_serializer.py b/onadata/libs/serializers/share_xform_serializer.py index c1c0f9e9e9..d89df7826d 100644 --- a/onadata/libs/serializers/share_xform_serializer.py +++ b/onadata/libs/serializers/share_xform_serializer.py @@ -19,7 +19,7 @@ class ShareXFormSerializer(serializers.Serializer): username = serializers.CharField(max_length=255) role = serializers.CharField(max_length=50) - # pylint: disable=unused-argument,no-self-use + # pylint: disable=unused-argument def update(self, instance, validated_data): """Make changes to form share to a user.""" instance.xform = validated_data.get("xform", instance.xform) @@ -29,7 +29,7 @@ def update(self, instance, validated_data): return instance - # pylint: disable=unused-argument,no-self-use + # pylint: disable=unused-argument def create(self, validated_data): """Assign role permission for a form to a user.""" instance = ShareXForm(**validated_data) @@ -37,7 +37,6 @@ def create(self, validated_data): return instance - # pylint: disable=no-self-use def validate_username(self, value): """Check that the username exists""" # pylint: disable=invalid-name @@ -51,7 +50,6 @@ def validate_username(self, value): return value - # pylint: disable=no-self-use def validate_role(self, value): """check that the role exists""" if value not in ROLES: diff --git a/onadata/libs/serializers/stats_serializer.py b/onadata/libs/serializers/stats_serializer.py index 2db6c85f6f..36e2b394d8 100644 --- a/onadata/libs/serializers/stats_serializer.py +++ b/onadata/libs/serializers/stats_serializer.py @@ -2,27 +2,24 @@ """ Stats API endpoint serializer. """ -from django.utils.translation import gettext as _ -from django.core.cache import cache from django.conf import settings +from django.core.cache import cache +from django.utils.translation import gettext as _ -from rest_framework import exceptions -from rest_framework import serializers +from rest_framework import exceptions, serializers from rest_framework.utils.serializer_helpers import ReturnList +from onadata.apps.logger.models.xform import XForm +from onadata.libs.data.query import get_form_submissions_grouped_by_field from onadata.libs.data.statistics import ( - get_median_for_numeric_fields_in_form, + get_all_stats, get_mean_for_numeric_fields_in_form, - get_mode_for_numeric_fields_in_form, + get_median_for_numeric_fields_in_form, get_min_max_range, - get_all_stats, + get_mode_for_numeric_fields_in_form, ) -from onadata.apps.logger.models.xform import XForm -from onadata.libs.data.query import get_form_submissions_grouped_by_field - from onadata.libs.utils.cache_tools import XFORM_SUBMISSION_STAT - SELECT_FIELDS = ["select one", "select multiple"] STATS_FUNCTIONS = { @@ -61,7 +58,7 @@ def to_representation(self, instance): if field is None: raise exceptions.ParseError( - _("Expecting `group` and `name`" " query parameters.") + _("Expecting `group` and `name` query parameters.") ) cache_key = f"{XFORM_SUBMISSION_STAT}{instance.pk}{field}{name}" diff --git a/onadata/libs/serializers/submission_review_serializer.py b/onadata/libs/serializers/submission_review_serializer.py index 8ed9e0c52f..5487e2142e 100644 --- a/onadata/libs/serializers/submission_review_serializer.py +++ b/onadata/libs/serializers/submission_review_serializer.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Submission Review Serializer Module """ @@ -6,55 +7,66 @@ from rest_framework import exceptions, serializers from onadata.apps.logger.models import Note, SubmissionReview -from onadata.libs.utils.common_tags import (COMMENT_REQUIRED, - SUBMISSION_REVIEW_INSTANCE_FIELD) +from onadata.libs.utils.common_tags import ( + COMMENT_REQUIRED, + SUBMISSION_REVIEW_INSTANCE_FIELD, +) class SubmissionReviewSerializer(serializers.ModelSerializer): """ SubmissionReviewSerializer Class """ + note = serializers.CharField( - source='note.note', required=False, allow_blank=True, - allow_null=True) + source="note.note", required=False, allow_blank=True, allow_null=True + ) class Meta: """ Meta Options for SubmissionReviewSerializer """ + model = SubmissionReview - fields = ('id', 'instance', 'created_by', 'status', 'date_created', - 'note', 'date_modified') + fields = ( + "id", + "instance", + "created_by", + "status", + "date_created", + "note", + "date_modified", + ) def validate(self, attrs): """ Custom Validate Method for SubmissionReviewSerializer """ - status = attrs.get('status') - note = attrs.get('note') + status = attrs.get("status") + note = attrs.get("note") if status == SubmissionReview.REJECTED and not note: - raise exceptions.ValidationError({'note': COMMENT_REQUIRED}) + raise exceptions.ValidationError({"note": COMMENT_REQUIRED}) return attrs def create(self, validated_data): """ Custom create method for SubmissionReviewSerializer """ - request = self.context.get('request') + request = self.context.get("request") if request: - validated_data['created_by'] = request.user + validated_data["created_by"] = request.user - if 'note' in validated_data: - note_data = validated_data.pop('note') - if note_data['note']: - note_data['instance'] = validated_data.get('instance') - note_data['created_by'] = validated_data.get('created_by') - note_data['instance_field'] = SUBMISSION_REVIEW_INSTANCE_FIELD + if "note" in validated_data: + note_data = validated_data.pop("note") + if note_data["note"]: + note_data["instance"] = validated_data.get("instance") + note_data["created_by"] = validated_data.get("created_by") + note_data["instance_field"] = SUBMISSION_REVIEW_INSTANCE_FIELD note = Note.objects.create(**note_data) - validated_data['note'] = note + validated_data["note"] = note submission_review = SubmissionReview.objects.create(**validated_data) @@ -65,12 +77,12 @@ def update(self, instance, validated_data): Custom update method for SubmissionReviewSerializer """ note = instance.note - note_data = validated_data.pop('note') + note_data = validated_data.pop("note") - note.note = note_data['note'] + note.note = note_data["note"] note.save() - instance.status = validated_data.get('status', instance.status) + instance.status = validated_data.get("status", instance.status) instance.save() diff --git a/onadata/libs/serializers/team_serializer.py b/onadata/libs/serializers/team_serializer.py index 6ce5888b5b..8adf09fd61 100644 --- a/onadata/libs/serializers/team_serializer.py +++ b/onadata/libs/serializers/team_serializer.py @@ -1,31 +1,43 @@ -from django.contrib.auth.models import User +# -*- coding: utf-8 -*- +""" +The TeamSerializer class - access and update Team model objects. +""" +from django.contrib.auth import get_user_model + from rest_framework import serializers -from onadata.libs.serializers.fields.hyperlinked_multi_identity_field import\ - HyperlinkedMultiIdentityField -from onadata.libs.serializers.user_serializer import UserSerializer from onadata.apps.api.models import OrganizationProfile, Team from onadata.apps.logger.models import Project from onadata.libs.permissions import get_team_project_default_permissions +from onadata.libs.serializers.fields.hyperlinked_multi_identity_field import ( + HyperlinkedMultiIdentityField, +) +from onadata.libs.serializers.user_serializer import UserSerializer + +User = get_user_model() class TeamSerializer(serializers.Serializer): - teamid = serializers.ReadOnlyField(source='id') - url = HyperlinkedMultiIdentityField(view_name='team-detail') - name = serializers.CharField(max_length=100, source='team_name', - required=True) + """ + The TeamSerializer class - access and update Team model objects. + """ + + teamid = serializers.ReadOnlyField(source="id") + url = HyperlinkedMultiIdentityField(view_name="team-detail") + name = serializers.CharField(max_length=100, source="team_name", required=True) organization = serializers.SlugRelatedField( - slug_field='username', - queryset=User.objects.filter( - pk__in=OrganizationProfile.objects.values('user'))) + slug_field="username", + queryset=User.objects.filter(pk__in=OrganizationProfile.objects.values("user")), + ) projects = serializers.SerializerMethodField() users = serializers.SerializerMethodField() def get_users(self, obj): + """Returns a users in a team.""" users = [] if obj: - for user in obj.user_set.all(): + for user in obj.user_set.filter(is_active=True): users.append(UserSerializer(instance=user).data) return users @@ -35,23 +47,23 @@ def get_projects(self, obj): projects = [] if obj: - for project in Project.objects.filter( - organization__id=obj.organization.id): + for project in Project.objects.filter(organization__id=obj.organization.id): project_map = {} - project_map['name'] = project.name - project_map['projectid'] = project.pk - project_map['default_role'] = \ - get_team_project_default_permissions(obj, project) + project_map["name"] = project.name + project_map["projectid"] = project.pk + project_map["default_role"] = get_team_project_default_permissions( + obj, project + ) projects.append(project_map) return projects def update(self, instance, validated_data): - org = validated_data.get('organization', None) - projects = validated_data.get('projects', []) + org = validated_data.get("organization", None) + projects = validated_data.get("projects", []) instance.organization = org if org else instance.organization - instance.name = validated_data.get('team_name', instance.name) + instance.name = validated_data.get("team_name", instance.name) instance.projects.clear() for project in projects: @@ -62,9 +74,9 @@ def update(self, instance, validated_data): return instance def create(self, validated_data): - org = validated_data.get('organization', None) - team_name = validated_data.get('team_name', None) - request = self.context.get('request') + org = validated_data.get("organization", None) + team_name = validated_data.get("team_name", None) + request = self.context.get("request") created_by = request.user return Team.objects.create( diff --git a/onadata/libs/serializers/textit_serializer.py b/onadata/libs/serializers/textit_serializer.py index 6293c1cf2e..982b5a24f9 100644 --- a/onadata/libs/serializers/textit_serializer.py +++ b/onadata/libs/serializers/textit_serializer.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +The TextItSerializer - supports creating TextIt integration service. +""" from django.conf import settings from rest_framework import serializers @@ -8,7 +12,12 @@ class TextItSerializer(serializers.Serializer): - id = serializers.IntegerField(source='pk', read_only=True) + """ + The TextItSerializer - supports creating TextIt integration service. + """ + + # pylint: disable=invalid-name + id = serializers.IntegerField(source="pk", read_only=True) xform = XFormField() auth_token = serializers.CharField(max_length=255, required=True) flow_title = serializers.CharField(max_length=255, default="") @@ -23,36 +32,47 @@ class TextItSerializer(serializers.Serializer): def to_representation(self, instance): meta_data = MetaData.objects.filter( - data_type=TEXTIT_DETAILS, object_id=instance.xform.pk).first() + data_type=TEXTIT_DETAILS, object_id=instance.xform.pk + ).first() flow_title = "" if meta_data: flow_title = meta_data.data_value - text_it = TextItService(pk=instance.pk, xform=instance.xform, - service_url=instance.service_url, - name=instance.name, - flow_title=flow_title) + text_it = TextItService( + pk=instance.pk, + xform=instance.xform, + service_url=instance.service_url, + name=instance.name, + flow_title=flow_title, + ) text_it.date_modified = instance.date_modified text_it.date_created = instance.date_created text_it.active = instance.active text_it.inactive_reason = instance.inactive_reason text_it.retrieve() - return super(TextItSerializer, self).to_representation(text_it) + return super().to_representation(text_it) def update(self, instance, validated_data): - data_value = MetaData.textit(instance.xform) or '' + data_value = MetaData.textit(instance.xform) or "" values = data_value.split(settings.METADATA_SEPARATOR) if len(values) < 3: - values = ['', '', ''] - xform = validated_data.get('xform', instance.xform) - auth_token = validated_data.get('auth_token', values[0]) - flow_uuid = validated_data.get('flow_uuid', values[1]) - contacts = validated_data.get('contacts', values[2]) - name = validated_data.get('name', instance.name) - service_url = validated_data.get('service_url', instance.service_url) + values = ["", "", ""] + xform = validated_data.get("xform", instance.xform) + auth_token = validated_data.get("auth_token", values[0]) + flow_uuid = validated_data.get("flow_uuid", values[1]) + contacts = validated_data.get("contacts", values[2]) + name = validated_data.get("name", instance.name) + service_url = validated_data.get("service_url", instance.service_url) - instance = TextItService(xform, service_url, name, auth_token, - flow_uuid, contacts, instance.pk, - flow_title=validated_data.get('flow_title')) + instance = TextItService( + xform, + service_url, + name, + auth_token, + flow_uuid, + contacts, + instance.pk, + flow_title=validated_data.get("flow_title"), + ) instance.save() return instance diff --git a/onadata/libs/serializers/user_profile_serializer.py b/onadata/libs/serializers/user_profile_serializer.py index 8172a3fb6a..b31aa16535 100644 --- a/onadata/libs/serializers/user_profile_serializer.py +++ b/onadata/libs/serializers/user_profile_serializer.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ UserProfile Serializers. """ @@ -26,7 +26,7 @@ from onadata.libs.authentication import expired from onadata.libs.permissions import CAN_VIEW_PROFILE, is_organization from onadata.libs.serializers.fields.json_field import JsonField -from onadata.libs.utils.analytics import track_object_event +from onadata.libs.utils.analytics import TrackObjectEvent from onadata.libs.utils.cache_tools import IS_ORG from onadata.libs.utils.email import get_verification_email_data, get_verification_url @@ -42,7 +42,7 @@ def _get_first_last_names(name, limit=30): if not isinstance(name, six.string_types): return name, name - if name.__len__() > (limit * 2): + if len(name) > (limit * 2): # since we are using the default django User Model, there is an # imposition of 30 characters on both first_name and last_name hence # ensure we only have 30 characters for either field @@ -179,7 +179,7 @@ def __init__(self, *args, **kwargs): for field in getattr(self.Meta, "owner_only_fields"): self.fields.pop(field) - def get_is_org(self, obj): # pylint: disable=no-self-use + def get_is_org(self, obj): """ Returns True if it is an organization profile. """ @@ -255,7 +255,7 @@ def update(self, instance, validated_data): return super().update(instance, params) - @track_object_event( + @TrackObjectEvent( user_field="user", properties={"name": "name", "country": "country"} ) def create(self, validated_data): @@ -348,7 +348,7 @@ def validate_email(self, value): return value - def validate_twitter(self, value): # pylint: disable=no-self-use + def validate_twitter(self, value): """ Checks if the twitter handle is valid. """ @@ -412,14 +412,12 @@ class Meta: "temp_token", ) - # pylint: disable=no-self-use def get_api_token(self, obj): """ Returns user's API Token. """ return obj.user.auth_token.key - # pylint: disable=no-self-use def get_temp_token(self, obj): """ This should return a valid temp token for this user profile. diff --git a/onadata/libs/serializers/user_serializer.py b/onadata/libs/serializers/user_serializer.py index 9f47ec52fc..e0767fbded 100644 --- a/onadata/libs/serializers/user_serializer.py +++ b/onadata/libs/serializers/user_serializer.py @@ -1,8 +1,19 @@ -from django.contrib.auth.models import User +# -*- coding: utf-8 -*- +""" +The UserSerializer class - Users serializer +""" +from django.contrib.auth import get_user_model from rest_framework import serializers +User = get_user_model() + + class UserSerializer(serializers.HyperlinkedModelSerializer): + """ + The UserSerializer class - Users serializer + """ + class Meta: model = User - fields = ('id', 'username', 'first_name', 'last_name') + fields = ("id", "username", "first_name", "last_name") diff --git a/onadata/libs/serializers/widget_serializer.py b/onadata/libs/serializers/widget_serializer.py index 88d1ac63ae..f531153dcb 100644 --- a/onadata/libs/serializers/widget_serializer.py +++ b/onadata/libs/serializers/widget_serializer.py @@ -63,8 +63,7 @@ def to_representation(self, value): self._setup_field(self.view_name) - # pylint: disable=bad-super-call - return super(GenericRelatedField, self).to_representation(value) + return super().to_representation(value) def to_internal_value(self, data): """Verifies that ``data`` is a valid URL.""" diff --git a/onadata/libs/serializers/xform_serializer.py b/onadata/libs/serializers/xform_serializer.py index 6bcb4789e6..c6d054157f 100644 --- a/onadata/libs/serializers/xform_serializer.py +++ b/onadata/libs/serializers/xform_serializer.py @@ -166,7 +166,6 @@ class XFormMixin: XForm mixins """ - # pylint: disable=no-self-use def get_xls_available(self, obj): """ Returns True if ``obj.xls.url`` is not None, indicates XLS is present. @@ -187,7 +186,6 @@ def _get_metadata(self, obj, key): return None - # pylint: disable=no-self-use def get_users(self, obj): """ Returns a list of users based on XForm permissions. @@ -305,7 +303,6 @@ def get_data_views(self, obj): return data_views return [] - # pylint: disable=no-self-use def get_num_of_submissions(self, obj): """ Returns number of submissions. @@ -476,7 +473,6 @@ class Meta: "deleted_by", ) - # pylint: disable=no-self-use def get_metadata(self, obj): """ Returns XForn ``obj`` metadata. @@ -496,7 +492,7 @@ def get_metadata(self, obj): return xform_metadata - def validate_public_key(self, value): # pylint: disable=no-self-use + def validate_public_key(self, value): """ Checks that the given RSA public key is a valid key by trying to use the key data to create an RSA key object using the cryptography @@ -510,7 +506,7 @@ def validate_public_key(self, value): # pylint: disable=no-self-use ) from e return clean_public_key(value) - def _check_if_allowed_public(self, value): # pylint: disable=no-self-use + def _check_if_allowed_public(self, value): """ Verify that users are allowed to create public forms @@ -531,7 +527,6 @@ def validate_public(self, value): """ return self._check_if_allowed_public(value) - # pylint: disable=no-self-use def get_form_versions(self, obj): """ Returns all form versions. @@ -565,7 +560,6 @@ class XFormCreateSerializer(XFormSerializer): has_id_string_changed = serializers.SerializerMethodField() - # pylint: disable=no-self-use def get_has_id_string_changed(self, obj): """ Returns the value of ``obj.has_id_string_changed`` @@ -646,7 +640,6 @@ def get_url(self, obj): return url - # pylint: disable=no-self-use @check_obj def get_hash(self, obj): """ @@ -683,7 +676,6 @@ def get_hash(self, obj): return f"{hsh or 'md5:'}" - # pylint: disable=no-self-use @check_obj def get_filename(self, obj): """ diff --git a/onadata/libs/test_utils/md_table.py b/onadata/libs/test_utils/md_table.py index 01a4850e8b..c79de9606e 100644 --- a/onadata/libs/test_utils/md_table.py +++ b/onadata/libs/test_utils/md_table.py @@ -22,8 +22,7 @@ def _extract_array(mdtablerow): mtchstr = match.groups()[0] if re.match(r"^[\|-]+$", mtchstr): return False - else: - return [_strp_cell(c) for c in mtchstr.split("|")] + return [_strp_cell(c) for c in mtchstr.split("|")] return False @@ -37,6 +36,7 @@ def _is_null_row(r_arr): def md_table_to_ss_structure(mdstr: str) -> List[Tuple[str, List[List[str]]]]: + """Transform markdown to an ss structure""" ss_arr = [] for item in mdstr.split("\n"): arr = _extract_array(item) @@ -61,12 +61,13 @@ def md_table_to_ss_structure(mdstr: str) -> List[Tuple[str, List[List[str]]]]: def md_table_to_workbook(mdstr: str) -> Workbook: """ - Convert Markdown table string to an openpyxl.Workbook. Call wb.save() to persist. + Convert Markdown table string to an openpyxl.Workbook. Call workbook.save() to + persist. """ md_data = md_table_to_ss_structure(mdstr=mdstr) - wb = Workbook(write_only=True) + workbook = Workbook(write_only=True) for key, rows in md_data: - sheet = wb.create_sheet(title=key) - for r in rows: - sheet.append(r) - return wb + sheet = workbook.create_sheet(title=key) + for row in rows: + sheet.append(row) + return workbook diff --git a/onadata/libs/test_utils/pyxform_test_case.py b/onadata/libs/test_utils/pyxform_test_case.py index 6fd6126430..4d46c83bf9 100644 --- a/onadata/libs/test_utils/pyxform_test_case.py +++ b/onadata/libs/test_utils/pyxform_test_case.py @@ -13,20 +13,21 @@ from lxml import etree -# noinspection PyProtectedMember -from lxml.etree import _Element - from pyxform.builder import create_survey_element_from_dict from pyxform.errors import PyXFormError from pyxform.utils import NSMAP from pyxform.validators.odk_validate import ODKValidateError, check_xform from pyxform.xls2json import workbook_to_json + from onadata.libs.test_utils.md_table import md_table_to_ss_structure logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler()) logger.setLevel(logging.DEBUG) +# noinspection PyProtectedMember +_Element = etree._Element # pylint: disable=protected-access + if TYPE_CHECKING: from typing import Dict, List, Set, Tuple, Union @@ -35,11 +36,13 @@ class PyxformTestError(Exception): - pass + """Pyxform test errors exception class.""" @dataclass class MatcherContext: + """Data class to store assertion context information.""" + debug: bool nsmap_xpath: "Dict[str, str]" nsmap_subs: "NSMAPSubs" @@ -47,9 +50,10 @@ class MatcherContext: class PyxformMarkdown: - """Transform markdown formatted xlsform to a pyxform survey object""" + """Transform markdown formatted XLSForm to a pyxform survey object""" def md_to_pyxform_survey(self, md_raw, kwargs=None, autoname=True, warnings=None): + """Transform markdown formatted XLSForm to pyxform survey object.""" if kwargs is None: kwargs = {} if autoname: @@ -59,13 +63,13 @@ def md_to_pyxform_survey(self, md_raw, kwargs=None, autoname=True, warnings=None if re.match(r"^\s+#", line): # ignore lines which start with pound sign continue - elif re.match(r"^(.*)(#[^|]+)$", line): + if re.match(r"^(.*)(#[^|]+)$", line): # keep everything before the # outside of the last occurrence # of | _md.append(re.match(r"^(.*)(#[^|]+)$", line).groups()[0].strip()) else: _md.append(line.strip()) - md = "\n".join(_md) + md = "\n".join(_md) # pylint: disable=invalid-name if kwargs.get("debug"): logger.debug(md) @@ -75,8 +79,7 @@ def list_to_dicts(arr): def _row_to_dict(row): out_dict = {} - for i in range(0, len(row)): - col = row[i] + for i, col in enumerate(row): if col not in [None, ""]: out_dict[headers[i]] = col return out_dict @@ -106,12 +109,13 @@ def _ss_structure_to_pyxform_survey(ss_structure, kwargs, warnings=None): def _run_odk_validate(xml): # On Windows, NamedTemporaryFile must be opened exclusively. # So it must be explicitly created, opened, closed, and removed + # pylint: disable=consider-using-with tmp = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) tmp.close() try: - with codecs.open(tmp.name, mode="w", encoding="utf-8") as fp: - fp.write(xml) - fp.close() + with codecs.open(tmp.name, mode="w", encoding="utf-8") as file_handle: + file_handle.write(xml) + file_handle.close() check_xform(tmp.name) finally: # Clean up the temporary file @@ -137,9 +141,12 @@ def _autoname_inputs(kwargs): class PyxformTestCase(PyxformMarkdown, TestCase): + """The pyxform markdown TestCase class""" + maxDiff = None - def assertPyxformXform(self, **kwargs): + # pylint: disable=invalid-name,too-many-locals,too-many-branches,too-many-statements + def assertPyxformXform(self, **kwargs): # noqa """ PyxformTestCase.assertPyxformXform() named arguments: ----------------------------------------------------- @@ -213,12 +220,12 @@ def assertPyxformXform(self, **kwargs): survey_valid = True try: - if "md" in kwargs.keys(): + if "md" in kwargs: kwargs = self._autoname_inputs(kwargs) survey = self.md_to_pyxform_survey( kwargs.get("md"), kwargs, warnings=warnings ) - elif "ss_structure" in kwargs.keys(): + elif "ss_structure" in kwargs: kwargs = self._autoname_inputs(kwargs) survey = self._ss_structure_to_pyxform_survey( kwargs.get("ss_structure"), @@ -228,7 +235,7 @@ def assertPyxformXform(self, **kwargs): else: survey = kwargs.get("survey") - xml = survey._to_pretty_xml() + xml = survey._to_pretty_xml() # pylint: disable=protected-access root = etree.fromstring(xml.encode("utf-8")) # Ensure all namespaces are present, even if unused @@ -249,7 +256,7 @@ def assertPyxformXform(self, **kwargs): def _pull_xml_node_from_root(element_selector): _r = root.findall( - ".//n:%s" % element_selector, + f".//n:{element_selector}", namespaces={"n": "http://www.w3.org/2002/xforms"}, ) if _r: @@ -279,7 +286,7 @@ def _pull_xml_node_from_root(element_selector): + "'odk_validate_error__contains'" + " was empty:" + str(e) - ) + ) from e for v_err in odk_validate_error__contains: self.assertContains( e.args[0], v_err, msg_prefix="odk_validate_error__contains" @@ -288,22 +295,21 @@ def _pull_xml_node_from_root(element_selector): if survey_valid: def _check(keyword, verb): - verb_str = "%s__%s" % (keyword, verb) + verb_str = f"{keyword}__{verb}" - bad_kwarg = "%s_%s" % (code, verb) + bad_kwarg = f"{code}_{verb}" if bad_kwarg in kwargs: - good_kwarg = "%s__%s" % (code, verb) + good_kwarg = f"{code}__{verb}" raise SyntaxError( ( - "'%s' is not a valid parameter. " - "Use double underscores: '%s'" + f"'{bad_kwarg}' is not a valid parameter. " + f"Use double underscores: '{good_kwarg}'" ) - % (bad_kwarg, good_kwarg) ) def check_content(content, expected): if content is None: - self.fail(msg="No '{}' found in document.".format(keyword)) + self.fail(msg=f"No '{keyword}' found in document.") cstr = etree.tostring(content, encoding=str, pretty_print=True) matcher_context = MatcherContext( debug=debug, @@ -342,7 +348,7 @@ def check_content(content, expected): if "body_contains" in kwargs or "body__contains" in kwargs: raise SyntaxError( - "Invalid parameter: 'body__contains'." "Use 'xml__contains' instead" + "Invalid parameter: 'body__contains'.Use 'xml__contains' instead" ) for code in ["xml", "instance", "model", "itext"]: @@ -364,7 +370,7 @@ def check_content(content, expected): "and or optionally 'error__contains=[...]'" "\nError(s): " + "\n".join(errors) ) - elif survey_valid and expecting_invalid_survey: + if survey_valid and expecting_invalid_survey: raise PyxformTestError("Expected survey to be invalid.") search_test_kwargs = ( @@ -381,13 +387,13 @@ def check_content(content, expected): elif k.endswith("__not_contains"): assertion = self.assertNotContains else: - raise PyxformTestError("Unexpected search test kwarg: {}".format(k)) + raise PyxformTestError(f"Unexpected search test kwarg: {k}") if k.startswith("error"): joined = "\n".join(errors) elif k.startswith("warnings"): joined = "\n".join(warnings) else: - raise PyxformTestError("Unexpected search test kwarg: {}".format(k)) + raise PyxformTestError(f"Unexpected search test kwarg: {k}") for text in kwargs[k]: assertion(joined, text, msg_prefix=k) if "warnings_count" in kwargs: @@ -408,7 +414,7 @@ def _assert_contains(content, text, msg_prefix): return text_repr, real_count, msg_prefix - def assertContains(self, content, text, count=None, msg_prefix=""): + def assertContains(self, content, text, count=None, msg_prefix=""): # noqa """ FROM: django source- testcases.py @@ -424,16 +430,16 @@ def assertContains(self, content, text, count=None, msg_prefix=""): self.assertEqual( real_count, count, - msg_prefix + "Found %d instances of %s in content" - " (expected %d)" % (real_count, text_repr, count), + msg_prefix + f"Found {real_count} instances of {text_repr} in content" + f" (expected {count})", ) else: self.assertTrue( real_count != 0, - msg_prefix + "Couldn't find %s in content:\n" % text_repr + content, + msg_prefix + f"Couldn't find {text_repr + content} in content:\n", ) - def assertNotContains(self, content, text, msg_prefix=""): + def assertNotContains(self, content, text, msg_prefix=""): # noqa """ Asserts that a content indicates that some content was retrieved successfully, (i.e., the HTTP status code was as expected), and that @@ -446,7 +452,7 @@ def assertNotContains(self, content, text, msg_prefix=""): self.assertEqual( real_count, 0, - msg_prefix + "Response should not contain %s" % text_repr, + msg_prefix + f"Response should not contain {text_repr}", ) def assert_xpath_exact( @@ -459,9 +465,10 @@ def assert_xpath_exact( """ Process an assertion for xml__xpath_exact. - Compares result strings since expected strings may contain xml namespace prefixes. - To allow parsing required to compare as ETrees would require injecting namespace - declarations into the expected match strings. + Compares result strings since expected strings may contain xml namespace + prefixes. + To allow parsing required to compare as ETrees would require injecting + namespace declarations into the expected match strings. :param matcher_context: A MatcherContext dataclass. :param content: XML to be examined. @@ -502,7 +509,10 @@ def assert_xpath_count( content=content, xpath=xpath, ) - msg = f"XPath found no matches:\n{xpath}\n\nXForm content:\n{matcher_context.content_str}" + msg = ( + f"XPath found no matches:\n{xpath}\n\n" + f"XForm content:\n{matcher_context.content_str}" + ) self.assertEqual(expected, len(observed), msg=msg) @@ -523,8 +533,8 @@ def reorder_attributes(root): In utils.node, it is based on xml.dom.minidom.Element objects. See https://github.com/XLSForm/pyxform/issues/414. """ - for el in root.iter(): - attrib = el.attrib + for elem in root.iter(): + attrib = elem.attrib if len(attrib) > 1: # Sort attributes. Attributes are represented as {namespace}name # so attributes with explicit namespaces will always sort after @@ -540,20 +550,21 @@ def xpath_clean_result_strings( """ Clean XPath results: stringify, remove namespace declarations, clean up whitespace. - :param nsmap_subs: namespace replacements e.g. [('x="http://www.w3.org/2002/xforms", "")] + :param nsmap_subs: namespace replacements e.g. + [('x="http://www.w3.org/2002/xforms", "")] :param results: XPath results to clean. """ xmlex = [(" >", ">"), (" />", "/>")] subs = nsmap_subs + xmlex cleaned = set() - for x in results: - if isinstance(x, _Element): - reorder_attributes(x) - x = etree.tostring(x, encoding=str, pretty_print=True) - x = x.strip() - for s in subs: - x = x.replace(*s) - cleaned.add(x) + for result in results: + if isinstance(result, _Element): + reorder_attributes(result) + result = etree.tostring(result, encoding=str, pretty_print=True) + result = result.strip() + for sub in subs: + result = result.replace(*sub) + cleaned.add(result) return cleaned @@ -580,15 +591,14 @@ def xpath_evaluate( raise PyxformTestError(msg) from e if matcher_context.debug: if 0 == len(results): - logger.debug(f"Results for XPath: {xpath}\n" + "(No matches)" + "\n") + logger.debug("Results for XPath: %s\n(No matches)\n", xpath) else: cleaned = xpath_clean_result_strings( nsmap_subs=matcher_context.nsmap_subs, results=results ) - logger.debug(f"Results for XPath: {xpath}\n" + "\n".join(cleaned) + "\n") + logger.debug("Results for XPath: %s\n%s\n", xpath, "\n".join(cleaned)) if for_exact: return xpath_clean_result_strings( nsmap_subs=matcher_context.nsmap_subs, results=results ) - else: - return set(results) + return set(results) diff --git a/onadata/libs/tests/data/test_tools.py b/onadata/libs/tests/data/test_tools.py index b6b6f18ef0..a7acf2b2b4 100644 --- a/onadata/libs/tests/data/test_tools.py +++ b/onadata/libs/tests/data/test_tools.py @@ -6,82 +6,91 @@ from onadata.apps.logger.models.instance import Instance from onadata.apps.main.tests.test_base import TestBase -from onadata.libs.data.query import get_form_submissions_grouped_by_field,\ - get_date_fields, get_field_records +from onadata.libs.data.query import ( + get_form_submissions_grouped_by_field, + get_date_fields, + get_field_records, +) class TestTools(TestBase): - def setUp(self): - super(self.__class__, self).setUp() + super().setUp() self._create_user_and_login() self._publish_transportation_form() - @patch('django.utils.timezone.now') + @patch("django.utils.timezone.now") def test_get_form_submissions_grouped_by_field(self, mock_time): mock_time.return_value = datetime.utcnow().replace(tzinfo=utc) self._make_submissions() - count_key = 'count' - fields = ['_submission_time', '_xform_id_string'] + count_key = "count" + fields = ["_submission_time", "_xform_id_string"] count = len(self.xform.instances.all()) for field in fields: - result = get_form_submissions_grouped_by_field( - self.xform, field)[0] + result = get_form_submissions_grouped_by_field(self.xform, field)[0] self.assertEqual([field, count_key], sorted(list(result))) self.assertEqual(result[count_key], count) - @patch('onadata.apps.logger.models.instance.submission_time') - def test_get_form_submissions_grouped_by_field_datetime_to_date( - self, mock_time): + @patch("onadata.apps.logger.models.instance.submission_time") + def test_get_form_submissions_grouped_by_field_datetime_to_date(self, mock_time): now = datetime(2014, 1, 1, tzinfo=utc) - times = [now, now + timedelta(seconds=1), now + timedelta(seconds=2), - now + timedelta(seconds=3)] + times = [ + now, + now + timedelta(seconds=1), + now + timedelta(seconds=2), + now + timedelta(seconds=3), + ] mock_time.side_effect = times self._make_submissions() - for i in self.xform.instances.all().order_by('-pk'): + for i in self.xform.instances.all().order_by("-pk"): i.date_created = times.pop() i.save() - count_key = 'count' - fields = ['_submission_time'] + count_key = "count" + fields = ["_submission_time"] count = len(self.xform.instances.all()) for field in fields: - result = get_form_submissions_grouped_by_field( - self.xform, field)[0] + result = get_form_submissions_grouped_by_field(self.xform, field)[0] self.assertEqual([field, count_key], sorted(list(result))) self.assertEqual(result[field], str(now.date())) self.assertEqual(result[count_key], count) - @patch('django.utils.timezone.now') + @patch("django.utils.timezone.now") def test_get_form_submissions_two_xforms(self, mock_time): mock_time.return_value = datetime.utcnow().replace(tzinfo=utc) self._make_submissions() - self._publish_xls_file(os.path.join( - "fixtures", - "gps", "gps.xlsx")) + self._publish_xls_file(os.path.join("fixtures", "gps", "gps.xlsx")) first_xform = self.xform - self.xform = self.user.xforms.all().order_by('-pk')[0] - - self._make_submission(os.path.join( - 'onadata', 'apps', 'main', 'tests', 'fixtures', 'gps', - 'instances', 'gps_1980-01-23_20-52-08.xml')) + xform = self.user.xforms.all().order_by("-pk")[0] + + self._make_submission( + os.path.join( + "onadata", + "apps", + "main", + "tests", + "fixtures", + "gps", + "instances", + "gps_1980-01-23_20-52-08.xml", + ) + ) + + count_key = "count" + fields = ["_submission_time", "_xform_id_string"] - count_key = 'count' - fields = ['_submission_time', '_xform_id_string'] - - count = len(self.xform.instances.all()) + count = len(xform.instances.all()) for field in fields: - result = get_form_submissions_grouped_by_field( - self.xform, field)[0] + result = get_form_submissions_grouped_by_field(xform, field)[0] self.assertEqual([field, count_key], sorted(list(result))) self.assertEqual(result[count_key], count) @@ -89,46 +98,41 @@ def test_get_form_submissions_two_xforms(self, mock_time): count = len(first_xform.instances.all()) for field in fields: - result = get_form_submissions_grouped_by_field( - first_xform, field)[0] + result = get_form_submissions_grouped_by_field(first_xform, field)[0] self.assertEqual([field, count_key], sorted(list(result))) self.assertEqual(result[count_key], count) - @patch('django.utils.timezone.now') + @patch("django.utils.timezone.now") def test_get_form_submissions_xform_no_submissions(self, mock_time): mock_time.return_value = datetime.utcnow().replace(tzinfo=utc) self._make_submissions() - self._publish_xls_file(os.path.join( - "fixtures", - "gps", "gps.xlsx")) + self._publish_xls_file(os.path.join("fixtures", "gps", "gps.xlsx")) - self.xform = self.user.xforms.all().order_by('-pk')[0] + xform = self.user.xforms.all().order_by("-pk")[0] - fields = ['_submission_time', '_xform_id_string'] + fields = ["_submission_time", "_xform_id_string"] - count = len(self.xform.instances.all()) + count = len(xform.instances.all()) self.assertEqual(count, 0) for field in fields: - result = get_form_submissions_grouped_by_field( - self.xform, field) + result = get_form_submissions_grouped_by_field(xform, field) self.assertEqual(result, []) - @patch('django.utils.timezone.now') + @patch("django.utils.timezone.now") def test_get_form_submissions_grouped_by_field_sets_name(self, mock_time): mock_time.return_value = datetime.utcnow().replace(tzinfo=utc) self._make_submissions() - count_key = 'count' - fields = ['_submission_time', '_xform_id_string'] - name = '_my_name' + count_key = "count" + fields = ["_submission_time", "_xform_id_string"] + name = "_my_name" xform = self.user.xforms.all()[0] count = len(xform.instances.all()) for field in fields: - result = get_form_submissions_grouped_by_field( - xform, field, name)[0] + result = get_form_submissions_grouped_by_field(xform, field, name)[0] self.assertEqual([name, count_key], sorted(list(result))) self.assertEqual(result[count_key], count) @@ -145,49 +149,68 @@ def test_get_form_submissions_when_response_not_provided(self): # make submission that doesnt have a response for # `available_transportation_types_to_referral_facility` path = os.path.join( - self.this_directory, 'fixtures', 'transportation', - 'instances', 'transport_no_response', 'transport_no_response.xml') + self.this_directory, + "fixtures", + "transportation", + "instances", + "transport_no_response", + "transport_no_response.xml", + ) self._make_submission(path, self.user.username) self.assertEqual(Instance.objects.count(), count + 1) - field = 'transport/available_transportation_types_to_referral_facility' + field = "transport/available_transportation_types_to_referral_facility" xform = self.user.xforms.all()[0] results = get_form_submissions_grouped_by_field( - xform, field, - 'available_transportation_types_to_referral_facility') + xform, field, "available_transportation_types_to_referral_facility" + ) # we should have a similar number of aggregates as submissions as each # submission has a unique value for the field self.assertEqual(len(results), count + 1) # the count where the value is None should have a count of 1 - result = [r for r in results if - r['available_transportation_types_to_referral_facility'] - is None][0] - self.assertEqual(result['count'], 1) + result = [ + r + for r in results + if r["available_transportation_types_to_referral_facility"] is None + ][0] + self.assertEqual(result["count"], 1) def test_get_date_fields_includes_start_end(self): path = os.path.join( - os.path.dirname(__file__), "fixtures", "tutorial", "tutorial.xlsx") + os.path.dirname(__file__), "fixtures", "tutorial", "tutorial.xlsx" + ) self._publish_xls_file_and_set_xform(path) fields = get_date_fields(self.xform) expected_fields = sorted( - ['_submission_time', 'date', 'start_time', 'end_time', 'today', - 'exactly']) + ["_submission_time", "date", "start_time", "end_time", "today", "exactly"] + ) self.assertEqual(sorted(fields), expected_fields) def test_get_field_records_when_some_responses_are_empty(self): - submissions = ['1', '2', '3', 'no_age'] + submissions = ["1", "2", "3", "no_age"] path = os.path.join( - os.path.dirname(__file__), "fixtures", "tutorial", "tutorial.xlsx") + os.path.dirname(__file__), "fixtures", "tutorial", "tutorial.xlsx" + ) self._publish_xls_file_and_set_xform(path) for i in submissions: - self._make_submission(os.path.join( - 'onadata', 'apps', 'api', 'tests', 'fixtures', 'forms', - 'tutorial', 'instances', '{}.xml'.format(i))) - - field = 'age' + self._make_submission( + os.path.join( + "onadata", + "apps", + "api", + "tests", + "fixtures", + "forms", + "tutorial", + "instances", + f"{i}.xml", + ) + ) + + field = "age" records = get_field_records(field, self.xform) self.assertEqual(sorted(records), sorted([23, 23, 35])) diff --git a/onadata/libs/tests/test_renderers.py b/onadata/libs/tests/test_renderers.py index 037c6d6aba..69e75b5329 100644 --- a/onadata/libs/tests/test_renderers.py +++ b/onadata/libs/tests/test_renderers.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Test Renderer module. """ diff --git a/onadata/libs/tests/utils/test_analytics.py b/onadata/libs/tests/utils/test_analytics.py index e665cf4871..cd359dc24e 100644 --- a/onadata/libs/tests/utils/test_analytics.py +++ b/onadata/libs/tests/utils/test_analytics.py @@ -58,7 +58,7 @@ def test_submission_tracking(self): onadata.libs.utils.analytics.init_analytics() self.assertEqual(segment_mock.write_key, '123') - # Test out that the track_object_event decorator + # Test out that the TrackObjectEvent decorator # Tracks created submissions, XForms and Projects view = XFormSubmissionViewSet.as_view({ 'post': 'create', diff --git a/onadata/libs/tests/utils/test_api_export_tools.py b/onadata/libs/tests/utils/test_api_export_tools.py index 2a4514ed15..53c162ea10 100644 --- a/onadata/libs/tests/utils/test_api_export_tools.py +++ b/onadata/libs/tests/utils/test_api_export_tools.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Test api_export_tools module. """ diff --git a/onadata/libs/tests/utils/test_backup_tools.py b/onadata/libs/tests/utils/test_backup_tools.py index 4225b5b46c..a353645764 100644 --- a/onadata/libs/tests/utils/test_backup_tools.py +++ b/onadata/libs/tests/utils/test_backup_tools.py @@ -3,7 +3,8 @@ from onadata.apps.main.tests.test_base import TestBase from onadata.libs.utils.backup_tools import ( restore_backup_from_xml_file, - restore_backup_from_path) + restore_backup_from_path, +) from onadata.apps.logger.models import Instance @@ -17,14 +18,15 @@ def test_restore_from_xml_file(self): count = count_qs.count() xml_file_path = os.path.join( self.this_directory, - 'fixtures', - 'transportation', - 'backup_restore', - '2011', '07', '25', - '2011-07-25-19-05-36.xml') - num_restored = restore_backup_from_xml_file( - xml_file_path, - self.user.username) + "fixtures", + "transportation", + "backup_restore", + "2011", + "07", + "25", + "2011-07-25-19-05-36.xml", + ) + num_restored = restore_backup_from_xml_file(xml_file_path, self.user.username) self.assertEqual(num_restored, 1) self.assertEqual(count_qs.count(), count + 1) @@ -33,13 +35,14 @@ def test_restore_backup_from_path(self): count = count_qs.count() path = os.path.join( self.this_directory, - 'fixtures', - 'transportation', - 'backup_restore',) + "fixtures", + "transportation", + "backup_restore", + ) num_instances, num_restored = restore_backup_from_path( path, self.user.username, - None) + ) self.assertEqual(num_instances, 1) self.assertEqual(num_restored, 1) self.assertEqual(count_qs.count(), count + 1) diff --git a/onadata/libs/tests/utils/test_cache_tools.py b/onadata/libs/tests/utils/test_cache_tools.py index aebf48920d..8b5644a55b 100644 --- a/onadata/libs/tests/utils/test_cache_tools.py +++ b/onadata/libs/tests/utils/test_cache_tools.py @@ -2,17 +2,28 @@ """ Test onadata.libs.utils.cache_tools module. """ +from unittest import TestCase + +from django.contrib.auth import get_user_model from django.core.cache import cache -from django.contrib.auth.models import User from django.http.request import HttpRequest -from unittest import TestCase -from onadata.apps.main.models.user_profile import UserProfile from onadata.apps.logger.models.project import Project +from onadata.apps.main.models.user_profile import UserProfile +from onadata.libs.serializers.project_serializer import ProjectSerializer from onadata.libs.utils.cache_tools import ( - PROJ_PERM_CACHE, PROJ_NUM_DATASET_CACHE, PROJ_SUB_DATE_CACHE, - PROJ_FORMS_CACHE, PROJ_BASE_FORMS_CACHE, PROJ_OWNER_CACHE, - safe_key, reset_project_cache, project_cache_prefixes) + PROJ_BASE_FORMS_CACHE, + PROJ_FORMS_CACHE, + PROJ_NUM_DATASET_CACHE, + PROJ_OWNER_CACHE, + PROJ_PERM_CACHE, + PROJ_SUB_DATE_CACHE, + project_cache_prefixes, + reset_project_cache, + safe_key, +) + +User = get_user_model() class TestCacheTools(TestCase): @@ -22,75 +33,75 @@ def test_safe_key(self): """Test safe_key() function returns a hashed key""" self.assertEqual( safe_key("hello world"), - "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9") + "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", + ) def test_reset_project_cache(self): """ Test reset_project_cache() function actually resets all project cache entries """ - bob = User.objects.create(username='bob', first_name='bob') + bob = User.objects.create(username="bob", first_name="bob") UserProfile.objects.create(user=bob) project = Project.objects.create( - name='Some Project', created_by=bob, organization=bob) + name="Some Project", created_by=bob, organization=bob + ) # Set dummy values in cache for prefix in project_cache_prefixes: - cache.set(f'{prefix}{project.pk}', 'stale') + cache.set(f"{prefix}{project.pk}", "stale") request = HttpRequest() request.user = bob - request.META = { - 'SERVER_NAME': 'testserver', - 'SERVER_PORT': '80' - } - reset_project_cache(project, request) + request.META = {"SERVER_NAME": "testserver", "SERVER_PORT": "80"} + reset_project_cache(project, request, ProjectSerializer) expected_project_cache = { - 'url': f'http://testserver/api/v1/projects/{project.pk}', - 'projectid': project.pk, - 'owner': 'http://testserver/api/v1/users/bob', - 'created_by': 'http://testserver/api/v1/users/bob', - 'metadata': {}, - 'starred': False, - 'users': [{ - 'is_org': False, - 'metadata': {}, - 'first_name': 'bob', - 'last_name': '', - 'user': 'bob', - 'role': 'owner' - }], - 'forms': [], - 'public': False, - 'tags': [], - 'num_datasets': 0, - 'last_submission_date': None, - 'teams': [], - 'data_views': [], - 'name': 'Some Project', - 'deleted_at': None + "url": f"http://testserver/api/v1/projects/{project.pk}", + "projectid": project.pk, + "owner": "http://testserver/api/v1/users/bob", + "created_by": "http://testserver/api/v1/users/bob", + "metadata": {}, + "starred": False, + "users": [ + { + "is_org": False, + "metadata": {}, + "first_name": "bob", + "last_name": "", + "user": "bob", + "role": "owner", + } + ], + "forms": [], + "public": False, + "tags": [], + "num_datasets": 0, + "last_submission_date": None, + "teams": [], + "data_views": [], + "name": "Some Project", + "deleted_at": None, } self.assertEqual( - cache.get(f'{PROJ_PERM_CACHE}{project.pk}'), - expected_project_cache['users']) + cache.get(f"{PROJ_PERM_CACHE}{project.pk}"), expected_project_cache["users"] + ) self.assertEqual( - cache.get(f'{PROJ_NUM_DATASET_CACHE}{project.pk}'), - expected_project_cache['num_datasets']) + cache.get(f"{PROJ_NUM_DATASET_CACHE}{project.pk}"), + expected_project_cache["num_datasets"], + ) self.assertEqual( - cache.get(f'{PROJ_SUB_DATE_CACHE}{project.pk}'), - expected_project_cache['last_submission_date']) + cache.get(f"{PROJ_SUB_DATE_CACHE}{project.pk}"), + expected_project_cache["last_submission_date"], + ) self.assertEqual( - cache.get(f'{PROJ_FORMS_CACHE}{project.pk}'), - expected_project_cache['forms']) - self.assertEqual( - cache.get(f'{PROJ_BASE_FORMS_CACHE}{project.pk}'), - None) + cache.get(f"{PROJ_FORMS_CACHE}{project.pk}"), + expected_project_cache["forms"], + ) + self.assertEqual(cache.get(f"{PROJ_BASE_FORMS_CACHE}{project.pk}"), None) - project_cache = cache.get(f'{PROJ_OWNER_CACHE}{project.pk}') - project_cache.pop('date_created') - project_cache.pop('date_modified') - self.assertEqual( - project_cache, - expected_project_cache) + project_cache = cache.get(f"{PROJ_OWNER_CACHE}{project.pk}") + project_cache.pop("date_created") + project_cache.pop("date_modified") + self.assertEqual(project_cache, expected_project_cache) diff --git a/onadata/libs/tests/utils/test_export_tools.py b/onadata/libs/tests/utils/test_export_tools.py index 7e30c9554b..978255cc78 100644 --- a/onadata/libs/tests/utils/test_export_tools.py +++ b/onadata/libs/tests/utils/test_export_tools.py @@ -2,12 +2,11 @@ """ Test export_tools module """ -import os import json +import os import shutil import tempfile import zipfile -from builtins import open from datetime import date, datetime, timedelta from django.conf import settings @@ -16,31 +15,33 @@ from django.core.files.temp import NamedTemporaryFile from django.test.utils import override_settings from django.utils import timezone + from pyxform.builder import create_survey_from_xls from rest_framework import exceptions from rest_framework.authtoken.models import Token - from savReaderWriter import SavWriter from onadata.apps.api import tests as api_tests -from onadata.apps.api.tests.viewsets.test_abstract_viewset import ( - TestAbstractViewSet) +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.apps.api.viewsets.data_viewset import DataViewSet from onadata.apps.logger.models import Attachment, Instance, XForm from onadata.apps.main.tests.test_base import TestBase from onadata.apps.viewer.models.export import Export from onadata.apps.viewer.models.parsed_instance import query_data -from onadata.apps.api.viewsets.data_viewset import DataViewSet from onadata.libs.serializers.merged_xform_serializer import MergedXFormSerializer from onadata.libs.serializers.xform_serializer import XFormSerializer -from onadata.libs.utils.export_builder import encode_if_str, get_value_or_attachment_uri -from onadata.libs.utils.export_tools import ( +from onadata.libs.utils.export_builder import ( ExportBuilder, + encode_if_str, + get_value_or_attachment_uri, +) +from onadata.libs.utils.export_tools import ( check_pending_export, generate_attachments_zip_export, generate_export, + generate_geojson_export, generate_kml_export, generate_osm_export, - generate_geojson_export, get_repeat_index_tags, kml_export_data, parse_request_export_options, @@ -556,8 +557,8 @@ def test_geojson_exports(self): xform1 = self._publish_markdown(geo_md, self.user, id_string="a") xml = '-1.28 36.83 0 0orange' Instance(xform=xform1, xml=xml).save() - request = self.factory.get('/', **self.extra) - XFormSerializer(xform1, context={'request': request}).data + request = self.factory.get("/", **self.extra) + XFormSerializer(xform1, context={"request": request}).data xform1 = XForm.objects.get(id_string="a") export_type = "geojson" options = { @@ -566,15 +567,16 @@ def test_geojson_exports(self): self._publish_transportation_form_and_submit_instance() # set metadata to xform data_type = "media" - data_value = 'xform_geojson {} {}'.format(xform1.pk, xform1.id_string) + data_value = "xform_geojson {} {}".format(xform1.pk, xform1.id_string) extra_data = { "data_title": "fruit", "data_geo_field": "gps", "data_simple_style": True, - "data_fields": "fruit,gps" + "data_fields": "fruit,gps", } response = self._add_form_metadata( - self.xform, data_type, data_value, extra_data=extra_data) + self.xform, data_type, data_value, extra_data=extra_data + ) self.assertEqual(response.status_code, 201) username = self.xform.user.username id_string = self.xform.id_string @@ -582,36 +584,25 @@ def test_geojson_exports(self): self.assertEqual(self.xform.metadata_set.count(), 1) metadata = self.xform.metadata_set.all()[0] export = generate_geojson_export( - export_type, - username, - id_string, - metadata, - options=options, - xform=xform1 + export_type, username, id_string, metadata, options=options, xform=xform1 ) self.assertIsNotNone(export) self.assertTrue(export.is_successful) with default_storage.open(export.filepath) as f2: - content = f2.read().decode('utf-8') + content = f2.read().decode("utf-8") geojson = { "type": "FeatureCollection", "features": [ { "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [ - 36.83, - -1.28 - ] - }, + "geometry": {"type": "Point", "coordinates": [36.83, -1.28]}, "properties": { "fruit": "orange", "gps": "-1.28 36.83 0 0", - "title": "orange" - } + "title": "orange", + }, } - ] + ], } content = json.loads(content) # remove xform and id from properties because they keep changing @@ -630,7 +621,7 @@ def test_geojson_exports(self): metadata, export_id=export_id, options=options, - xform=xform1 + xform=xform1, ) self.assertIsNotNone(export) diff --git a/onadata/libs/tests/utils/test_project_utils.py b/onadata/libs/tests/utils/test_project_utils.py index 92462e53f1..6b343dabff 100644 --- a/onadata/libs/tests/utils/test_project_utils.py +++ b/onadata/libs/tests/utils/test_project_utils.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Test onadata.libs.utils.project_utils """ diff --git a/onadata/libs/utils/analytics.py b/onadata/libs/utils/analytics.py index b58e96cfdc..eb4fec4bfd 100644 --- a/onadata/libs/utils/analytics.py +++ b/onadata/libs/utils/analytics.py @@ -1,30 +1,35 @@ # -*- coding: utf-8 -*- -# Analytics package for tracking and measuring with services like Segment. -# Heavily borrowed from RapidPro's temba.utils.analytics -import analytics as segment_analytics +""" +Analytics package for tracking and measuring with services like Segment. +""" from typing import Dict, List, Optional from django.conf import settings -from django.contrib.auth.models import User +from django.contrib.auth import get_user_model from django.utils import timezone -from onadata.apps.logger.models import Instance, XForm, Project +# Heavily borrowed from RapidPro's temba.utils.analytics +import analytics as segment_analytics + +from onadata.apps.logger.models import Instance, Project, XForm from onadata.apps.main.models import UserProfile from onadata.libs.utils.common_tags import ( INSTANCE_CREATE_EVENT, INSTANCE_UPDATE_EVENT, - XFORM_CREATION_EVENT, PROJECT_CREATION_EVENT, - USER_CREATION_EVENT) + USER_CREATION_EVENT, + XFORM_CREATION_EVENT, +) -_segment = False +_segment = False # pylint: disable=invalid-name +User = get_user_model() def init_analytics(): """Initialize the analytics agents with write credentials.""" - segment_write_key = getattr(settings, 'SEGMENT_WRITE_KEY', None) + segment_write_key = getattr(settings, "SEGMENT_WRITE_KEY", None) if segment_write_key: - global _segment + global _segment # pylint: disable=global-statement,invalid-name segment_analytics.write_key = segment_write_key _segment = True @@ -34,10 +39,10 @@ def get_user_id(user): if user: return user.username - return 'anonymous' + return "anonymous" -class track_object_event(object): +class TrackObjectEvent: # pylint: disable=invalid-name,too-many-instance-attributes """ Decorator that helps track create and update actions for model objects. @@ -47,13 +52,15 @@ class track_object_event(object): precise control of what is tracked utilize the track() function """ + # pylint: disable=too-many-arguments def __init__( - self, - user_field: str, - properties: Dict[str, str], - event_name: str = '', - event_label: str = '', - additional_context: Dict[str, str] = None): + self, + user_field: str, + properties: Dict[str, str], + event_name: str = "", + event_label: str = "", + additional_context: Dict[str, str] = None, + ): self.user_field = user_field self.properties = properties self.event_start = None @@ -71,31 +78,37 @@ def _get_field_from_path(self, field_path: List[str]): return value def set_user(self) -> Optional[User]: - if '__' in self.user_field: - field_path = self.user_field.split('__') + """Set's the user attribute.""" + # pylint: disable=attribute-defined-outside-init + if "__" in self.user_field: + field_path = self.user_field.split("__") self.user = self._get_field_from_path(field_path) else: self.user = self._getattr_or_none(self.user_field) def get_tracking_properties(self, label: str = None) -> dict: + """Returns tracking properties""" tracking_properties = {} for tracking_property, model_field in self.properties.items(): - if '__' in model_field: - field_path = model_field.split('__') - tracking_properties[tracking_property] = self. \ - _get_field_from_path(field_path) + if "__" in model_field: + field_path = model_field.split("__") + tracking_properties[tracking_property] = self._get_field_from_path( + field_path + ) else: - tracking_properties[tracking_property] = self. \ - _getattr_or_none(model_field) + tracking_properties[tracking_property] = self._getattr_or_none( + model_field + ) if self.additional_context: tracking_properties.update(self.additional_context) - if label and 'label' not in tracking_properties: - tracking_properties['label'] = label + if label and "label" not in tracking_properties: + tracking_properties["label"] = label return tracking_properties def get_event_name(self) -> str: + """Returns an event name.""" event_name = self.event_name if isinstance(self.tracked_obj, Instance) and not event_name: last_edited = self.tracked_obj.last_edited @@ -112,6 +125,7 @@ def get_event_name(self) -> str: return event_name def get_event_label(self) -> str: + """Returns an event label.""" event_label = self.event_label if isinstance(self.tracked_obj, Instance) and not event_label: form_id = self.tracked_obj.xform.pk @@ -120,41 +134,46 @@ def get_event_label(self) -> str: return event_label def get_request_origin(self, request, tracking_properties): + """Returns the request origin""" if isinstance(self.tracked_obj, Instance): try: - user_agent = request.META['HTTP_USER_AGENT'] - if 'Android' in user_agent: - event_source = 'Submission collected from ODK COLLECT' - elif 'Chrome' or 'Mozilla' or 'Safari' in user_agent: - event_source = 'Submission collected from Enketo' + user_agent = request.META["HTTP_USER_AGENT"] + if "Android" in user_agent: + event_source = "Submission collected from ODK COLLECT" + elif ( + "Chrome" in user_agent + or "Mozilla" in user_agent + or "Safari" in user_agent + ): + event_source = "Submission collected from Enketo" except KeyError: event_source = "" else: event_source = "" - tracking_properties.update({'from': event_source}) + tracking_properties.update({"from": event_source}) return tracking_properties def _track_object_event(self, obj, request=None) -> None: + # pylint: disable=attribute-defined-outside-init self.tracked_obj = obj self.set_user() event_name = self.get_event_name() label = self.get_event_label() tracking_properties = self.get_tracking_properties(label=label) try: - if tracking_properties['from'] == 'XML Submissions': + if tracking_properties["from"] == "XML Submissions": tracking_properties = self.get_request_origin( - request, tracking_properties) + request, tracking_properties + ) except KeyError: pass - track( - self.user, event_name, - properties=tracking_properties, request=request) + track(self.user, event_name, properties=tracking_properties, request=request) def __call__(self, func): def decorator(obj, *args): request = None if hasattr(obj, "context"): - request = obj.context.get('request') + request = obj.context.get("request") self.event_start = timezone.now() return_value = func(obj, *args) if isinstance(return_value, list): @@ -163,6 +182,7 @@ def decorator(obj, *args): else: self._track_object_event(return_value, request) return return_value + return decorator @@ -173,28 +193,28 @@ def track(user, event_name, properties=None, context=None, request=None): properties = properties or {} context = context or {} # Introduce inner page and campaign object within the context - context['page'] = {} - context['campaign'] = {} + context["page"] = {} + context["campaign"] = {} - if 'value' not in properties: - properties['value'] = 1 + if "value" not in properties: + properties["value"] = 1 - if 'submitted_by' in properties: - submitted_by_user = properties.pop('submitted_by') + if "submitted_by" in properties: + submitted_by_user = properties.pop("submitted_by") submitted_by = get_user_id(submitted_by_user) - properties['event_by'] = submitted_by + properties["event_by"] = submitted_by - context['active'] = True + context["active"] = True if request: - context['ip'] = request.META.get('REMOTE_ADDR', '') - context['userId'] = user.id - context['receivedAt'] = request.headers.get('Date', '') - context['userAgent'] = request.headers.get('User-Agent', '') - context['campaign']['source'] = settings.HOSTNAME - context['page']['path'] = request.path - context['page']['referrer'] = request.headers.get('Referer', '') - context['page']['url'] = request.build_absolute_uri() + context["ip"] = request.META.get("REMOTE_ADDR", "") + context["userId"] = user.id + context["receivedAt"] = request.headers.get("Date", "") + context["userAgent"] = request.headers.get("User-Agent", "") + context["campaign"]["source"] = settings.HOSTNAME + context["page"]["path"] = request.path + context["page"]["referrer"] = request.headers.get("Referer", "") + context["page"]["url"] = request.build_absolute_uri() if _segment: segment_analytics.track(user_id, event_name, properties, context) diff --git a/onadata/libs/utils/api_export_tools.py b/onadata/libs/utils/api_export_tools.py index 719c64eb3f..b59cb78205 100644 --- a/onadata/libs/utils/api_export_tools.py +++ b/onadata/libs/utils/api_export_tools.py @@ -1,24 +1,25 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ -API export util functions. +API export utility functions. """ import json import os import sys from datetime import datetime -import six -from celery.backends.rpc import BacklogLimitExceeded -from celery.result import AsyncResult from django.conf import settings from django.http import Http404, HttpResponseRedirect from django.shortcuts import get_object_or_404 from django.utils.translation import gettext as _ + +import six +from celery.backends.rpc import BacklogLimitExceeded +from celery.result import AsyncResult +from google.oauth2.credentials import Credentials # noqa from kombu.exceptions import OperationalError from rest_framework import exceptions, status from rest_framework.response import Response from rest_framework.reverse import reverse -from google.oauth2.credentials import Credentials try: from savReaderWriter import SPSSIOError @@ -36,7 +37,6 @@ ) from onadata.libs.permissions import filter_queryset_xform_meta_perms_sql from onadata.libs.utils import log -from onadata.libs.utils.google import create_flow from onadata.libs.utils.async_status import ( FAILED, PENDING, @@ -56,13 +56,14 @@ generate_attachments_zip_export, generate_export, generate_external_export, + generate_geojson_export, generate_kml_export, generate_osm_export, - generate_geojson_export, newest_export_for, parse_request_export_options, should_create_new_export, ) +from onadata.libs.utils.google import create_flow from onadata.libs.utils.logger_tools import response_with_mimetype_and_name from onadata.libs.utils.model_tools import get_columns_with_hxl @@ -188,8 +189,12 @@ def custom_response_handler( # noqa: C0901 # we always re-generate if a filter is specified def _new_export(): return _generate_new_export( - request, xform, query, export_type, - dataview_pk=dataview_pk, metadata=metadata + request, + xform, + query, + export_type, + dataview_pk=dataview_pk, + metadata=metadata, ) if should_create_new_export(xform, export_type, options, request=request): diff --git a/onadata/libs/utils/backup_tools.py b/onadata/libs/utils/backup_tools.py index 801a71dc07..6d62312c8f 100644 --- a/onadata/libs/utils/backup_tools.py +++ b/onadata/libs/utils/backup_tools.py @@ -1,11 +1,15 @@ +# -*- coding: utf-8 -*- +""" +Backup utilities. +""" import codecs -from datetime import datetime import errno import os import shutil import sys import tempfile import zipfile +from datetime import datetime from time import sleep from onadata.apps.logger.import_tools import django_file @@ -13,25 +17,27 @@ from onadata.libs.utils.logger_tools import create_instance from onadata.libs.utils.model_tools import queryset_iterator - DATE_FORMAT = "%Y-%m-%d-%H-%M-%S" def _date_created_from_filename(filename): - base_name, ext = os.path.splitext(filename) + base_name, _ext = os.path.splitext(filename) parts = base_name.split("-") if len(parts) < 6: raise ValueError( - "Inavlid filename - it must be in the form" - " 'YYYY-MM-DD-HH-MM-SS[-i].xml'") - parts_dict = dict( - zip(["year", "month", "day", "hour", "min", "sec"], parts)) + "Inavlid filename - it must be in the form 'YYYY-MM-DD-HH-MM-SS[-i].xml'" + ) + parts_dict = dict(zip(["year", "month", "day", "hour", "min", "sec"], parts)) + # pylint: disable=consider-using-f-string return datetime.strptime( - "%(year)s-%(month)s-%(day)s-%(hour)s-%(min)s-%(sec)s" % - parts_dict, DATE_FORMAT) + "%(year)s-%(month)s-%(day)s-%(hour)s-%(min)s-%(sec)s" % parts_dict, + DATE_FORMAT, + ) +# pylint: disable=too-many-locals def create_zip_backup(zip_output_file, user, xform=None): + """Create a ZIP file with a user's XForms and submissions.""" # create a temp dir that we'll create our structure within and zip it # when we are done tmp_dir_path = tempfile.mkdtemp() @@ -43,14 +49,14 @@ def create_zip_backup(zip_output_file, user, xform=None): # for each submission in the database - create an xml file in this # form # //YYYY/MM/DD/YYYY-MM-DD-HH-MM-SS.xml - qs = Instance.objects.filter(xform__user=user) + queryset = Instance.objects.filter(xform__user=user) if xform: - qs = qs.filter(xform=xform) + queryset = queryset.filter(xform=xform) - num_instances = qs.count() + num_instances = queryset.count() done = 0 sys.stdout.write("Creating XML Instances\n") - for instance in queryset_iterator(qs, 100): + for instance in queryset_iterator(queryset, 100): # get submission time date_time_str = instance.date_created.strftime(DATE_FORMAT) date_parts = date_time_str.split("-") @@ -68,96 +74,95 @@ def create_zip_backup(zip_output_file, user, xform=None): # check for duplicate file names file_index = 1 while os.path.exists(full_xml_path): - full_xml_path = os.path.join( - full_path, "%s-%d.xml" % (date_time_str, file_index)) + full_xml_path = os.path.join(full_path, f"{date_time_str}-{file_index}.xml") file_index += 1 # create the instance xml with codecs.open(full_xml_path, "wb", "utf-8") as f: f.write(instance.xml) done += 1 - sys.stdout.write("\r%.2f %% done" % ( - float(done)/float(num_instances) * 100)) + # pylint: disable=consider-using-f-string + sys.stdout.write("\r%.2f %% done" % (float(done) / float(num_instances) * 100)) sys.stdout.flush() sleep(0) # write zip file - sys.stdout.write("\nWriting to ZIP arhive.\n") - zf = zipfile.ZipFile(zip_output_file, "w", zipfile.ZIP_DEFLATED, - allowZip64=True) - done = 0 - for dir_path, dir_names, file_names in os.walk(tmp_dir_path): - for file_name in file_names: - archive_path = dir_path.replace(tmp_dir_path + os.path.sep, - "", 1) - zf.write(os.path.join(dir_path, file_name), - os.path.join(archive_path, file_name)) - done += 1 - sys.stdout.write("\r%.2f %% done" % ( - float(done)/float(num_instances) * 100)) - sys.stdout.flush() - sleep(0) - zf.close() + sys.stdout.write("\nWriting to ZIP archive.\n") + with zipfile.ZipFile( + zip_output_file, "w", zipfile.ZIP_DEFLATED, allowZip64=True + ) as zip_file: + done = 0 + for dir_path, _dir_names, file_names in os.walk(tmp_dir_path): + for file_name in file_names: + archive_path = dir_path.replace(tmp_dir_path + os.path.sep, "", 1) + zip_file.write( + os.path.join(dir_path, file_name), + os.path.join(archive_path, file_name), + ) + done += 1 + # pylint: disable=consider-using-f-string + sys.stdout.write( + "\r%.2f %% done" % (float(done) / float(num_instances) * 100) + ) + sys.stdout.flush() + sleep(0) # removed dir tree shutil.rmtree(tmp_dir_path) - sys.stdout.write("\nBackup saved to %s\n" % zip_output_file) + sys.stdout.write(f"\nBackup saved to {zip_output_file}\n") def restore_backup_from_zip(zip_file_path, username): + """Restores XForms and submission instances from a ZIP backup file.""" try: temp_directory = tempfile.mkdtemp() - zf = zipfile.ZipFile(zip_file_path) - - zf.extractall(temp_directory) + with zipfile.ZipFile(zip_file_path) as zip_file: + zip_file.extractall(temp_directory) except zipfile.BadZipfile: - sys.stderr.write("Bad zip arhcive.") + sys.stderr.write("Bad zip archive.") else: - return restore_backup_from_path(temp_directory, username, "backup") + return restore_backup_from_path(temp_directory, username) finally: shutil.rmtree(temp_directory) + return None def restore_backup_from_xml_file(xml_instance_path, username): + """Creates submission instances in the DB from a submission XML file.""" # check if its a valid xml instance file_name = os.path.basename(xml_instance_path) xml_file = django_file( - xml_instance_path, - field_name="xml_file", - content_type="text/xml") + xml_instance_path, field_name="xml_file", content_type="text/xml" + ) media_files = [] try: date_created = _date_created_from_filename(file_name) except ValueError: sys.stderr.write( - "Couldn't determine date created from filename: '%s'\n" % - file_name) + f"Couldn't determine date created from filename: '{file_name}'\n" + ) date_created = datetime.now() - sys.stdout.write("Creating instance from '%s'\n" % file_name) + sys.stdout.write(f"Creating instance from '{file_name}'\n") try: create_instance( - username, xml_file, media_files, - date_created_override=date_created) + username, xml_file, media_files, date_created_override=date_created + ) return 1 - except Exception as e: - sys.stderr.write( - "Could not restore %s, create instance said: %s\n" % - (file_name, e)) + except Exception as e: # pylint: disable=broad-except + sys.stderr.write(f"Could not restore {file_name}, create instance said: {e}\n") return 0 -def restore_backup_from_path(dir_path, username, status): +def restore_backup_from_path(dir_path, username): """ Only restores xml submissions, media files are assumed to still be in storage and will be retrieved by the filename stored within the submission """ num_instances = 0 num_restored = 0 - for dir_path, dir_names, file_names in os.walk(dir_path): + for _dir_path, _dir_names, file_names in os.walk(dir_path): for file_name in file_names: # check if its a valid xml instance - xml_instance_path = os.path.join(dir_path, file_name) + xml_instance_path = os.path.join(_dir_path, file_name) num_instances += 1 - num_restored += restore_backup_from_xml_file( - xml_instance_path, - username) + num_restored += restore_backup_from_xml_file(xml_instance_path, username) return num_instances, num_restored diff --git a/onadata/libs/utils/cache_tools.py b/onadata/libs/utils/cache_tools.py index 4698bf7dcc..033b37c6f5 100644 --- a/onadata/libs/utils/cache_tools.py +++ b/onadata/libs/utils/cache_tools.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Cache utilities. +""" import hashlib from django.core.cache import cache @@ -10,9 +14,14 @@ PROJ_FORMS_CACHE = "ps-project_forms-" PROJ_BASE_FORMS_CACHE = "ps-project_base_forms-" PROJ_OWNER_CACHE = "ps-project_owner-" -project_cache_prefixes = [PROJ_PERM_CACHE, PROJ_NUM_DATASET_CACHE, - PROJ_SUB_DATE_CACHE, PROJ_FORMS_CACHE, - PROJ_BASE_FORMS_CACHE, PROJ_OWNER_CACHE] +project_cache_prefixes = [ + PROJ_PERM_CACHE, + PROJ_NUM_DATASET_CACHE, + PROJ_SUB_DATE_CACHE, + PROJ_FORMS_CACHE, + PROJ_BASE_FORMS_CACHE, + PROJ_OWNER_CACHE, +] # Cache names used in user_profile_serializer IS_ORG = "ups-is_org-" @@ -33,13 +42,13 @@ PROJECT_LINKED_DATAVIEWS = "ps-project-linked_dataviews" # Cache names used in organization profile viewset -ORG_PROFILE_CACHE = 'org-profile-' +ORG_PROFILE_CACHE = "org-profile-" # cache login attempts LOCKOUT_IP = "lockout_ip-" LOGIN_ATTEMPTS = "login_attempts-" -LOCKOUT_CHANGE_PASSWORD_USER = 'lockout_change_password_user-' -CHANGE_PASSWORD_ATTEMPTS = 'change_password_attempts-' +LOCKOUT_CHANGE_PASSWORD_USER = "lockout_change_password_user-" # noqa +CHANGE_PASSWORD_ATTEMPTS = "change_password_attempts-" # noqa # Cache names used in XForm Model XFORM_SUBMISSION_COUNT_FOR_DAY = "xfm-get_submission_count-" @@ -58,18 +67,18 @@ def safe_key(key): return hashlib.sha256(force_bytes(key)).hexdigest() -def reset_project_cache(project, request): +def reset_project_cache(project, request, project_serializer_class): """ Clears and sets project cache """ - from onadata.libs.serializers.project_serializer import ProjectSerializer # Clear all project cache entries for prefix in project_cache_prefixes: - safe_delete(f'{prefix}{project.pk}') + safe_delete(f"{prefix}{project.pk}") # Reserialize project and cache value # Note: The ProjectSerializer sets all the other cache entries - project_cache_data = ProjectSerializer( - project, context={'request': request}).data - cache.set(f'{PROJ_OWNER_CACHE}{project.pk}', project_cache_data) + project_cache_data = project_serializer_class( + project, context={"request": request} + ).data + cache.set(f"{PROJ_OWNER_CACHE}{project.pk}", project_cache_data) diff --git a/onadata/libs/utils/chart_tools.py b/onadata/libs/utils/chart_tools.py index a26796c627..f5683e9a0f 100644 --- a/onadata/libs/utils/chart_tools.py +++ b/onadata/libs/utils/chart_tools.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Chart utility functions. """ diff --git a/onadata/libs/utils/common_tools.py b/onadata/libs/utils/common_tools.py index 9ae8533276..e5a9e051a1 100644 --- a/onadata/libs/utils/common_tools.py +++ b/onadata/libs/utils/common_tools.py @@ -19,7 +19,11 @@ import sentry_sdk import six +from celery import current_task +from onadata.libs.utils.common_tags import ATTACHMENTS + +DEFAULT_UPDATE_BATCH = 100 TRUE_VALUES = ["TRUE", "T", "1", 1] @@ -123,7 +127,7 @@ def json_stream(data, json_string): """ yield "[" try: - data = data.__iter__() + data = iter(data) item = next(data) while item: try: @@ -231,3 +235,132 @@ def __ne__(self, other): return mycmp(self.obj, other.obj) != 0 return ComparatorClass + + +def current_site_url(path): + """ + Returns fully qualified URL (no trailing slash) for the current site. + :param path + :return: complete url + """ + # pylint: disable=import-outside-toplevel + from django.contrib.sites.models import Site + + current_site = Site.objects.get_current() + protocol = getattr(settings, "ONA_SITE_PROTOCOL", "http") + port = getattr(settings, "ONA_SITE_PORT", "") + url = f"{protocol}://{current_site.domain}" + if port: + url += f":{port}" + if path: + url += f"{path}" + + return url + + +def get_choice_label(label, data_dictionary, language=None): + """ + Return the label matching selected language or simply just the label. + """ + if isinstance(label, dict): + languages = list(label.keys()) + _language = ( + language + if language in languages + else data_dictionary.get_language(languages) + ) + + return label[_language] + + return label + + +def get_choice_label_value(key, value, data_dictionary, language=None): + """ + Return the label of a choice matching the value if the key xpath is a + SELECT_ONE otherwise it returns the value unchanged. + """ + + def _get_choice_label_value(lookup): + _label = None + for choice in data_dictionary.get_survey_element(key).children: + if choice.name == lookup: + _label = get_choice_label(choice.label, data_dictionary, language) + break + + return _label + + label = None + if key in data_dictionary.get_select_one_xpaths(): + label = _get_choice_label_value(value) + + if key in data_dictionary.get_select_multiple_xpaths(): + answers = [] + for item in value.split(" "): + answer = _get_choice_label_value(item) + answers.append(answer or item) + if [_i for _i in answers if _i is not None]: + label = " ".join(answers) + + return label or value + + +# pylint: disable=too-many-arguments +def get_value_or_attachment_uri( + key, + value, + row, + data_dictionary, + media_xpaths, + attachment_list=None, + show_choice_labels=False, + language=None, +): + """ + Gets either the attachment value or the attachment url + :param key: used to retrieve survey element + :param value: filename + :param row: current records row + :param data_dictionary: form structure + :param include_images: boolean value to either inlcude images or not + :param attachment_list: to be used incase row doesn't have ATTACHMENTS key + :return: value + """ + if show_choice_labels: + value = get_choice_label_value(key, value, data_dictionary, language) + + if not media_xpaths: + return value + + if key in media_xpaths: + attachments = [ + a + for a in row.get(ATTACHMENTS, attachment_list or []) + if a.get("name") == value + ] + if attachments: + value = current_site_url(attachments[0].get("download_url", "")) + + return value + + +def track_task_progress(additions, total=None): + """ + Updates the current export task with number of submission processed. + Updates in batches of settings EXPORT_TASK_PROGRESS_UPDATE_BATCH defaults + to 100. + :param additions: + :param total: + :return: + """ + batch_size = getattr( + settings, "EXPORT_TASK_PROGRESS_UPDATE_BATCH", DEFAULT_UPDATE_BATCH + ) + if additions % batch_size == 0: + meta = {"progress": additions} + if total: + meta.update({"total": total}) + try: + current_task.update_state(state="PROGRESS", meta=meta) + except AttributeError: + pass diff --git a/onadata/libs/utils/country_field.py b/onadata/libs/utils/country_field.py index 6974a6cf23..3ebe612af8 100644 --- a/onadata/libs/utils/country_field.py +++ b/onadata/libs/utils/country_field.py @@ -1,268 +1,273 @@ +# -*- coding: utf-8 -*- +""" +CountryField - provides a list of countries. +""" from django.db import models from django.utils.translation import gettext_lazy as _ # http://www.unece.org/cefact/locode/service/location.html COUNTRIES = ( - ('AF', _('Afghanistan')), - ('AL', _('Albania')), - ('DZ', _('Algeria')), - ('AS', _('American Samoa')), - ('AD', _('Andorra')), - ('AO', _('Angola')), - ('AI', _('Anguilla')), - ('AQ', _('Antarctica')), - ('AG', _('Antigua and Barbuda')), - ('AR', _('Argentina')), - ('AM', _('Armenia')), - ('AW', _('Aruba')), - ('AU', _('Australia')), - ('AT', _('Austria')), - ('AZ', _('Azerbaijan')), - ('BS', _('Bahamas')), - ('BH', _('Bahrain')), - ('BD', _('Bangladesh')), - ('BB', _('Barbados')), - ('BY', _('Belarus')), - ('BE', _('Belgium')), - ('BZ', _('Belize')), - ('BJ', _('Benin')), - ('BM', _('Bermuda')), - ('BT', _('Bhutan')), - ('BO', _('Bolivia')), - ('BQ', _('Bonaire, Sint Eustatius and Saba')), - ('BA', _('Bosnia and Herzegovina')), - ('BW', _('Botswana')), - ('BR', _('Brazil')), - ('IO', _('British Indian Ocean Territory')), - ('BN', _('Brunei Darussalam')), - ('BG', _('Bulgaria')), - ('BF', _('Burkina Faso')), - ('BI', _('Burundi')), - ('KH', _('Cambodia')), - ('CM', _('Cameroon')), - ('CA', _('Canada')), - ('CV', _('Cape Verde')), - ('KY', _('Cayman Islands')), - ('CF', _('Central African Republic')), - ('TD', _('Chad')), - ('CL', _('Chile')), - ('CN', _('China')), - ('CX', _('Christmas Island')), - ('CC', _('Cocos (Keeling) Islands')), - ('CO', _('Colombia')), - ('KM', _('Comoros')), - ('CG', _('Congo')), - ('CD', _('Congo, The Democratic Republic of the')), - ('CK', _('Cook Islands')), - ('CR', _('Costa Rica')), - ('CI', _('Ivory Coast')), - ('HR', _('Croatia')), - ('CU', _('Cuba')), - ('CW', _('Curacao')), - ('CY', _('Cyprus')), - ('CZ', _('Czech Republic')), - ('DK', _('Denmark')), - ('DJ', _('Djibouti')), - ('DM', _('Dominica')), - ('DO', _('Dominican Republic')), - ('EC', _('Ecuador')), - ('EG', _('Egypt')), - ('SV', _('El Salvador')), - ('GQ', _('Equatorial Guinea')), - ('ER', _('Eritrea')), - ('EE', _('Estonia')), - ('ET', _('Ethiopia')), - ('FK', _('Falkland Islands (Malvinas)')), - ('FO', _('Faroe Islands')), - ('FJ', _('Fiji')), - ('FI', _('Finland')), - ('FR', _('France')), - ('GF', _('French Guiana')), - ('PF', _('French Polynesia')), - ('TF', _('French Southern Territories')), - ('GA', _('Gabon')), - ('GM', _('Gambia')), - ('GE', _('Georgia')), - ('DE', _('Germany')), - ('GH', _('Ghana')), - ('GI', _('Gibraltar')), - ('GR', _('Greece')), - ('GL', _('Greenland')), - ('GD', _('Grenada')), - ('GP', _('Guadeloupe')), - ('GU', _('Guam')), - ('GT', _('Guatemala')), - ('GG', _('Guernsey')), - ('GN', _('Guinea')), - ('GW', _('Guinea-Bissau')), - ('GY', _('Guyana')), - ('HT', _('Haiti')), - ('HM', _('Heard Island and McDonald Islands')), - ('VA', _('Holy See (Vatican City State)')), - ('HN', _('Honduras')), - ('HK', _('Hong Kong')), - ('HU', _('Hungary')), - ('IS', _('Iceland')), - ('IN', _('India')), - ('ID', _('Indonesia')), - ('XZ', _('Installations in International Waters')), - ('IR', _('Iran, Islamic Republic of')), - ('IQ', _('Iraq')), - ('IE', _('Ireland')), - ('IM', _('Isle of Man')), - ('IL', _('Israel')), - ('IT', _('Italy')), - ('JM', _('Jamaica')), - ('JP', _('Japan')), - ('JE', _('Jersey')), - ('JO', _('Jordan')), - ('KZ', _('Kazakhstan')), - ('KE', _('Kenya')), - ('KI', _('Kiribati')), - ('KP', _('Korea, Democratic People\'s Republic of')), - ('KR', _('Korea, Republic of')), + ("AF", _("Afghanistan")), + ("AL", _("Albania")), + ("DZ", _("Algeria")), + ("AS", _("American Samoa")), + ("AD", _("Andorra")), + ("AO", _("Angola")), + ("AI", _("Anguilla")), + ("AQ", _("Antarctica")), + ("AG", _("Antigua and Barbuda")), + ("AR", _("Argentina")), + ("AM", _("Armenia")), + ("AW", _("Aruba")), + ("AU", _("Australia")), + ("AT", _("Austria")), + ("AZ", _("Azerbaijan")), + ("BS", _("Bahamas")), + ("BH", _("Bahrain")), + ("BD", _("Bangladesh")), + ("BB", _("Barbados")), + ("BY", _("Belarus")), + ("BE", _("Belgium")), + ("BZ", _("Belize")), + ("BJ", _("Benin")), + ("BM", _("Bermuda")), + ("BT", _("Bhutan")), + ("BO", _("Bolivia")), + ("BQ", _("Bonaire, Sint Eustatius and Saba")), + ("BA", _("Bosnia and Herzegovina")), + ("BW", _("Botswana")), + ("BR", _("Brazil")), + ("IO", _("British Indian Ocean Territory")), + ("BN", _("Brunei Darussalam")), + ("BG", _("Bulgaria")), + ("BF", _("Burkina Faso")), + ("BI", _("Burundi")), + ("KH", _("Cambodia")), + ("CM", _("Cameroon")), + ("CA", _("Canada")), + ("CV", _("Cape Verde")), + ("KY", _("Cayman Islands")), + ("CF", _("Central African Republic")), + ("TD", _("Chad")), + ("CL", _("Chile")), + ("CN", _("China")), + ("CX", _("Christmas Island")), + ("CC", _("Cocos (Keeling) Islands")), + ("CO", _("Colombia")), + ("KM", _("Comoros")), + ("CG", _("Congo")), + ("CD", _("Congo, The Democratic Republic of the")), + ("CK", _("Cook Islands")), + ("CR", _("Costa Rica")), + ("CI", _("Ivory Coast")), + ("HR", _("Croatia")), + ("CU", _("Cuba")), + ("CW", _("Curacao")), + ("CY", _("Cyprus")), + ("CZ", _("Czech Republic")), + ("DK", _("Denmark")), + ("DJ", _("Djibouti")), + ("DM", _("Dominica")), + ("DO", _("Dominican Republic")), + ("EC", _("Ecuador")), + ("EG", _("Egypt")), + ("SV", _("El Salvador")), + ("GQ", _("Equatorial Guinea")), + ("ER", _("Eritrea")), + ("EE", _("Estonia")), + ("ET", _("Ethiopia")), + ("FK", _("Falkland Islands (Malvinas)")), + ("FO", _("Faroe Islands")), + ("FJ", _("Fiji")), + ("FI", _("Finland")), + ("FR", _("France")), + ("GF", _("French Guiana")), + ("PF", _("French Polynesia")), + ("TF", _("French Southern Territories")), + ("GA", _("Gabon")), + ("GM", _("Gambia")), + ("GE", _("Georgia")), + ("DE", _("Germany")), + ("GH", _("Ghana")), + ("GI", _("Gibraltar")), + ("GR", _("Greece")), + ("GL", _("Greenland")), + ("GD", _("Grenada")), + ("GP", _("Guadeloupe")), + ("GU", _("Guam")), + ("GT", _("Guatemala")), + ("GG", _("Guernsey")), + ("GN", _("Guinea")), + ("GW", _("Guinea-Bissau")), + ("GY", _("Guyana")), + ("HT", _("Haiti")), + ("HM", _("Heard Island and McDonald Islands")), + ("VA", _("Holy See (Vatican City State)")), + ("HN", _("Honduras")), + ("HK", _("Hong Kong")), + ("HU", _("Hungary")), + ("IS", _("Iceland")), + ("IN", _("India")), + ("ID", _("Indonesia")), + ("XZ", _("Installations in International Waters")), + ("IR", _("Iran, Islamic Republic of")), + ("IQ", _("Iraq")), + ("IE", _("Ireland")), + ("IM", _("Isle of Man")), + ("IL", _("Israel")), + ("IT", _("Italy")), + ("JM", _("Jamaica")), + ("JP", _("Japan")), + ("JE", _("Jersey")), + ("JO", _("Jordan")), + ("KZ", _("Kazakhstan")), + ("KE", _("Kenya")), + ("KI", _("Kiribati")), + ("KP", _("Korea, Democratic People's Republic of")), + ("KR", _("Korea, Republic of")), # see http://geonames.wordpress.com/2010/03/08/xk-country-code-for-kosovo/ - ('XK', _('Kosovo')), - ('KW', _('Kuwait')), - ('KG', _('Kyrgyzstan')), - ('LA', _('Lao People\'s Democratic Republic')), - ('LV', _('Latvia')), - ('LB', _('Lebanon')), - ('LS', _('Lesotho')), - ('LR', _('Liberia')), - ('LY', _('Libyan Arab Jamahiriya')), - ('LI', _('Liechtenstein')), - ('LT', _('Lithuania')), - ('LU', _('Luxembourg')), - ('MO', _('Macao')), - ('MK', _('Macedonia, The former Yugoslav Republic of')), - ('MG', _('Madagascar')), - ('MW', _('Malawi')), - ('MY', _('Malaysia')), - ('MV', _('Maldives')), - ('ML', _('Mali')), - ('MT', _('Malta')), - ('MH', _('Marshall Islands')), - ('MQ', _('Martinique')), - ('MR', _('Mauritania')), - ('MU', _('Mauritius')), - ('YT', _('Mayotte')), - ('MX', _('Mexico')), - ('FM', _('Micronesia, Federated States of')), - ('MD', _('Moldova, Republic of')), - ('MC', _('Monaco')), - ('MN', _('Mongolia')), - ('ME', _('Montenegro')), - ('MS', _('Montserrat')), - ('MA', _('Morocco')), - ('MZ', _('Mozambique')), - ('MM', _('Myanmar')), - ('NA', _('Namibia')), - ('NR', _('Nauru')), - ('NP', _('Nepal')), - ('NL', _('Netherlands')), - ('NC', _('New Caledonia')), - ('NZ', _('New Zealand')), - ('NI', _('Nicaragua')), - ('NE', _('Niger')), - ('NG', _('Nigeria')), - ('NU', _('Niue')), - ('NF', _('Norfolk Island')), - ('MP', _('Northern Mariana Islands')), - ('NO', _('Norway')), - ('OM', _('Oman')), - ('PK', _('Pakistan')), - ('PW', _('Palau')), - ('PS', _('Palestinian Territory, Occupied')), - ('PA', _('Panama')), - ('PG', _('Papua New Guinea')), - ('PY', _('Paraguay')), - ('PE', _('Peru')), - ('PH', _('Philippines')), - ('PN', _('Pitcairn')), - ('PL', _('Poland')), - ('PT', _('Portugal')), - ('PR', _('Puerto Rico')), - ('QA', _('Qatar')), - ('RE', _('Reunion')), - ('RO', _('Romania')), - ('RU', _('Russian Federation')), - ('RW', _('Rwanda')), - ('SH', _('Saint Helena')), - ('KN', _('Saint Kitts and Nevis')), - ('LC', _('Saint Lucia')), - ('PM', _('Saint Pierre and Miquelon')), - ('VC', _('Saint Vincent and the Grenadines')), - ('WS', _('Samoa')), - ('SM', _('San Marino')), - ('ST', _('Sao Tome and Principe')), - ('SA', _('Saudi Arabia')), - ('SN', _('Senegal')), - ('RS', _('Serbia')), - ('SC', _('Seychelles')), - ('SL', _('Sierra Leone')), - ('SG', _('Singapore')), - ('SX', _('Sint Maarten (Dutch Part)')), - ('SK', _('Slovakia')), - ('SI', _('Slovenia')), - ('SB', _('Solomon Islands')), - ('SO', _('Somalia')), - ('ZA', _('South Africa')), - ('GS', _('South Georgia and the South Sandwich Islands')), - ('SS', _('South Sudan')), - ('ES', _('Spain')), - ('LK', _('Sri Lanka')), - ('SD', _('Sudan')), - ('SR', _('Suriname')), - ('SJ', _('Svalbard and Jan Mayen')), - ('SZ', _('Swaziland')), - ('SE', _('Sweden')), - ('CH', _('Switzerland')), - ('SY', _('Syrian Arab Republic')), - ('TW', _('Taiwan, Province of China')), - ('TJ', _('Tajikistan')), - ('TZ', _('Tanzania, United Republic of')), - ('TH', _('Thailand')), - ('TL', _('Timor-Leste')), - ('TG', _('Togo')), - ('TK', _('Tokelau')), - ('TO', _('Tonga')), - ('TT', _('Trinidad and Tobago')), - ('TN', _('Tunisia')), - ('TR', _('Turkey')), - ('TM', _('Turkmenistan')), - ('TC', _('Turks and Caicos Islands')), - ('TV', _('Tuvalu')), - ('UG', _('Uganda')), - ('UA', _('Ukraine')), - ('AE', _('United Arab Emirates')), - ('GB', _('United Kingdom')), - ('US', _('United States')), - ('UM', _('United States Minor Outlying Islands')), - ('UY', _('Uruguay')), - ('UZ', _('Uzbekistan')), - ('VU', _('Vanuatu')), - ('VE', _('Venezuela')), - ('VN', _('Viet Nam')), - ('VG', _('Virgin Islands, British')), - ('VI', _('Virgin Islands, U.S.')), - ('WF', _('Wallis and Futuna')), - ('EH', _('Western Sahara')), - ('YE', _('Yemen')), - ('ZM', _('Zambia')), - ('ZW', _('Zimbabwe')), - ('ZZ', _('Unknown or unspecified country')), + ("XK", _("Kosovo")), + ("KW", _("Kuwait")), + ("KG", _("Kyrgyzstan")), + ("LA", _("Lao People's Democratic Republic")), + ("LV", _("Latvia")), + ("LB", _("Lebanon")), + ("LS", _("Lesotho")), + ("LR", _("Liberia")), + ("LY", _("Libyan Arab Jamahiriya")), + ("LI", _("Liechtenstein")), + ("LT", _("Lithuania")), + ("LU", _("Luxembourg")), + ("MO", _("Macao")), + ("MK", _("Macedonia, The former Yugoslav Republic of")), + ("MG", _("Madagascar")), + ("MW", _("Malawi")), + ("MY", _("Malaysia")), + ("MV", _("Maldives")), + ("ML", _("Mali")), + ("MT", _("Malta")), + ("MH", _("Marshall Islands")), + ("MQ", _("Martinique")), + ("MR", _("Mauritania")), + ("MU", _("Mauritius")), + ("YT", _("Mayotte")), + ("MX", _("Mexico")), + ("FM", _("Micronesia, Federated States of")), + ("MD", _("Moldova, Republic of")), + ("MC", _("Monaco")), + ("MN", _("Mongolia")), + ("ME", _("Montenegro")), + ("MS", _("Montserrat")), + ("MA", _("Morocco")), + ("MZ", _("Mozambique")), + ("MM", _("Myanmar")), + ("NA", _("Namibia")), + ("NR", _("Nauru")), + ("NP", _("Nepal")), + ("NL", _("Netherlands")), + ("NC", _("New Caledonia")), + ("NZ", _("New Zealand")), + ("NI", _("Nicaragua")), + ("NE", _("Niger")), + ("NG", _("Nigeria")), + ("NU", _("Niue")), + ("NF", _("Norfolk Island")), + ("MP", _("Northern Mariana Islands")), + ("NO", _("Norway")), + ("OM", _("Oman")), + ("PK", _("Pakistan")), + ("PW", _("Palau")), + ("PS", _("Palestinian Territory, Occupied")), + ("PA", _("Panama")), + ("PG", _("Papua New Guinea")), + ("PY", _("Paraguay")), + ("PE", _("Peru")), + ("PH", _("Philippines")), + ("PN", _("Pitcairn")), + ("PL", _("Poland")), + ("PT", _("Portugal")), + ("PR", _("Puerto Rico")), + ("QA", _("Qatar")), + ("RE", _("Reunion")), + ("RO", _("Romania")), + ("RU", _("Russian Federation")), + ("RW", _("Rwanda")), + ("SH", _("Saint Helena")), + ("KN", _("Saint Kitts and Nevis")), + ("LC", _("Saint Lucia")), + ("PM", _("Saint Pierre and Miquelon")), + ("VC", _("Saint Vincent and the Grenadines")), + ("WS", _("Samoa")), + ("SM", _("San Marino")), + ("ST", _("Sao Tome and Principe")), + ("SA", _("Saudi Arabia")), + ("SN", _("Senegal")), + ("RS", _("Serbia")), + ("SC", _("Seychelles")), + ("SL", _("Sierra Leone")), + ("SG", _("Singapore")), + ("SX", _("Sint Maarten (Dutch Part)")), + ("SK", _("Slovakia")), + ("SI", _("Slovenia")), + ("SB", _("Solomon Islands")), + ("SO", _("Somalia")), + ("ZA", _("South Africa")), + ("GS", _("South Georgia and the South Sandwich Islands")), + ("SS", _("South Sudan")), + ("ES", _("Spain")), + ("LK", _("Sri Lanka")), + ("SD", _("Sudan")), + ("SR", _("Suriname")), + ("SJ", _("Svalbard and Jan Mayen")), + ("SZ", _("Swaziland")), + ("SE", _("Sweden")), + ("CH", _("Switzerland")), + ("SY", _("Syrian Arab Republic")), + ("TW", _("Taiwan, Province of China")), + ("TJ", _("Tajikistan")), + ("TZ", _("Tanzania, United Republic of")), + ("TH", _("Thailand")), + ("TL", _("Timor-Leste")), + ("TG", _("Togo")), + ("TK", _("Tokelau")), + ("TO", _("Tonga")), + ("TT", _("Trinidad and Tobago")), + ("TN", _("Tunisia")), + ("TR", _("Turkey")), + ("TM", _("Turkmenistan")), + ("TC", _("Turks and Caicos Islands")), + ("TV", _("Tuvalu")), + ("UG", _("Uganda")), + ("UA", _("Ukraine")), + ("AE", _("United Arab Emirates")), + ("GB", _("United Kingdom")), + ("US", _("United States")), + ("UM", _("United States Minor Outlying Islands")), + ("UY", _("Uruguay")), + ("UZ", _("Uzbekistan")), + ("VU", _("Vanuatu")), + ("VE", _("Venezuela")), + ("VN", _("Viet Nam")), + ("VG", _("Virgin Islands, British")), + ("VI", _("Virgin Islands, U.S.")), + ("WF", _("Wallis and Futuna")), + ("EH", _("Western Sahara")), + ("YE", _("Yemen")), + ("ZM", _("Zambia")), + ("ZW", _("Zimbabwe")), + ("ZZ", _("Unknown or unspecified country")), ) class CountryField(models.CharField): + """A CharField that limits the choices to country codes.""" def __init__(self, *args, **kwargs): - kwargs.setdefault('maxlength', 2) - kwargs.setdefault('choices', COUNTRIES) + kwargs.setdefault("maxlength", 2) + kwargs.setdefault("choices", COUNTRIES) - super(CountryField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def get_internal_type(self): return "CharField" diff --git a/onadata/libs/utils/csv_builder.py b/onadata/libs/utils/csv_builder.py index 6ea7293454..a39025b18a 100644 --- a/onadata/libs/utils/csv_builder.py +++ b/onadata/libs/utils/csv_builder.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ CSV export utility functions. """ @@ -9,11 +9,10 @@ from django.db.models.query import QuerySet from django.utils.translation import gettext as _ -from six import iteritems - import unicodecsv as csv from pyxform.question import Question from pyxform.section import RepeatingSection, Section +from six import iteritems from onadata.apps.logger.models import OsmData from onadata.apps.logger.models.xform import XForm, question_types_to_exclude @@ -47,12 +46,12 @@ VERSION, XFORM_ID_STRING, ) -from onadata.libs.utils.export_builder import ( +from onadata.libs.utils.common_tools import ( get_choice_label, get_value_or_attachment_uri, + str_to_bool, track_task_progress, ) -from onadata.libs.utils.export_tools import str_to_bool from onadata.libs.utils.model_tools import get_columns_with_hxl # the bind type of select multiples that we use to compare @@ -465,9 +464,6 @@ def _split_gps_fields(cls, record, gps_fields): gps_parts = {xpath: None for xpath in gps_xpaths} # hack, check if its a list and grab the object within that parts = value.split(" ") - # TODO: check whether or not we can have a gps recording - # from ODKCollect that has less than four components, - # for now we are assuming that this is not the case. if len(parts) == 4: gps_parts = dict(zip(gps_xpaths, parts)) updated_gps_fields.update(gps_parts) @@ -512,8 +508,6 @@ def _query_data( "fields": fields, "start": self.start, "end": self.end, - # TODO: we might want to add this in for the user - # to sepcify a sort order "sort": "id", "start_index": start, "limit": limit, diff --git a/onadata/libs/utils/csv_import.py b/onadata/libs/utils/csv_import.py index b369592ea5..f75c3d5386 100644 --- a/onadata/libs/utils/csv_import.py +++ b/onadata/libs/utils/csv_import.py @@ -29,7 +29,7 @@ from onadata.apps.logger.models import Instance, XForm from onadata.apps.messaging.constants import SUBMISSION_DELETED, XFORM from onadata.apps.messaging.serializers import send_message -from onadata.celery import app +from onadata.celeryapp import app from onadata.libs.serializers.metadata_serializer import MetaDataSerializer from onadata.libs.utils import analytics from onadata.libs.utils.async_status import FAILED, async_status, celery_state_to_status diff --git a/onadata/libs/utils/csv_reader.py b/onadata/libs/utils/csv_reader.py index 7f3153c6c8..4c20086a74 100644 --- a/onadata/libs/utils/csv_reader.py +++ b/onadata/libs/utils/csv_reader.py @@ -1,9 +1,11 @@ -# vim: ai ts=4 sts=4 et sw=4 encoding=utf-8 - +# vim: ai ts=4 sts=4 et sw=4 fileencoding=utf-8 +""" +CsvReader class module. +""" import csv -class CsvReader(object): +class CsvReader: """ Typical usage:: @@ -17,16 +19,19 @@ def __init__(self, path): self.open(path) def open(self, path): - self._file = open(path, 'rU') # universal new-line mode + """Opens a file handle sets a CSV reader.""" + # pylint: disable=consider-using-with,unspecified-encoding + self._file = open(path, "rU") # universal new-line mode # http://stackoverflow.com/questions/904041/reading-a-utf8-csv-file-wit # h-python/904085#904085 self._csv_reader = csv.reader(self._file) def close(self): + """Closes the file handle.""" self._file.close() def __iter__(self): - return self + return iter(self) def next(self): """ @@ -35,12 +40,14 @@ def next(self): of data. """ row = self._csv_reader.next() - return [cell for cell in row] + return list(row) def _set_headers(self): + # pylint: disable=attribute-defined-outside-init self._headers = self.next() def iter_dicts(self): + """Iterate over CSV rows as dict items.""" self._set_headers() for row in self: result = {} diff --git a/onadata/libs/utils/dict_tools.py b/onadata/libs/utils/dict_tools.py index a348bf26aa..fe796c6632 100644 --- a/onadata/libs/utils/dict_tools.py +++ b/onadata/libs/utils/dict_tools.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Dict utility functions module. """ diff --git a/onadata/libs/utils/export_builder.py b/onadata/libs/utils/export_builder.py index 5084047231..91d2e17866 100644 --- a/onadata/libs/utils/export_builder.py +++ b/onadata/libs/utils/export_builder.py @@ -12,19 +12,14 @@ from zipfile import ZIP_DEFLATED, ZipFile from django.conf import settings -from django.contrib.sites.models import Site from django.core.files.temp import NamedTemporaryFile -from six import iteritems - -from celery import current_task from openpyxl.utils.datetime import to_excel from openpyxl.workbook import Workbook from pyxform.question import Question from pyxform.section import RepeatingSection, Section - - from savReaderWriter import SavWriter +from six import iteritems from onadata.apps.logger.models.osmdata import OsmData from onadata.apps.logger.models.xform import ( @@ -60,123 +55,23 @@ VERSION, XFORM_ID_STRING, ) -from onadata.libs.utils.common_tools import str_to_bool +from onadata.libs.utils.common_tools import ( + get_choice_label, + get_choice_label_value, + get_value_or_attachment_uri, + str_to_bool, + track_task_progress, +) from onadata.libs.utils.mongo import _decode_from_mongo, _is_invalid_for_mongo # the bind type of select multiples that we use to compare GEOPOINT_BIND_TYPE = "geopoint" OSM_BIND_TYPE = "osm" -DEFAULT_UPDATE_BATCH = 100 YES = 1 NO = 0 -def current_site_url(path): - """ - Returns fully qualified URL (no trailing slash) for the current site. - :param path - :return: complete url - """ - - current_site = Site.objects.get_current() - protocol = getattr(settings, "ONA_SITE_PROTOCOL", "http") - port = getattr(settings, "ONA_SITE_PORT", "") - url = f"{protocol}://{current_site.domain}" - if port: - url += f":{port}" - if path: - url += f"{path}" - - return url - - -def get_choice_label(label, data_dictionary, language=None): - """ - Return the label matching selected language or simply just the label. - """ - if isinstance(label, dict): - languages = list(label.keys()) - _language = ( - language - if language in languages - else data_dictionary.get_language(languages) - ) - - return label[_language] - - return label - - -def get_choice_label_value(key, value, data_dictionary, language=None): - """ - Return the label of a choice matching the value if the key xpath is a - SELECT_ONE otherwise it returns the value unchanged. - """ - - def _get_choice_label_value(lookup): - _label = None - for choice in data_dictionary.get_survey_element(key).children: - if choice.name == lookup: - _label = get_choice_label(choice.label, data_dictionary, language) - break - - return _label - - label = None - if key in data_dictionary.get_select_one_xpaths(): - label = _get_choice_label_value(value) - - if key in data_dictionary.get_select_multiple_xpaths(): - answers = [] - for item in value.split(" "): - answer = _get_choice_label_value(item) - answers.append(answer or item) - if [_i for _i in answers if _i is not None]: - label = " ".join(answers) - - return label or value - - -# pylint: disable=too-many-arguments -def get_value_or_attachment_uri( - key, - value, - row, - data_dictionary, - media_xpaths, - attachment_list=None, - show_choice_labels=False, - language=None, -): - """ - Gets either the attachment value or the attachment url - :param key: used to retrieve survey element - :param value: filename - :param row: current records row - :param data_dictionary: form structure - :param include_images: boolean value to either inlcude images or not - :param attachment_list: to be used incase row doesn't have ATTACHMENTS key - :return: value - """ - if show_choice_labels: - value = get_choice_label_value(key, value, data_dictionary, language) - - if not media_xpaths: - return value - - if key in media_xpaths: - attachments = [ - a - for a in row.get(ATTACHMENTS, attachment_list or []) - if a.get("name") == value - ] - if attachments: - value = current_site_url(attachments[0].get("download_url", "")) - - return value - - def get_data_dictionary_from_survey(survey): """Creates a DataDictionary instance from an XML survey instance.""" data_dicionary = DataDictionary() @@ -301,28 +196,6 @@ def is_all_numeric(items): ) -def track_task_progress(additions, total=None): - """ - Updates the current export task with number of submission processed. - Updates in batches of settings EXPORT_TASK_PROGRESS_UPDATE_BATCH defaults - to 100. - :param additions: - :param total: - :return: - """ - batch_size = getattr( - settings, "EXPORT_TASK_PROGRESS_UPDATE_BATCH", DEFAULT_UPDATE_BATCH - ) - if additions % batch_size == 0: - meta = {"progress": additions} - if total: - meta.update({"total": total}) - try: - current_task.update_state(state="PROGRESS", meta=meta) - except AttributeError: - pass - - # pylint: disable=invalid-name def string_to_date_with_xls_validation(date_str): """Try to convert a string to a date object. @@ -548,6 +421,7 @@ def get_choice_dict(xpath, label): return choices + # pylint: disable=too-many-statements def set_survey(self, survey, xform=None, include_reviews=False): """Set's the XForm XML ``survey`` instance.""" if self.INCLUDE_REVIEWS or include_reviews: @@ -557,7 +431,7 @@ def set_survey(self, survey, xform=None, include_reviews=False): REVIEW_COMMENT, REVIEW_DATE, ] - self.__init__() + self.__init__() # pylint: disable=unnecessary-dunder-call data_dicionary = get_data_dictionary_from_survey(survey) # pylint: disable=too-many-locals,too-many-branches,too-many-arguments @@ -875,6 +749,7 @@ def split_select_multiples( @classmethod def split_gps_components(cls, row, gps_fields): + """Splits GPS components into their own fields.""" # for each gps_field, get associated data and split it for (xpath, gps_components) in iteritems(gps_fields): data = row.get(xpath) @@ -886,6 +761,7 @@ def split_gps_components(cls, row, gps_fields): @classmethod def decode_mongo_encoded_fields(cls, row, encoded_fields): + """Update encoded fields with their corresponding xpath""" for (xpath, encoded_xpath) in iteritems(encoded_fields): if row.get(encoded_xpath): val = row.pop(encoded_xpath) diff --git a/onadata/libs/utils/export_tools.py b/onadata/libs/utils/export_tools.py index 6cc125c5be..61ebed378b 100644 --- a/onadata/libs/utils/export_tools.py +++ b/onadata/libs/utils/export_tools.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# pylint: disable=too-many-lines """ Export tools """ @@ -49,7 +50,6 @@ retry, str_to_bool, ) -from onadata.libs.utils.export_builder import ExportBuilder from onadata.libs.utils.model_tools import get_columns_with_hxl, queryset_iterator from onadata.libs.utils.osm import get_combined_osm from onadata.libs.utils.viewer_tools import create_attachments_zipfile, image_urls @@ -167,40 +167,49 @@ def generate_export(export_type, xform, export_id=None, options=None): # noqa C if isinstance(records, QuerySet): records = records.iterator() + # pylint: disable=import-outside-toplevel + from onadata.libs.utils.export_builder import ExportBuilder + export_builder = ExportBuilder() - export_builder.TRUNCATE_GROUP_TITLE = ( + export_builder.TRUNCATE_GROUP_TITLE = ( # noqa True if export_type == Export.SAV_ZIP_EXPORT else remove_group_name ) - export_builder.GROUP_DELIMITER = options.get( + export_builder.GROUP_DELIMITER = options.get( # noqa "group_delimiter", DEFAULT_GROUP_DELIMITER ) - export_builder.SPLIT_SELECT_MULTIPLES = options.get("split_select_multiples", True) - export_builder.BINARY_SELECT_MULTIPLES = options.get( + export_builder.SPLIT_SELECT_MULTIPLES = options.get( # noqa + "split_select_multiples", True + ) + export_builder.BINARY_SELECT_MULTIPLES = options.get( # noqa "binary_select_multiples", False ) - export_builder.INCLUDE_LABELS = options.get("include_labels", False) + export_builder.INCLUDE_LABELS = options.get("include_labels", False) # noqa include_reviews = options.get("include_reviews", False) - export_builder.INCLUDE_LABELS_ONLY = options.get("include_labels_only", False) - export_builder.INCLUDE_HXL = options.get("include_hxl", False) + export_builder.INCLUDE_LABELS_ONLY = options.get( # noqa + "include_labels_only", False + ) + export_builder.INCLUDE_HXL = options.get("include_hxl", False) # noqa - export_builder.INCLUDE_IMAGES = options.get( + export_builder.INCLUDE_IMAGES = options.get( # noqa "include_images", settings.EXPORT_WITH_IMAGE_DEFAULT ) - export_builder.VALUE_SELECT_MULTIPLES = options.get("value_select_multiples", False) + export_builder.VALUE_SELECT_MULTIPLES = options.get( # noqa + "value_select_multiples", False + ) - export_builder.REPEAT_INDEX_TAGS = options.get( + export_builder.REPEAT_INDEX_TAGS = options.get( # noqa "repeat_index_tags", DEFAULT_INDEX_TAGS ) - export_builder.SHOW_CHOICE_LABELS = options.get("show_choice_labels", False) + export_builder.SHOW_CHOICE_LABELS = options.get("show_choice_labels", False) # noqa export_builder.language = options.get("language") # 'win_excel_utf8' is only relevant for CSV exports if "win_excel_utf8" in options and export_type != Export.CSV_EXPORT: del options["win_excel_utf8"] - export_builder.INCLUDE_REVIEWS = include_reviews + export_builder.INCLUDE_REVIEWS = include_reviews # noqa export_builder.set_survey(xform.survey, xform, include_reviews=include_reviews) temp_file = NamedTemporaryFile(suffix=("." + extension)) @@ -213,7 +222,7 @@ def generate_export(export_type, xform, export_id=None, options=None): # noqa C func = getattr(export_builder, export_type_func_map[export_type]) # pylint: disable=broad-except try: - func.__call__( + func( temp_file.name, records, username, @@ -556,7 +565,7 @@ def generate_geojson_export( metadata=None, export_id=None, options=None, - xform=None + xform=None, ): """ Generates Linked Geojson export @@ -579,12 +588,12 @@ def generate_geojson_export( "geo_field": extra_data.get("data_geo_field"), "simple_style": extra_data.get("data_simple_style"), "title": extra_data.get("data_title"), - "fields": extra_data.get("data_fields") + "fields": extra_data.get("data_fields"), } _context = {} - _context['request'] = request + _context["request"] = request content = GeoJsonSerializer(xform.instances.all(), many=True, context=_context) - data_to_write = json.dumps(content.data).encode('utf-8') + data_to_write = json.dumps(content.data).encode("utf-8") timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") basename = f"{id_string}_{timestamp}" filename = basename + "." + extension diff --git a/onadata/libs/utils/google.py b/onadata/libs/utils/google.py index 33fb4cd5f9..be323ff8bc 100644 --- a/onadata/libs/utils/google.py +++ b/onadata/libs/utils/google.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Google utility functions. """ diff --git a/onadata/libs/utils/image_tools.py b/onadata/libs/utils/image_tools.py index 3230a9953c..0d2ce32084 100644 --- a/onadata/libs/utils/image_tools.py +++ b/onadata/libs/utils/image_tools.py @@ -1,17 +1,22 @@ -import boto3 -import urllib +# -*- coding: utf-8 -*- +""" +Image utility functions module. +""" import logging +import urllib from datetime import datetime, timedelta from tempfile import NamedTemporaryFile +from wsgiref.util import FileWrapper -from PIL import Image from django.conf import settings from django.core.files.base import ContentFile from django.core.files.storage import get_storage_class from django.http import HttpResponse, HttpResponseRedirect -from botocore.exceptions import ClientError + +import boto3 from botocore.client import Config -from wsgiref.util import FileWrapper +from botocore.exceptions import ClientError +from PIL import Image from onadata.libs.utils.viewer_tools import get_path @@ -26,25 +31,29 @@ def flat(*nums): def generate_media_download_url(obj, expiration: int = 3600): + """ + Returns a HTTP response of a media object or a redirect to the image URL for S3 and + Azure storage objects. + """ file_path = obj.media_file.name default_storage = get_storage_class()() filename = file_path.split("/")[-1] - s3 = None + content_disposition = urllib.parse.quote(f"attachment; filename={filename}") + s3_class = None azure = None try: - s3 = get_storage_class("storages.backends.s3boto3.S3Boto3Storage")() + s3_class = get_storage_class("storages.backends.s3boto3.S3Boto3Storage")() except ModuleNotFoundError: pass try: azure = get_storage_class("storages.backends.azure_storage.AzureStorage")() except ModuleNotFoundError: - if s3 is None: + if s3_class is None: return HttpResponseRedirect(obj.media_file.url) - content_disposition = urllib.parse.quote(f"attachment; filename={filename}") - if isinstance(default_storage, type(s3)): + if isinstance(default_storage, type(s3_class)): try: url = generate_aws_media_url(file_path, content_disposition, expiration) except ClientError as e: @@ -52,22 +61,25 @@ def generate_media_download_url(obj, expiration: int = 3600): return None else: return HttpResponseRedirect(url) - elif isinstance(default_storage, type(azure)): + + if isinstance(default_storage, type(azure)): media_url = generate_media_url_with_sas(file_path, expiration) return HttpResponseRedirect(media_url) - else: - file_obj = open(settings.MEDIA_ROOT + file_path, "rb") - response = HttpResponse(FileWrapper(file_obj), content_type=obj.mimetype) - response["Content-Disposition"] = content_disposition - return response + # pylint: disable=consider-using-with + file_obj = open(settings.MEDIA_ROOT + file_path, "rb") + response = HttpResponse(FileWrapper(file_obj), content_type=obj.mimetype) + response["Content-Disposition"] = content_disposition + + return response def generate_aws_media_url( file_path: str, content_disposition: str, expiration: int = 3600 ): - s3 = get_storage_class("storages.backends.s3boto3.S3Boto3Storage")() - bucket_name = s3.bucket.name + """Generate S3 URL.""" + s3_class = get_storage_class("storages.backends.s3boto3.S3Boto3Storage")() + bucket_name = s3_class.bucket.name s3_config = Config( signature_version=getattr(settings, "AWS_S3_SIGNATURE_VERSION", "s3v4"), region_name=getattr(settings, "AWS_S3_REGION_NAME", ""), @@ -88,11 +100,17 @@ def generate_aws_media_url( def generate_media_url_with_sas(file_path: str, expiration: int = 3600): - from azure.storage.blob import generate_blob_sas, AccountSasPermissions + """ + Generate Azure storage URL. + """ + # pylint: disable=import-outside-toplevel + from azure.storage.blob import AccountSasPermissions, generate_blob_sas account_name = getattr(settings, "AZURE_ACCOUNT_NAME", "") container_name = getattr(settings, "AZURE_CONTAINER", "") - media_url = f"https://{account_name}.blob.core.windows.net/{container_name}/{file_path}" # noqa + media_url = ( + f"https://{account_name}.blob.core.windows.net/{container_name}/{file_path}" + ) sas_token = generate_blob_sas( account_name=account_name, account_key=getattr(settings, "AZURE_ACCOUNT_KEY", ""), @@ -105,6 +123,7 @@ def generate_media_url_with_sas(file_path: str, expiration: int = 3600): def get_dimensions(size, longest_side): + """Return integer tuple of width and height given size and longest_side length.""" width, height = size if width > height: @@ -121,23 +140,22 @@ def get_dimensions(size, longest_side): def _save_thumbnails(image, path, size, suffix, extension): - nm = NamedTemporaryFile(suffix=".%s" % extension) - default_storage = get_storage_class()() + with NamedTemporaryFile(suffix=f".{extension}") as temp_file: + default_storage = get_storage_class()() - try: - # Ensure conversion to float in operations - image.thumbnail(get_dimensions(image.size, float(size)), Image.ANTIALIAS) - except ZeroDivisionError: - pass + try: + # Ensure conversion to float in operations + image.thumbnail(get_dimensions(image.size, float(size)), Image.ANTIALIAS) + except ZeroDivisionError: + pass - image.save(nm.name) - default_storage.save(get_path(path, suffix), ContentFile(nm.read())) - nm.close() + image.save(temp_file.name) + default_storage.save(get_path(path, suffix), ContentFile(temp_file.read())) + temp_file.close() def resize(filename, extension): - if extension == "non": - extension = settings.DEFAULT_IMG_FILE_TYPE + """Resize an image into multiple sizes.""" default_storage = get_storage_class()() try: @@ -147,24 +165,31 @@ def resize(filename, extension): for key in settings.THUMB_ORDER: _save_thumbnails( - image, filename, conf[key]["size"], conf[key]["suffix"], extension + image, + filename, + conf[key]["size"], + conf[key]["suffix"], + settings.DEFAULT_IMG_FILE_TYPE if extension == "non" else extension, ) - except IOError: - raise Exception("The image file couldn't be identified") + except IOError as exc: + raise Exception("The image file couldn't be identified") from exc def resize_local_env(filename, extension): - if extension == "non": - extension = settings.DEFAULT_IMG_FILE_TYPE + """Resize images in a local environment.""" default_storage = get_storage_class()() path = default_storage.path(filename) image = Image.open(path) conf = settings.THUMB_CONF - [ - _save_thumbnails(image, path, conf[key]["size"], conf[key]["suffix"], extension) - for key in settings.THUMB_ORDER - ] + for key in settings.THUMB_ORDER: + _save_thumbnails( + image, + path, + conf[key]["size"], + conf[key]["suffix"], + settings.DEFAULT_IMG_FILE_TYPE if extension == "non" else extension, + ) def image_url(attachment, suffix): @@ -181,33 +206,33 @@ def image_url(attachment, suffix): if suffix == "original": return url - else: - default_storage = get_storage_class()() - fs = get_storage_class("django.core.files.storage.FileSystemStorage")() - - if suffix in settings.THUMB_CONF: - size = settings.THUMB_CONF[suffix]["suffix"] - filename = attachment.media_file.name - - if default_storage.exists(filename): - if ( - default_storage.exists(get_path(filename, size)) - and default_storage.size(get_path(filename, size)) > 0 - ): - file_path = get_path(filename, size) - url = ( - generate_media_url_with_sas(file_path) - if isinstance(default_storage, type(azure)) - else default_storage.url(file_path) - ) - else: - if default_storage.__class__ != fs.__class__: - resize(filename, extension=attachment.extension) - else: - resize_local_env(filename, extension=attachment.extension) - return image_url(attachment, suffix) + default_storage = get_storage_class()() + file_storage = get_storage_class("django.core.files.storage.FileSystemStorage")() + + if suffix in settings.THUMB_CONF: + size = settings.THUMB_CONF[suffix]["suffix"] + filename = attachment.media_file.name + + if default_storage.exists(filename): + if ( + default_storage.exists(get_path(filename, size)) + and default_storage.size(get_path(filename, size)) > 0 + ): + file_path = get_path(filename, size) + url = ( + generate_media_url_with_sas(file_path) + if isinstance(default_storage, type(azure)) + else default_storage.url(file_path) + ) else: - return None + if default_storage.__class__ != file_storage.__class__: + resize(filename, extension=attachment.extension) + else: + resize_local_env(filename, extension=attachment.extension) + + return image_url(attachment, suffix) + else: + return None return url diff --git a/onadata/libs/utils/log.py b/onadata/libs/utils/log.py index 8b2d470c19..0e35001209 100644 --- a/onadata/libs/utils/log.py +++ b/onadata/libs/utils/log.py @@ -1,12 +1,16 @@ +# -*- coding: utf-8 -*- +""" +Log utility functions and classes. +""" import logging from datetime import datetime -from django.utils.translation import gettext as _ - from onadata.libs.utils.viewer_tools import get_client_ip -class Enum(object): +class Enum: + """Enum class - dict-like class""" + __name__ = "Enum" def __init__(self, **enums): @@ -19,7 +23,7 @@ def __getitem__(self, item): return self.__getattr__(item) def __iter__(self): - return self.enums.itervalues() + return iter(self.enums.values()) Actions = Enum( @@ -60,43 +64,44 @@ def __iter__(self): class AuditLogHandler(logging.Handler): + """Audit logging handler class.""" def __init__(self, model=""): - super(AuditLogHandler, self).__init__() + super().__init__() self.model_name = model def _format(self, record): created_on = datetime.utcfromtimestamp(record.created).isoformat() created_on = created_on[:23] + created_on[26:] data = { - 'action': record.formhub_action, - 'user': record.request_username, - 'account': record.account_username, - 'audit': {}, - 'msg': record.msg, + "action": record.formhub_action, + "user": record.request_username, + "account": record.account_username, + "audit": {}, + "msg": record.msg, # save as python datetime object # to have mongo convert to ISO date and allow queries - 'created_on': created_on, - 'levelno': record.levelno, - 'levelname': record.levelname, - 'args': record.args, - 'funcName': record.funcName, - 'msecs': record.msecs, - 'relativeCreated': record.relativeCreated, - 'thread': record.thread, - 'name': record.name, - 'threadName': record.threadName, - 'exc_info': record.exc_info, - 'pathname': record.pathname, - 'exc_text': record.exc_text, - 'lineno': record.lineno, - 'process': record.process, - 'filename': record.filename, - 'module': record.module, - 'processName': record.processName + "created_on": created_on, + "levelno": record.levelno, + "levelname": record.levelname, + "args": record.args, + "funcName": record.funcName, + "msecs": record.msecs, + "relativeCreated": record.relativeCreated, + "thread": record.thread, + "name": record.name, + "threadName": record.threadName, + "exc_info": record.exc_info, + "pathname": record.pathname, + "exc_text": record.exc_text, + "lineno": record.lineno, + "process": record.process, + "filename": record.filename, + "module": record.module, + "processName": record.processName, } - if hasattr(record, 'audit') and isinstance(record.audit, dict): - data['audit'] = record.audit + if hasattr(record, "audit") and isinstance(record.audit, dict): + data["audit"] = record.audit return data @@ -105,21 +110,24 @@ def emit(self, record): # save to mongodb audit_log try: model = self.get_model(self.model_name) - except Exception as e: - logging.exception(_(u'Get model threw exception: %s' % str(e))) + except Exception as e: # pylint: disable=broad-except + logging.exception("Get model threw exception: %s", str(e)) else: log_entry = model(data) log_entry.save() def get_model(self, name): - names = name.split('.') - mod = __import__('.'.join(names[:-1]), fromlist=names[-1:]) + """Import and return the model under the given ``name``.""" + names = name.split(".") + mod = __import__(".".join(names[:-1]), fromlist=names[-1:]) return getattr(mod, names[-1]) -def audit_log(action, request_user, account_user, message, audit, request, - level=logging.DEBUG): +# pylint: disable=too-many-arguments +def audit_log( + action, request_user, account_user, message, audit, request, level=logging.DEBUG +): """ Create a log message based on these params @@ -135,12 +143,14 @@ def audit_log(action, request_user, account_user, message, audit, request, """ logger = logging.getLogger("audit_logger") extra = { - 'formhub_action': action, - 'request_username': - request_user.username if request_user.username else str(request_user), - 'account_username': - account_user.username if account_user.username else str(account_user), - 'client_ip': get_client_ip(request), - 'audit': audit + "formhub_action": action, + "request_username": request_user.username + if request_user.username + else str(request_user), + "account_username": account_user.username + if account_user.username + else str(account_user), + "client_ip": get_client_ip(request), + "audit": audit, } logger.log(level, message, extra=extra) diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index 928cd8cd12..0f0b5a10d3 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -78,7 +78,7 @@ from onadata.apps.viewer.models.data_dictionary import DataDictionary from onadata.apps.viewer.models.parsed_instance import ParsedInstance from onadata.apps.viewer.signals import process_submission -from onadata.libs.utils.analytics import track_object_event +from onadata.libs.utils.analytics import TrackObjectEvent from onadata.libs.utils.common_tags import METADATA_FIELDS from onadata.libs.utils.common_tools import get_uuid, report_exception from onadata.libs.utils.model_tools import set_uuid @@ -654,11 +654,11 @@ def safe_create_instance( # noqa C901 error = OpenRosaResponseBadRequest(e) except DjangoUnicodeDecodeError: error = OpenRosaResponseBadRequest( - _("File likely corrupted during " "transmission, please try later.") + _("File likely corrupted during transmission, please try later.") ) except NonUniqueFormIdError: error = OpenRosaResponseBadRequest( - _("Unable to submit because there are multiple forms with" " this formID.") + _("Unable to submit because there are multiple forms with this formID.") ) except DataError as e: error = OpenRosaResponseBadRequest((str(e))) @@ -755,7 +755,7 @@ def publish_form(callback): return { "type": "alert-error", "text": _( - ("An error occurred while publishing the form. " "Please try again.") + ("An error occurred while publishing the form. Please try again.") ), } except (AttributeError, DuplicateUUIDError, ValidationError) as e: @@ -763,7 +763,7 @@ def publish_form(callback): return {"type": "alert-error", "text": text(e)} -@track_object_event( +@TrackObjectEvent( user_field="user", properties={"created_by": "user", "xform_id": "pk", "xform_name": "title"}, additional_context={"from": "Publish XLS Form"}, @@ -788,7 +788,7 @@ def publish_xls_form(xls_file, user, project, id_string=None, created_by=None): return dd -@track_object_event( +@TrackObjectEvent( user_field="user", properties={"created_by": "user", "xform_id": "pk", "xform_name": "title"}, additional_context={"from": "Publish XML Form"}, diff --git a/onadata/libs/utils/middleware.py b/onadata/libs/utils/middleware.py index 25d6eaa7d5..5eb11cd789 100644 --- a/onadata/libs/utils/middleware.py +++ b/onadata/libs/utils/middleware.py @@ -1,18 +1,25 @@ +# -*- coding: utf-8 -*- +""" +Custom middleware classes. +""" import logging import traceback +from sys import stdout from django.conf import settings -from django.db import connection -from django.db import OperationalError +from django.db import OperationalError, connection from django.http import HttpResponseNotAllowed -from django.template import loader from django.middleware.locale import LocaleMiddleware +from django.template import loader from django.utils.translation import gettext as _ from django.utils.translation.trans_real import parse_accept_lang_header + from multidb.pinning import use_master class BaseMiddleware: + """BaseMiddleware - The base middleware class.""" + def __init__(self, get_response): self.get_response = get_response @@ -20,23 +27,28 @@ def __call__(self, request): return self.get_response(request) -class ExceptionLoggingMiddleware(object): +class ExceptionLoggingMiddleware: + """The exception logging middleware class - prints the exception traceback.""" + def __init__(self, get_response): self.get_response = get_response + # pylint: disable=unused-argument def process_exception(self, request, exception): + """Prints the exception traceback.""" print(traceback.format_exc()) -class HTTPResponseNotAllowedMiddleware(object): +class HTTPResponseNotAllowedMiddleware: + """The HTTP Not Allowed middleware class - renders the 405.html template.""" + def __init__(self, get_response): self.get_response = get_response def __call__(self, request): response = self.get_response(request) if isinstance(response, HttpResponseNotAllowed): - response.content = loader.render_to_string( - "405.html", request=request) + response.content = loader.render_to_string("405.html", request=request) return response @@ -48,31 +60,39 @@ class LocaleMiddlewareWithTweaks(LocaleMiddleware): """ def process_request(self, request): - accept = request.headers.get('Accept-Language', '') + accept = request.headers.get("Accept-Language", "") try: codes = [code for code, r in parse_accept_lang_header(accept)] - if 'km' in codes and 'km-kh' not in codes: - request.META['HTTP_ACCEPT_LANGUAGE'] = accept.replace('km', - 'km-kh') - except Exception as e: + if "km" in codes and "km-kh" not in codes: + request.META["HTTP_ACCEPT_LANGUAGE"] = accept.replace("km", "km-kh") + except Exception as e: # pylint: disable=broad-except # this might fail if i18n is disabled. - logging.exception(_(u'Settings request META HTTP accept language ' - 'threw exceptions: %s' % str(e))) + logging.exception( + _( + "Settings request META HTTP accept language " + f"threw exceptions: {str(e)}" + ) + ) - super(LocaleMiddlewareWithTweaks, self).process_request(request) + super().process_request(request) -class SqlLogging(object): +class SqlLogging: + """ + SQL logging middleware. + """ + def __init__(self, get_response): self.get_response = get_response def __call__(self, request): response = self.get_response(request) - from sys import stdout + if stdout.isatty(): for query in connection.queries: - print("\033[1;31m[%s]\033[0m \033[1m%s\033[0m" % ( - query['time'], " ".join(query['sql'].split()))) + time = query["time"] + sql = " ".join(query["sql"].split()) + print(f"\033[1;31m[{time}]\033[0m \033[1m{sql}\033[0m") return response @@ -82,7 +102,11 @@ class OperationalErrorMiddleware(BaseMiddleware): Captures requests returning 500 status code. Then retry it against master database. """ + def process_exception(self, request, exception): + """ + Handle retrying OperatuonalError exceptions. + """ # Filter out OperationalError Exceptions if isinstance(exception, OperationalError): already_raised = getattr(settings, "ALREADY_RAISED", False) @@ -94,3 +118,5 @@ def process_exception(self, request, exception): response = self.get_response(request) return response settings.ALREADY_RAISED = False + + return None diff --git a/onadata/libs/utils/model_tools.py b/onadata/libs/utils/model_tools.py index a3ade05174..5f08d9741d 100644 --- a/onadata/libs/utils/model_tools.py +++ b/onadata/libs/utils/model_tools.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Model utility functions. """ diff --git a/onadata/libs/utils/mongo.py b/onadata/libs/utils/mongo.py index 34da61045e..1b2d1d9cae 100644 --- a/onadata/libs/utils/mongo.py +++ b/onadata/libs/utils/mongo.py @@ -1,18 +1,32 @@ +# -*- coding: utf-8 -*- +""" +Utility functions for MongoDB operations. +""" from __future__ import unicode_literals import base64 import re from functools import reduce - -key_whitelist = ['$or', '$and', '$exists', '$in', '$gt', '$gte', - '$lt', '$lte', '$regex', '$options', '$all'] -b64dollar = base64.b64encode(b'$').decode('utf-8') -b64dot = base64.b64encode(b'.').decode('utf-8') -re_b64dollar = re.compile(r'^%s' % b64dollar) -re_b64dot = re.compile(r'%s' % b64dot) -re_dollar = re.compile(r'^\$') -re_dot = re.compile(r'\.') +key_whitelist = [ + "$or", + "$and", + "$exists", + "$in", + "$gt", + "$gte", + "$lt", + "$lte", + "$regex", + "$options", + "$all", +] +b64dollar = base64.b64encode(b"$").decode("utf-8") +b64dot = base64.b64encode(b".").decode("utf-8") +re_b64dollar = re.compile(rf"^{b64dollar}") +re_b64dot = re.compile(rf"{b64dot}") +re_dollar = re.compile(r"^\$") +re_dot = re.compile(r"\.") def _pattern_transform(key, transform_list): @@ -20,7 +34,7 @@ def _pattern_transform(key, transform_list): def _decode_from_mongo(key): - return _pattern_transform(key, [(re_b64dollar, '$'), (re_b64dot, '.')]) + return _pattern_transform(key, [(re_b64dollar, "$"), (re_b64dot, ".")]) def _encode_for_mongo(key): @@ -28,5 +42,4 @@ def _encode_for_mongo(key): def _is_invalid_for_mongo(key): - return key not in\ - key_whitelist and (key.startswith('$') or key.count('.') > 0) + return key not in key_whitelist and (key.startswith("$") or key.count(".") > 0) diff --git a/onadata/libs/utils/numeric.py b/onadata/libs/utils/numeric.py index 9ba9a6635a..c376e43020 100644 --- a/onadata/libs/utils/numeric.py +++ b/onadata/libs/utils/numeric.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +The int_or_parse_error utility function. +""" from rest_framework.exceptions import ParseError @@ -8,5 +12,5 @@ def int_or_parse_error(value, error_string): """ try: int(value) - except ValueError: - raise ParseError(error_string) + except ValueError as exc: + raise ParseError(error_string) from exc diff --git a/onadata/libs/utils/openid_connect_tools.py b/onadata/libs/utils/openid_connect_tools.py index 73c8f48046..1ecdb6e729 100644 --- a/onadata/libs/utils/openid_connect_tools.py +++ b/onadata/libs/utils/openid_connect_tools.py @@ -1,21 +1,22 @@ +# -*- coding: utf-8 -*- """ OpenID Connect Tools """ import json -from django.http import HttpResponseRedirect, Http404 from django.core.cache import cache +from django.http import Http404, HttpResponseRedirect from django.utils.translation import gettext as _ import jwt import requests from jwt.algorithms import RSAAlgorithm -EMAIL = 'email' -NAME = 'name' -FIRST_NAME = 'given_name' -LAST_NAME = 'family_name' -NONCE = 'nonce' +EMAIL = "email" +NAME = "name" +FIRST_NAME = "given_name" +LAST_NAME = "family_name" +NONCE = "nonce" class OpenIDHandler: @@ -26,60 +27,54 @@ class OpenIDHandler: 'code' or 'id_token' authorization flow """ - def __init__( - self, - provider_configuration: dict - ): + def __init__(self, provider_configuration: dict): """ Initializes a OpenIDHandler Object to handle all OpenID Connect grant flows """ self.provider_configuration = provider_configuration - self.client_id = provider_configuration.get('client_id') - self.client_secret = provider_configuration.get('client_secret') + self.client_id = provider_configuration.get("client_id") + self.client_secret = provider_configuration.get("client_secret") def make_login_request(self, nonce: int, state=None): """ Makes a login request to the "authorization_endpoint" listed in the provider_configuration """ - if 'authorization_endpoint' in self.provider_configuration: - url = self.provider_configuration['authorization_endpoint'] - url += f'?nonce={nonce}' + if "authorization_endpoint" in self.provider_configuration: + url = self.provider_configuration["authorization_endpoint"] + url += f"?nonce={nonce}" if state: - url += f'&state={state}' + url += f"&state={state}" else: raise ValueError( - 'authorization_endpoint not found in provider configuration') + "authorization_endpoint not found in provider configuration" + ) - if 'client_id' in self.provider_configuration: - url += '&client_id=' + self.provider_configuration['client_id'] + if "client_id" in self.provider_configuration: + url += "&client_id=" + self.provider_configuration["client_id"] else: - raise ValueError('client_id not found in provider configuration') + raise ValueError("client_id not found in provider configuration") - if 'callback_uri' in self.provider_configuration: - url += '&redirect_uri=' + self.provider_configuration[ - 'callback_uri'] + if "callback_uri" in self.provider_configuration: + url += "&redirect_uri=" + self.provider_configuration["callback_uri"] else: - raise ValueError('client_id not found in provider configuration') + raise ValueError("client_id not found in provider configuration") - if 'scope' in self.provider_configuration: - url += '&scope=' + self.provider_configuration['scope'] + if "scope" in self.provider_configuration: + url += "&scope=" + self.provider_configuration["scope"] else: - raise ValueError('scope not found in provider configuration') + raise ValueError("scope not found in provider configuration") - if 'response_type' in self.provider_configuration: - url += '&response_type=' + self.provider_configuration[ - 'response_type'] + if "response_type" in self.provider_configuration: + url += "&response_type=" + self.provider_configuration["response_type"] else: - raise ValueError( - 'response_type not found in provider configuration') + raise ValueError("response_type not found in provider configuration") - if 'response_mode' in self.provider_configuration: - url += '&response_mode=' + self.provider_configuration[ - 'response_mode'] + if "response_mode" in self.provider_configuration: + url += "&response_mode=" + self.provider_configuration["response_mode"] return HttpResponseRedirect(url) @@ -94,7 +89,7 @@ def get_claim_values(self, claim_list: list, decoded_token: dict): decoded_token: A dict containing the decoded values of an ID Token """ claim_values = {} - claim_names = self.provider_configuration.get('claims') + claim_names = self.provider_configuration.get("claims") for claim in claim_list: claim_name = claim @@ -113,51 +108,54 @@ def _retrieve_jwk_related_to_kid(self, kid): from the JSON Web Key Set Endpoint """ if "jwks_endpoint" not in self.provider_configuration: - raise ValueError( - "jwks_endpoint not found in provider configuration") + raise ValueError("jwks_endpoint not found in provider configuration") - response = requests.get(self.provider_configuration['jwks_endpoint']) + response = requests.get(self.provider_configuration["jwks_endpoint"]) if response.status_code == 200: jwks = response.json() - for jwk in jwks.get('keys'): - if jwk.get('kid') == kid: + for jwk in jwks.get("keys"): + if jwk.get("kid") == kid: return jwk - def obtain_id_token_from_code(self, code: str, openid_provider: str = ''): + return None + + def obtain_id_token_from_code(self, code: str, openid_provider: str = ""): """ Obtain an ID Token using the Authorization Code flow """ - headers = {'Content-Type': 'application/x-www-form-urlencoded'} + headers = {"Content-Type": "application/x-www-form-urlencoded"} payload = { - 'grant_type': 'authorization_code', - 'code': code, - 'client_id': self.client_id, - 'client_secret': self.client_secret, - 'redirect_uri': self.provider_configuration.get('callback_uri') + "grant_type": "authorization_code", + "code": code, + "client_id": self.client_id, + "client_secret": self.client_secret, + "redirect_uri": self.provider_configuration.get("callback_uri"), } if "token_endpoint" not in self.provider_configuration: raise ValueError("token_endpoint not in provider configuration") response = requests.post( - self.provider_configuration['token_endpoint'], + self.provider_configuration["token_endpoint"], params=payload, - headers=headers) + headers=headers, + ) if response.status_code == 200: - id_token = response.json().get('id_token') + id_token = response.json().get("id_token") return id_token - else: - retry_message = 'Failed to retrieve ID Token, ' + \ - f'retry' + \ - 'the authentication process' - raise Http404(_(retry_message)) + + retry_message = ( + "Failed to retrieve ID Token, " + + f'retry' + + "the authentication process" + ) + raise Http404(_(retry_message)) def verify_and_decode_id_token( - self, id_token: str, - cached_nonce: bool = False, - openid_provider: str = ''): + self, id_token: str, cached_nonce: bool = False, openid_provider: str = "" + ): """ Verifies that the ID Token passed was signed and sent by the Open ID Connect Provider and that the client is one of the audiences then @@ -166,46 +164,44 @@ def verify_and_decode_id_token( unverified_header = jwt.get_unverified_header(id_token) # Get public key thumbprint - kid = unverified_header.get('kid') + kid = unverified_header.get("kid") jwk = self._retrieve_jwk_related_to_kid(kid) if jwk: - alg = unverified_header.get('alg') + alg = unverified_header.get("alg") public_key = RSAAlgorithm.from_jwk(json.dumps(jwk)) - try: - decoded_token = jwt.decode( - id_token, - public_key, - audience=[self.client_id], - algorithms=alg) - - if cached_nonce: - # Verify that the cached nonce is present and that - # the provider the nonce was initiated for, is the same - # provider returning it - provider_initiated_for = cache.get( - decoded_token.get(NONCE)) - - if provider_initiated_for != openid_provider: - raise Exception('Incorrect nonce value returned') - return decoded_token - except Exception as e: - raise e + decoded_token = jwt.decode( + id_token, public_key, audience=[self.client_id], algorithms=alg + ) + + if cached_nonce: + # Verify that the cached nonce is present and that + # the provider the nonce was initiated for, is the same + # provider returning it + provider_initiated_for = cache.get(decoded_token.get(NONCE)) + + if provider_initiated_for != openid_provider: + raise Exception("Incorrect nonce value returned") + return decoded_token + + return None def end_openid_provider_session(self): """ Clears the SSO cookie set at authentication and redirects the User to the end_session endpoint provided by the provider configuration """ - end_session_endpoint = self.provider_configuration.get( - 'end_session_endpoint') + end_session_endpoint = self.provider_configuration.get("end_session_endpoint") target_url_after_logout = self.provider_configuration.get( - 'target_url_after_logout') + "target_url_after_logout" + ) response = HttpResponseRedirect( - end_session_endpoint + - '?post_logout_redirect_uri=' + target_url_after_logout) - response.delete_cookie('SSO') + end_session_endpoint + + "?post_logout_redirect_uri=" + + target_url_after_logout + ) + response.delete_cookie("SSO") return response diff --git a/onadata/libs/utils/osm.py b/onadata/libs/utils/osm.py index cfad61280e..fc721fb9b5 100644 --- a/onadata/libs/utils/osm.py +++ b/onadata/libs/utils/osm.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ OSM utility module. """ @@ -17,7 +17,7 @@ from onadata.apps.logger.models.instance import Instance from onadata.apps.logger.models.osmdata import OsmData from onadata.apps.restservice.signals import trigger_webhook -from onadata.celery import app +from onadata.celeryapp import app def _get_xml_obj(xml): @@ -25,8 +25,8 @@ def _get_xml_obj(xml): xml = xml.strip().encode() try: return fromstring(xml) - except _etree.XMLSyntaxError as e: # pylint: disable=no-member - if "Attribute action redefined" in e.msg: + except _etree.XMLSyntaxError as e: # pylint: disable=c-extension-no-member + if "Attribute action redefined" in str(e): xml = xml.replace(b'action="modify" ', b"") return _get_xml_obj(xml) diff --git a/onadata/libs/utils/profiler.py b/onadata/libs/utils/profiler.py deleted file mode 100644 index 296f7c3667..0000000000 --- a/onadata/libs/utils/profiler.py +++ /dev/null @@ -1,49 +0,0 @@ -import hotshot -import os -import time -from django.conf import settings -import tempfile - -if hasattr(settings, 'PROFILE_LOG_BASE'): - PROFILE_LOG_BASE = settings.PROFILE_LOG_BASE -else: - PROFILE_LOG_BASE = tempfile.gettempdir() - - -def profile(log_file): - """Profile some callable. - - This decorator uses the hotshot profiler to profile some callable (like - a view function or method) and dumps the profile data somewhere sensible - for later processing and examination. - - It takes one argument, the profile log name. If it's a relative path, - it places it under the PROFILE_LOG_BASE. It also inserts a time stamp - into the file name, such that 'my_view.prof' become - 'my_view-20100211T170321.prof', where the time stamp is in UTC. This - makes it easy to run and compare multiple trials. - """ - - if not os.path.isabs(log_file): - log_file = os.path.join(PROFILE_LOG_BASE, log_file) - - def _outer(f): - if not settings.PROFILE_API_ACTION_FUNCTION: - return f - - def _inner(*args, **kwargs): - # Add a timestamp to the profile output when the callable - # is actually called. - (base, ext) = os.path.splitext(log_file) - base = base + "-" + time.strftime("%Y%m%dT%H%M%S", time.gmtime()) - final_log_file = base + ext - - prof = hotshot.Profile(final_log_file) - try: - ret = prof.runcall(f, *args, **kwargs) - finally: - prof.close() - return ret - - return _inner - return _outer diff --git a/onadata/libs/utils/project_utils.py b/onadata/libs/utils/project_utils.py index 234a53f360..60c3e3d1dc 100644 --- a/onadata/libs/utils/project_utils.py +++ b/onadata/libs/utils/project_utils.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ project_utils module - apply project permissions to a form. """ @@ -7,16 +7,24 @@ from django.conf import settings from django.db import IntegrityError -from onadata.apps.logger.models import Project, XForm -from onadata.celery import app +from multidb.pinning import use_master + +from onadata.apps.logger.models.project import Project +from onadata.apps.logger.models.xform import XForm +from onadata.celeryapp import app from onadata.libs.permissions import ( - ROLES, OwnerRole, get_object_users_with_permissions, - is_organization, get_role) + ROLES, + OwnerRole, + get_object_users_with_permissions, + get_role, + is_organization, +) from onadata.libs.utils.common_tags import OWNER_TEAM_NAME from onadata.libs.utils.common_tools import report_exception def get_project_users(project): + """Return project users with the role assigned to them.""" ret = {} for perm in project.projectuserobjectpermission_set.all(): @@ -24,17 +32,17 @@ def get_project_users(project): user = perm.user ret[user.username] = { - 'permissions': [], - 'is_org': is_organization(user.profile), - 'first_name': user.first_name, - 'last_name': user.last_name, + "permissions": [], + "is_org": is_organization(user.profile), + "first_name": user.first_name, + "last_name": user.last_name, } - ret[perm.user.username]['permissions'].append(perm.permission.codename) + ret[perm.user.username]["permissions"].append(perm.permission.codename) - for user in ret.keys(): - ret[user]['role'] = get_role(ret[user]['permissions'], project) - del ret[user]['permissions'] + for user, val in ret.items(): + val["role"] = get_role(val["permissions"], project) + del val["permissions"] return ret @@ -53,26 +61,24 @@ def set_project_perms_to_xform(xform, project): xform.save() # clear existing permissions - for perm in get_object_users_with_permissions( - xform, with_group_users=True): - user = perm['user'] - role_name = perm['role'] + for perm in get_object_users_with_permissions(xform, with_group_users=True): + user = perm["user"] + role_name = perm["role"] role = ROLES.get(role_name) - if role and (user != xform.user and project.user != user and - project.created_by != user): + if role and (user not in (xform.user, project.user, project.created_by)): role.remove_obj_permissions(user, xform) owners = project.organization.team_set.filter( - name="{}#{}".format(project.organization.username, OWNER_TEAM_NAME), - organization=project.organization) + name=f"{project.organization.username}#{OWNER_TEAM_NAME}", + organization=project.organization, + ) if owners: OwnerRole.add(owners[0], xform) - for perm in get_object_users_with_permissions( - project, with_group_users=True): - user = perm['user'] - role_name = perm['role'] + for perm in get_object_users_with_permissions(project, with_group_users=True): + user = perm["user"] + role_name = perm["role"] role = ROLES.get(role_name) if user == xform.created_by: @@ -88,13 +94,16 @@ def set_project_perms_to_xform_async(self, xform_id, project_id): """ Apply project permissions for ``project_id`` to a form ``xform_id`` task. """ + def _set_project_perms(): try: xform = XForm.objects.get(id=xform_id) project = Project.objects.get(id=project_id) except (Project.DoesNotExist, XForm.DoesNotExist) as e: - msg = '%s: Setting project %d permissions to form %d failed.' % ( - type(e), project_id, xform_id) + msg = ( + f"{type(e)}: Setting project {project_id} permissions to " + f"form {xform_id} failed." + ) # make a report only on the 3rd try. if self.request.retries > 2: report_exception(msg, e, sys.exc_info()) @@ -103,8 +112,8 @@ def _set_project_perms(): set_project_perms_to_xform(xform, project) try: - if getattr(settings, 'SLAVE_DATABASES', []): - from multidb.pinning import use_master + if getattr(settings, "SLAVE_DATABASES", []): + with use_master: _set_project_perms() else: @@ -112,8 +121,10 @@ def _set_project_perms(): except (Project.DoesNotExist, XForm.DoesNotExist) as e: # make a report only on the 3rd try. if self.request.retries > 2: - msg = '%s: Setting project %d permissions to form %d failed.' % ( - type(e), project_id, xform_id) + msg = ( + f"{type(e)}: Setting project {project_id} permissions to " + f"form {xform_id} failed." + ) report_exception(msg, e, sys.exc_info()) # let's retry if the record may still not be available in read replica. self.retry(countdown=60 * self.request.retries) @@ -122,6 +133,8 @@ def _set_project_perms(): # already. pass except Exception as e: # pylint: disable=broad-except - msg = '%s: Setting project %d permissions to form %d failed.' % ( - type(e), project_id, xform_id) + msg = ( + f"{type(e)}: Setting project {project_id} permissions to " + f"form {xform_id} failed." + ) report_exception(msg, e, sys.exc_info()) diff --git a/onadata/libs/utils/quick_converter.py b/onadata/libs/utils/quick_converter.py index 8923c3be36..d57b682d32 100644 --- a/onadata/libs/utils/quick_converter.py +++ b/onadata/libs/utils/quick_converter.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +The QuickConverter form class - publishes XLSForms. +""" from django import forms from django.utils.translation import gettext_lazy @@ -5,10 +9,17 @@ class QuickConverter(forms.Form): + """ + The QuickConverter form - publishes XLSForms. + """ + xls_file = forms.FileField(label=gettext_lazy("XLS File")) def publish(self, user): + """Create and return a DataDictionary object.""" if self.is_valid(): return DataDictionary.objects.create( - user=user, - xls=self.cleaned_data['xls_file']) + user=user, xls=self.cleaned_data["xls_file"] + ) + + return None diff --git a/onadata/libs/utils/timing.py b/onadata/libs/utils/timing.py index 88143e7b75..c2e4209bc8 100644 --- a/onadata/libs/utils/timing.py +++ b/onadata/libs/utils/timing.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Custom date utility functions. +""" import datetime import six @@ -5,11 +9,16 @@ def get_header_date_format(date_modified): - format = "%a, %d %b %Y %H:%M:%S GMT" - return date_modified.strftime(format) + """ + Returns ``date_modified`` as a string with the format %a, %d %b %Y %H:%M:%S GMT + """ + return date_modified.strftime("%a, %d %b %Y %H:%M:%S GMT") def get_date(_object=None): + """ + Returns a date formatted string of the date_modified of the given ``_object``. + """ if hasattr(_object, "date_modified"): _date = _object.date_modified elif hasattr(_object, "instance"): @@ -20,9 +29,9 @@ def get_date(_object=None): _date = _object.profile.date_modified elif isinstance(_object, dict): # most likely an instance json, use _submission_time - _date = _object.get('_submission_time') + _date = _object.get("_submission_time") if isinstance(_date, six.string_types): - _date = datetime.datetime.strptime(_date[:19], '%Y-%m-%dT%H:%M:%S') + _date = datetime.datetime.strptime(_date[:19], "%Y-%m-%dT%H:%M:%S") else: # default value to avoid the UnboundLocalError _date = timezone.now() @@ -31,7 +40,8 @@ def get_date(_object=None): def last_modified_header(last_modified_date): - return {'Last-Modified': last_modified_date} + """Returns a dictionary with the 'Last-Modified' key and value.""" + return {"Last-Modified": last_modified_date} def calculate_duration(start_time, end_time): @@ -45,7 +55,7 @@ def calculate_duration(start_time, end_time): _start = datetime.datetime.strptime(start_time[:19], _format) _end = datetime.datetime.strptime(end_time[:19], _format) except (TypeError, ValueError): - return '' + return "" duration = (_end - _start).total_seconds() diff --git a/onadata/libs/utils/user_auth.py b/onadata/libs/utils/user_auth.py index 425ac4fe1d..d7b55dc428 100644 --- a/onadata/libs/utils/user_auth.py +++ b/onadata/libs/utils/user_auth.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ User authentication utility functions. """ @@ -6,6 +6,7 @@ import re from functools import wraps +from django.apps import apps from django.contrib.auth import authenticate, get_user_model from django.contrib.sites.models import Site from django.http import HttpResponse @@ -14,13 +15,18 @@ from guardian.shortcuts import assign_perm, get_perms_for_model from rest_framework.authtoken.models import Token -from onadata.apps.api.models import OrganizationProfile, Team, TempToken -from onadata.apps.logger.models import MergedXForm, Note, Project, XForm -from onadata.apps.main.models import UserProfile +from onadata.apps.api.models.team import Team +from onadata.apps.api.models.temp_token import TempToken +from onadata.apps.logger.models.note import Note +from onadata.apps.logger.models.project import Project +from onadata.apps.logger.models.xform import XForm from onadata.libs.utils.viewer_tools import get_form # pylint: disable=invalid-name User = get_user_model() +UserProfile = apps.get_model("main", "UserProfile") +OrganizationProfile = apps.get_model("api", "OrganizationProfile") +MergedXForm = apps.get_model("logger", "MergedXForm") class HttpResponseNotAuthorized(HttpResponse): diff --git a/onadata/settings/common.py b/onadata/settings/common.py index 83ef6581c3..2a1e2f1773 100644 --- a/onadata/settings/common.py +++ b/onadata/settings/common.py @@ -242,9 +242,10 @@ "USE_RAPIDPRO_VIEWSET": False, } +MSFT_OAUTH_ENDPOINT = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize" OPENID_CONNECT_AUTH_SERVERS = { "microsoft": { - "AUTHORIZATION_ENDPOINT": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize", + "AUTHORIZATION_ENDPOINT": MSFT_OAUTH_ENDPOINT, "CLIENT_ID": "client_id", "JWKS_ENDPOINT": "https://login.microsoftonline.com/common/discovery/v2.0/keys", "SCOPE": "openid profile", @@ -257,10 +258,11 @@ } } +DEFAULT_MODEL_SERIALIZER_CLASS = "rest_framework.serializers.HyperlinkedModelSerializer" REST_FRAMEWORK = { # Use hyperlinked styles by default. # Only used if the `serializer_class` attribute is not set on a view. - "DEFAULT_MODEL_SERIALIZER_CLASS": "rest_framework.serializers.HyperlinkedModelSerializer", + "DEFAULT_MODEL_SERIALIZER_CLASS": DEFAULT_MODEL_SERIALIZER_CLASS, # Use Django's standard `django.contrib.auth` permissions, # or allow read-only access for unauthenticated users. "DEFAULT_PERMISSION_CLASSES": [ @@ -439,7 +441,7 @@ def configure_logging(logger, **kwargs): GOOGLE_STEP2_URI = "http://ona.io/gwelcome" GOOGLE_OAUTH2_CLIENT_ID = "REPLACE ME" -GOOGLE_OAUTH2_CLIENT_SECRET = "REPLACE ME" +GOOGLE_OAUTH2_CLIENT_SECRET = "REPLACE ME" # noqa THUMB_CONF = { "large": {"size": 1280, "suffix": "-large"}, @@ -578,7 +580,7 @@ def configure_logging(logger, **kwargs): # Prevents "The number of GET/POST parameters exceeded" exception DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000000 -SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" # noqa # Time in minutes to lock out user from account LOCKOUT_TIME = 30 * 60 diff --git a/onadata/settings/default_settings.py b/onadata/settings/default_settings.py index 16669f7741..409ff48a47 100644 --- a/onadata/settings/default_settings.py +++ b/onadata/settings/default_settings.py @@ -31,4 +31,4 @@ SLAVE_DATABASES = [] # Make a unique unique key just for testing, and don't share it with anybody. -SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" +SECRET_KEY = "mlfs33^s1l4xf6a36$0#j%dd*sisfoi&)&4s-v=91#^l01v)*j" # noqa diff --git a/onadata/settings/docker.py b/onadata/settings/docker.py index 558b73635a..b53ea61909 100644 --- a/onadata/settings/docker.py +++ b/onadata/settings/docker.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Example local_settings.py used by the Dockerfile. """ @@ -33,7 +33,7 @@ SLAVE_DATABASES = [] # Make a unique unique key just for testing, and don't share it with anybody. -SECRET_KEY = "~&nN9d`bxmJL2[$HhYE9qAk=+4P:cf3b" +SECRET_KEY = "~&nN9d`bxmJL2[$HhYE9qAk=+4P:cf3b" # noqa ALLOWED_HOSTS = ["127.0.0.1", "localhost"] diff --git a/onadata/settings/drone_test.py b/onadata/settings/drone_test.py index b611bab2cd..31d3bb96bd 100644 --- a/onadata/settings/drone_test.py +++ b/onadata/settings/drone_test.py @@ -1,4 +1,4 @@ -# -*- coding:utf-8 -*- +# -*- coding: utf-8 -*- """ Example local_settings.py for use with DroneCI. """ diff --git a/onadata/settings/production_example.py b/onadata/settings/production_example.py index ea57dcf820..aed2573692 100644 --- a/onadata/settings/production_example.py +++ b/onadata/settings/production_example.py @@ -1,4 +1,4 @@ -# -*- coding=utf-8 -*- +# -*- coding: utf-8 -*- """ Example local_settings.py used by the Dockerfile. """ diff --git a/onadata/settings/staging_example.py b/onadata/settings/staging_example.py index 9436355644..d46fd4d8bd 100644 --- a/onadata/settings/staging_example.py +++ b/onadata/settings/staging_example.py @@ -27,7 +27,7 @@ # TIME_ZONE = 'UTC' -SECRET_KEY = "please replace this text" +SECRET_KEY = "please replace this text" # noqa # This trick works only when we run tests from the command line. TESTING_MODE = len(sys.argv) >= 2 and (